Make Piggy part of the product rather than a guest in it
Piggy arrived as a chat panel bolted onto a CRM and then grew a workspace around it. The layout was already right — the audit found the approval card to be the best-designed object in the repo, and the account page's empty panels less finished than anything in the workspace. What was wrong was vocabulary: nobody had written the small things down, so both halves kept inventing them. Piggy was drawn with five different marks — a pig in the dock, a sparkle in the sidebar and again on the model picker, a speech bubble on the Ask buttons, and a stock robot glyph on every assistant message, which is the one people look at most. There is now one mark. The composer, which is the first control in the product since sign-in lands on /piggy, was the only un-adapted shadcn field left: 6px radius against a 12px Send button it sat 8px from. A stat tile had been reinvented six times at three numeral scales, and the same uppercase micro-label existed in five variants, two of them one tab apart in the same rail. There were 63 hand-written font sizes: not a scale, sixty-three opinions. Underneath that, the focus ring was invisible. The global rule used ring-accent, which Tailwind deliberately aliases onto the hover tint, so the ring measured 1.01:1 against the light canvas — no visible focus indicator anywhere in the product, for any accent, in either theme. It is ring-brand now and measures 17:1. The warning, positive and info tones were darkened until each clears 4.5:1 on a card, on inset and on its own chip, and the light canvas moved to 98% so a card lifts without leaning on its shadow. The mobile work is the part worth reading. A landscape phone gave the transcript 28% of the viewport and a keyboard-up phone 16%, against a 45% floor — and the fixed tab bar painted over the composer, covering the safety sentence and half the Send button, because two source comments asserted the bar stood down on short viewports and it never had. Both fixed and measured by hit-testing rather than by screenshot. The composer itself was 64px tall for a blank second line nobody typed, because the auto-resize effect sizes to scrollHeight and scrollHeight counts rows — a CSS height could not win against an inline style, so the attribute was the honest lever. Verified across both themes driven through the app's own control: no horizontal overflow on 15 routes at four viewports, 672 stat values that fit, 297 labels at exactly 11px/500, Escape returning focus to its opener rather than the body on every overlay, and a rejected write no longer reporting "Succeeded" with a green check. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com>
This commit is contained in:
claude
+516
-18
@@ -27,7 +27,12 @@ import {
|
||||
type PiggyTurnBudget,
|
||||
} from './agent/session';
|
||||
import { toPrimeTools } from './agent/tool-bridge';
|
||||
import { loadPiggyTurnLimits, type PiggyTurnLimits } from './config';
|
||||
import {
|
||||
loadPiggyStallLimits,
|
||||
loadPiggyTurnLimits,
|
||||
type PiggyStallLimits,
|
||||
type PiggyTurnLimits,
|
||||
} from './config';
|
||||
import { assertPigToolBoundary, type PiggyChatContext } from './chat';
|
||||
import { createInteractivePigTools } from './chat-tools';
|
||||
import { createPigWriteTools, type PigWriteToolDeps } from './write-tools';
|
||||
@@ -152,6 +157,12 @@ export interface PiggyChatServerOptions {
|
||||
* two that can disagree.
|
||||
*/
|
||||
limits?: PiggyTurnLimits;
|
||||
/**
|
||||
* How long a turn may go silent before the endpoint is presumed to have gone
|
||||
* quiet. Read from the environment when absent, for the same reason as
|
||||
* `limits`.
|
||||
*/
|
||||
stallLimits?: PiggyStallLimits;
|
||||
}
|
||||
|
||||
export function startPiggyChatServer(db: Database, options: PiggyChatServerOptions): Server {
|
||||
@@ -174,6 +185,7 @@ export function startPiggyChatServer(db: Database, options: PiggyChatServerOptio
|
||||
// Resolved once, at bind time, so a malformed ceiling fails the process
|
||||
// rather than the first user to ask a question.
|
||||
limits: options.limits ?? loadPiggyTurnLimits(),
|
||||
stall: options.stallLimits ?? loadPiggyStallLimits(),
|
||||
};
|
||||
const defaultModel = defaultModelOption(resolved.models);
|
||||
|
||||
@@ -223,6 +235,7 @@ interface ResolvedOptions {
|
||||
createReadTools: (db: Database, context: PiggyChatContext | undefined) => ToolDefinition[];
|
||||
approvals: ApprovalRegistry;
|
||||
limits: PiggyTurnLimits;
|
||||
stall: PiggyStallLimits;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -240,6 +253,210 @@ function defaultModelOption(models: readonly PiggyModelOption[]): PiggyModelOpti
|
||||
return first;
|
||||
}
|
||||
|
||||
// -------------------------------------------------------------- the stall
|
||||
|
||||
/** Which silence ended the turn. */
|
||||
type StallPhase = 'first_progress' | 'idle';
|
||||
|
||||
/** A turn the endpoint stopped answering, and how long it was given first. */
|
||||
interface TurnStall {
|
||||
phase: StallPhase;
|
||||
/** How long the turn had been silent when the deadline bit. */
|
||||
waitedMs: number;
|
||||
/** The deadline it passed, in its own units. */
|
||||
ceilingMs: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* The events that mean the model itself is working.
|
||||
*
|
||||
* Deliberately narrower than "any event". The harness announces `agent_start`
|
||||
* and `turn_start` the instant a prompt is submitted, before a byte has left the
|
||||
* process, so counting those as progress would start the first-progress clock
|
||||
* and satisfy it in the same tick — which is exactly the hang this guard is for.
|
||||
* Once a turn has genuinely started, any event at all is accepted as a sign of
|
||||
* life, because by then the harness is demonstrably running its loop.
|
||||
*/
|
||||
const PROGRESS_EVENTS: ReadonlySet<AgentSessionEvent['type']> = new Set([
|
||||
'message_update',
|
||||
'tool_execution_start',
|
||||
'turn_end',
|
||||
]);
|
||||
|
||||
/**
|
||||
* How long a stalled turn is given to unwind itself before the server stops
|
||||
* waiting for it.
|
||||
*
|
||||
* `session.abort()` should reject the in-flight request and settle `prompt()`
|
||||
* promptly. Should. The whole reason this guard exists is that the layer holding
|
||||
* the socket had no deadline of its own, so trusting the same layer to honour an
|
||||
* abort — and leaving the browser hanging if it does not — would rebuild the bug
|
||||
* one level up. Two seconds is long enough for a clean unwind and short enough
|
||||
* that nobody watches it.
|
||||
*/
|
||||
const STALL_UNWIND_GRACE_MS = 2_000;
|
||||
|
||||
/**
|
||||
* The turn-level stall detector.
|
||||
*
|
||||
* It watches the session's event stream rather than the HTTP call, because the
|
||||
* HTTP call belongs to the harness and this must be a guard the harness cannot
|
||||
* swallow. Two deadlines, and the distinction is the point of the whole class:
|
||||
*
|
||||
* first progress — nothing has arrived since `prompt()` was called. The turn
|
||||
* never started; the request went out and the endpoint did
|
||||
* not answer.
|
||||
* idle — the turn started and then went quiet. This clock restarts
|
||||
* on every event, so a long answer that keeps arriving runs
|
||||
* as long as it likes. A flat overall deadline would kill
|
||||
* exactly the legitimate long turns this product wants.
|
||||
*
|
||||
* `parked` is the third rule and the one that would otherwise break the write
|
||||
* flow. A `confirm`-mode turn sits inside `propose()` waiting up to five minutes
|
||||
* for a human, and by design that produces no events whatsoever. That is not a
|
||||
* stall, it is the product working, so the clock is pushed forward for as long
|
||||
* as the rendezvous holds an unanswered card.
|
||||
*/
|
||||
class TurnStallWatchdog {
|
||||
private phaseStartedAt = Date.now();
|
||||
private progressed = false;
|
||||
private timer?: ReturnType<typeof setTimeout>;
|
||||
private stopped = false;
|
||||
private fired = false;
|
||||
private abandon: (() => void) | undefined;
|
||||
|
||||
/**
|
||||
* Resolves only when a stalled turn's harness has not unwound within the
|
||||
* grace period. Raced against `prompt()` so a hung request cannot hold the
|
||||
* browser open even if the abort is ignored.
|
||||
*/
|
||||
readonly abandoned: Promise<void>;
|
||||
|
||||
constructor(
|
||||
private readonly limits: PiggyStallLimits,
|
||||
/** True while a proposed write is waiting on a human. */
|
||||
private readonly parked: () => boolean,
|
||||
private readonly onStall: (stall: TurnStall) => void,
|
||||
) {
|
||||
this.abandoned = new Promise<void>((resolve) => {
|
||||
this.abandon = resolve;
|
||||
});
|
||||
}
|
||||
|
||||
/** Starts the first-progress clock. Call immediately before `prompt()`. */
|
||||
start(): void {
|
||||
this.phaseStartedAt = Date.now();
|
||||
this.arm();
|
||||
}
|
||||
|
||||
/** One event off the session stream. */
|
||||
observe(type: AgentSessionEvent['type']): void {
|
||||
const wasProgressing = this.progressed;
|
||||
if (PROGRESS_EVENTS.has(type)) this.progressed = true;
|
||||
else if (!this.progressed) return;
|
||||
this.phaseStartedAt = Date.now();
|
||||
if (!wasProgressing) {
|
||||
/*
|
||||
* The one reset that must not wait for the sleeping timer.
|
||||
*
|
||||
* Every other reset only ever pushes the deadline later, and a timer that
|
||||
* wakes too early simply re-arms for what is left. The first progress
|
||||
* event is different: it swaps the generous first-progress window for the
|
||||
* tighter idle one, and a timer already asleep on the longer of the two
|
||||
* cannot notice until it has expired. Measured, before this line existed:
|
||||
* a turn with a 120ms idle deadline that fell silent after its first token
|
||||
* ran for 30,001ms — the first-progress window — which is the hang this
|
||||
* whole guard is for, wearing the guard's own clothes.
|
||||
*/
|
||||
this.rearm();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The human rendezvous moved — a card was raised, or answered.
|
||||
*
|
||||
* Not a model event, but not silence either, and it closes the narrow window
|
||||
* where an approval settles a moment before the deadline it was suspending
|
||||
* would have fired.
|
||||
*/
|
||||
touch(): void {
|
||||
this.phaseStartedAt = Date.now();
|
||||
}
|
||||
|
||||
stop(): void {
|
||||
this.stopped = true;
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
}
|
||||
|
||||
private window(): number {
|
||||
return this.progressed ? this.limits.idleMs : this.limits.firstProgressMs;
|
||||
}
|
||||
|
||||
private rearm(): void {
|
||||
if (this.timer) clearTimeout(this.timer);
|
||||
this.arm();
|
||||
}
|
||||
|
||||
/**
|
||||
* One self-rescheduling timer rather than a poll: it sleeps exactly as long as
|
||||
* the current deadline has left, and every reset simply moves the deadline it
|
||||
* wakes up to compare against.
|
||||
*/
|
||||
private arm(): void {
|
||||
if (this.stopped || this.fired) return;
|
||||
const waited = Date.now() - this.phaseStartedAt;
|
||||
const ceilingMs = this.window();
|
||||
if (waited < ceilingMs) {
|
||||
this.timer = setTimeout(() => this.arm(), ceilingMs - waited);
|
||||
// A deadline must never be a reason for the process to stay alive.
|
||||
this.timer.unref?.();
|
||||
return;
|
||||
}
|
||||
if (this.parked()) {
|
||||
// Waiting on a person, not on the endpoint. The clock restarts from now,
|
||||
// so the turn gets a full window once the card is answered.
|
||||
this.phaseStartedAt = Date.now();
|
||||
this.arm();
|
||||
return;
|
||||
}
|
||||
this.fired = true;
|
||||
this.onStall({ phase: this.progressed ? 'idle' : 'first_progress', waitedMs: waited, ceilingMs });
|
||||
this.timer = setTimeout(() => this.abandon?.(), STALL_UNWIND_GRACE_MS);
|
||||
this.timer.unref?.();
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Tell the user the endpoint went quiet, and tell the ledger which silence it
|
||||
* was.
|
||||
*
|
||||
* A distinct code, because an operator must be able to tell three different
|
||||
* events apart without reading a log: `inference_stalled` is the endpoint saying
|
||||
* nothing, `inference_failed` is the model or the endpoint reporting a fault,
|
||||
* and `turn_limit_exceeded` is PIG's own policy stopping a turn that was working
|
||||
* perfectly well. They want three different responses — wait, investigate, and
|
||||
* nothing at all — so they must not share a name.
|
||||
*/
|
||||
function reportStall(
|
||||
stall: TurnStall,
|
||||
spend: ChatRunOutcome,
|
||||
emit: (event: PiggyChatEvent) => void,
|
||||
): void {
|
||||
spend.stall = stall;
|
||||
spend.error =
|
||||
`turn stopped by the ${stall.phase} deadline: no session event for ${stall.waitedMs}ms, ` +
|
||||
`deadline ${stall.ceilingMs}ms`;
|
||||
console.warn(`[piggy] ${spend.error}`);
|
||||
emit({
|
||||
type: 'error',
|
||||
message:
|
||||
stall.phase === 'first_progress'
|
||||
? 'Piggy asked the inference endpoint and it never answered, so nothing was attempted. That is the endpoint rather than your question — try again shortly.'
|
||||
: 'The inference endpoint went quiet part way through this answer, so it is incomplete. Try again shortly.',
|
||||
code: 'inference_stalled',
|
||||
});
|
||||
}
|
||||
|
||||
// ------------------------------------------------------------------- the turn
|
||||
|
||||
async function handleChatTurn(
|
||||
@@ -319,8 +536,16 @@ async function handleChatTurn(
|
||||
|
||||
beginStream(response);
|
||||
|
||||
// Declared before `emit` because `emit` feeds it: an approval being raised or
|
||||
// answered is movement on the turn, and it is the only movement the session's
|
||||
// own event stream never reports.
|
||||
let watchdog: TurnStallWatchdog | undefined;
|
||||
|
||||
const emit = (event: PiggyChatEvent): void => {
|
||||
recordEvent(spend, event);
|
||||
if (event.type === 'approval_required' || event.type === 'approval_resolved') {
|
||||
watchdog?.touch();
|
||||
}
|
||||
// An abandoned turn still has approvals to cancel and a session to unwind,
|
||||
// and both emit as they settle. Writing those to a closed socket would
|
||||
// throw inside the unwinding and take the ledger down with it.
|
||||
@@ -344,7 +569,7 @@ async function handleChatTurn(
|
||||
* every turn that went wrong is answering the wrong question, and it errs in
|
||||
* the reassuring direction, which is the worst way for it to be wrong.
|
||||
*/
|
||||
const state: TurnState = { inputTokens: 0, outputTokens: 0, modelCalls: 0 };
|
||||
const state: TurnState = { inputTokens: 0, outputTokens: 0, modelCalls: 0, retries: 0 };
|
||||
try {
|
||||
const tools = buildToolSet(db, options, {
|
||||
principal,
|
||||
@@ -395,32 +620,94 @@ async function handleChatTurn(
|
||||
void session?.session.abort().catch(() => {});
|
||||
};
|
||||
|
||||
/*
|
||||
* The third brake, and the only one that fires when the harness is not
|
||||
* running at all.
|
||||
*
|
||||
* `shouldStopAfterTurn` and the `turn_end` counter above both need the loop
|
||||
* to be turning; a turn stuck on a request the endpoint never answers turns
|
||||
* nothing, spends nothing and trips neither. It is stopped the same way a
|
||||
* runaway is — abort the session — and reported as its own thing, because
|
||||
* "the endpoint went quiet" and "the turn cost too much" are not the same
|
||||
* news for either the user or the operator.
|
||||
*/
|
||||
watchdog = new TurnStallWatchdog(options.stall, turn.hasPending, (stall) => {
|
||||
if (state.stopping) return;
|
||||
state.stopping = true;
|
||||
state.stalled = stall;
|
||||
// Fire and forget, for the same reason the budget's abort is: this runs on
|
||||
// a timer inside the run, and awaiting the run going idle from within it
|
||||
// would deadlock.
|
||||
void session?.session.abort().catch(() => {});
|
||||
});
|
||||
|
||||
/*
|
||||
* The fourth brake, and the only one that stops the turn to protect the
|
||||
* transcript rather than the bill.
|
||||
*
|
||||
* `piggyAgentSettings` buys most of its retries inside the SDK's request
|
||||
* wrapper, where nothing has been delivered yet and a retry is invisible by
|
||||
* construction. This is the other kind: the harness's session-level retry
|
||||
* fires after the response has already started, discards the errored
|
||||
* assistant message and generates a replacement. Measured against a stubbed
|
||||
* endpoint, a turn that had streamed "Idle is " before the stream dropped
|
||||
* came back as "Idle is Idle is $12,000." — the reader is shown a sentence
|
||||
* twice, in a panel whose whole job is to be trusted about numbers.
|
||||
*
|
||||
* So the rule is the simple one rather than the clever one: retry only
|
||||
* before the first delta. Once anything has been delivered the turn is
|
||||
* stopped and reported as incomplete, which is what the stall guard already
|
||||
* says about a half-finished answer and what the reader can act on. Aborting
|
||||
* during the backoff is enough: `AgentSession.abort` cancels the retry sleep
|
||||
* before it re-drives (agent-session.js:1168-1172).
|
||||
*/
|
||||
const refuseReplay = (): void => {
|
||||
if (!state.delivered || state.stopping) return;
|
||||
state.stopping = true;
|
||||
state.replayRefused = true;
|
||||
// Fire and forget, for the same reason the other two brakes are: this
|
||||
// runs inside the run's own event handling, and `abort` awaits the run
|
||||
// going idle.
|
||||
void session?.session.abort().catch(() => {});
|
||||
};
|
||||
|
||||
const unsubscribe = session.session.subscribe((event) => {
|
||||
watchdog?.observe(event.type);
|
||||
translateSessionEvent(event, emit, state);
|
||||
if (event.type === 'turn_end') enforceBudget();
|
||||
if (event.type === 'auto_retry_start') refuseReplay();
|
||||
});
|
||||
try {
|
||||
await session.session.prompt(body.message);
|
||||
watchdog.start();
|
||||
const prompt = session.session.prompt(body.message);
|
||||
// Two handlers on one promise. The race is what ends the turn when an
|
||||
// aborted harness does not unwind; this `catch` is what keeps the prompt's
|
||||
// eventual rejection from becoming an unhandled rejection once the race
|
||||
// has already been decided against it. Awaiting `prompt` still throws, so
|
||||
// the ordinary failure path is untouched.
|
||||
void prompt.catch(() => {});
|
||||
await Promise.race([prompt, watchdog.abandoned]);
|
||||
} finally {
|
||||
unsubscribe();
|
||||
watchdog.stop();
|
||||
abort.signal.removeEventListener('abort', disposeOnAbort);
|
||||
}
|
||||
|
||||
if (abort.signal.aborted) {
|
||||
if (state.stalled) {
|
||||
// Checked before the abort state, and it costs nothing to do so: a reader
|
||||
// who had already left would have disposed the session and settled the
|
||||
// prompt long before this deadline could bite, so a stall recorded here
|
||||
// came first and is the reason the turn ended.
|
||||
reportStall(state.stalled, spend, emit);
|
||||
} else if (abort.signal.aborted) {
|
||||
// The reader left. The harness may have resolved the prompt rather than
|
||||
// rejecting it, and recording that as a completed turn would report an
|
||||
// answer nobody received as delivered.
|
||||
spend.aborted = true;
|
||||
} else if (cutShortByBudget(budget, state)) {
|
||||
reportBudgetBreach(budget, spend, emit);
|
||||
} else if (state.errorMessage !== undefined) {
|
||||
// The model stopped on a fault of its own rather than throwing, so the
|
||||
// turn ends as an error frame and the run is recorded as failed. A `done`
|
||||
// here would present a truncated answer as a finished one. The ledger
|
||||
// keeps the real reason; the browser gets the sanitised one.
|
||||
spend.error = state.errorMessage;
|
||||
console.error('[piggy] chat turn ended in an inference error:', state.errorMessage);
|
||||
emit({ type: 'error', message: 'Piggy could not finish this answer.', code: 'inference_failed' });
|
||||
} else if (state.replayRefused || state.errorMessage !== undefined) {
|
||||
reportInferenceFailure(state, spend, emit);
|
||||
} else {
|
||||
// A turn that passed a ceiling on its own last call still records the
|
||||
// breach below, because that is the reading an operator tuning the
|
||||
@@ -446,13 +733,27 @@ async function handleChatTurn(
|
||||
// Server-side, with the real reason. The client gets none of it: the
|
||||
// upstream body is echoed into these messages and is not ours to relay.
|
||||
console.error('[piggy] chat turn failed:', failure);
|
||||
if (!spend.aborted && budget.breach) {
|
||||
if (!spend.aborted && state.stalled) {
|
||||
// The abort this server fires at a silent endpoint usually surfaces here,
|
||||
// as a rejected prompt rather than a resolved one. What ended the turn is
|
||||
// still the stall, and saying "Piggy chat failed" would send an operator
|
||||
// hunting a fault in code that behaved correctly.
|
||||
reportStall(state.stalled, spend, emit);
|
||||
if (!response.writableEnded) response.end();
|
||||
} else if (!spend.aborted && budget.breach) {
|
||||
// The abort this server fires to stop a runaway can surface here as a
|
||||
// rejected prompt rather than as a resolved one. The ceiling is what
|
||||
// ended the turn; reporting it as "Piggy chat failed" would send an
|
||||
// operator hunting a fault that is really a policy.
|
||||
reportBudgetBreach(budget, spend, emit);
|
||||
if (!response.writableEnded) response.end();
|
||||
} else if (!spend.aborted && (state.replayRefused || state.errorMessage !== undefined)) {
|
||||
// The abort this server fires to refuse a replay surfaces here as a
|
||||
// rejected prompt. What ended the turn is the upstream fault that
|
||||
// provoked the retry, and the reader is owed that reason rather than a
|
||||
// bare "Piggy chat failed" for a decision this server took on purpose.
|
||||
reportInferenceFailure(state, spend, emit);
|
||||
if (!response.writableEnded) response.end();
|
||||
} else {
|
||||
spend.error = failure;
|
||||
if (!response.writableEnded) {
|
||||
@@ -481,6 +782,10 @@ async function handleChatTurn(
|
||||
spend.costMicroCents ??= costMicroCents(state, model);
|
||||
spend.modelCalls = state.modelCalls;
|
||||
spend.overran = budget.overran;
|
||||
// `??=` because a reported failure has already written the reading it was
|
||||
// measured against, and a turn that recovered still needs its count here.
|
||||
spend.attempts ??= state.retries + 1;
|
||||
spend.retryReason ??= state.retryReason;
|
||||
// In a finally so that every exit closes the row, including the exit
|
||||
// that is not a fault at all: a reader who navigates away aborts the
|
||||
// turn mid-answer. A row left `running` cannot be told from a turn still
|
||||
@@ -530,6 +835,25 @@ interface TurnState {
|
||||
errorMessage?: string;
|
||||
/** The session has already been told to stop; do not tell it twice. */
|
||||
stopping?: boolean;
|
||||
/** Set when the stall watchdog, rather than the model, ended the turn. */
|
||||
stalled?: TurnStall;
|
||||
/**
|
||||
* Turn-level retries the harness announced, which is one per `auto_retry_start`.
|
||||
*
|
||||
* It counts the retries that were visible. The provider-level ones — the four
|
||||
* attempts `piggyAgentSettings` buys inside the SDK's own request wrapper —
|
||||
* produce no assistant message and no session event, so no honest counter can
|
||||
* see them from here. An operator reading `agent_runs.result.inference` should
|
||||
* therefore read `attempts` as "times this turn had to be started again after
|
||||
* the endpoint had already begun answering", not as a count of HTTP requests.
|
||||
*/
|
||||
retries: number;
|
||||
/** How the endpoint described the fault that caused the last retry. */
|
||||
retryReason?: string;
|
||||
/** Content deltas have been written to the reader's transcript. */
|
||||
delivered?: boolean;
|
||||
/** A retry was refused because it would have replayed a delivered answer. */
|
||||
replayRefused?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
@@ -578,6 +902,75 @@ function reportBudgetBreach(
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Is this the endpoint throttling us, or something that will fail again?
|
||||
*
|
||||
* Matched on the text because that is all there is: the harness reports a
|
||||
* failed model call as an `errorMessage` on the assistant message and keeps no
|
||||
* status code, so by the time the fault reaches PIG the HTTP response is long
|
||||
* gone. The strings are the ones Prime Inference actually sends — `429:
|
||||
* {"message":"Rate limit reached. Please retry shortly.","type":
|
||||
* "rate_limit_exceeded","code":"rate_limited"}` — plus the shapes the SDK
|
||||
* substitutes when it never got a body, and `ResourceExhausted`, which is what
|
||||
* a gRPC-backed model behind the same endpoint says instead.
|
||||
*/
|
||||
function isRateLimited(errorMessage: string): boolean {
|
||||
return /\b429\b|rate.?limit|rate_limited|too many requests|resourceexhausted/i.test(errorMessage);
|
||||
}
|
||||
|
||||
/**
|
||||
* Tell the user which fault it was, and tell the ledger how hard we tried.
|
||||
*
|
||||
* `inference_rate_limited` is its own code because it is the one inference
|
||||
* fault the person at the keyboard can do something about: waiting ten seconds
|
||||
* genuinely fixes it, and it is not worth an operator's pager. `inference_failed`
|
||||
* keeps its old meaning — something broke and somebody should look — so the two
|
||||
* must not share a name any more than `inference_stalled` and
|
||||
* `turn_limit_exceeded` do.
|
||||
*
|
||||
* The counts go to `agent_runs`, not to the browser. "Piggy tried four times"
|
||||
* is not a sentence that helps anybody decide what to type next; it is exactly
|
||||
* the sentence an operator needs when deciding whether today's rate limiting is
|
||||
* worse than yesterday's.
|
||||
*/
|
||||
function reportInferenceFailure(
|
||||
state: TurnState,
|
||||
spend: ChatRunOutcome,
|
||||
emit: (event: PiggyChatEvent) => void,
|
||||
): void {
|
||||
const errorMessage = state.errorMessage ?? 'the endpoint failed without saying why';
|
||||
const rateLimited = isRateLimited(errorMessage);
|
||||
const attempts = state.retries + 1;
|
||||
// Server-side, with the real reason and the counts. The upstream body is
|
||||
// echoed into this and is not ours to relay to a browser.
|
||||
spend.error =
|
||||
`${errorMessage} (${attempts} attempt${attempts === 1 ? '' : 's'}` +
|
||||
(state.replayRefused ? ', retry refused: part of the answer had already been delivered' : '') +
|
||||
')';
|
||||
spend.attempts = attempts;
|
||||
console.error('[piggy] chat turn ended in an inference error:', spend.error);
|
||||
|
||||
if (state.replayRefused) {
|
||||
// The reader keeps what arrived, and is told plainly that it is not all of
|
||||
// it. Restarting would have shown them the first half twice.
|
||||
emit({
|
||||
type: 'error',
|
||||
message: rateLimited
|
||||
? 'The inference endpoint started rate limiting us part way through this answer, so it is incomplete. Piggy will not restart it, because that would repeat what you have already been shown. Ask again in a few seconds.'
|
||||
: 'The inference endpoint failed part way through this answer, so it is incomplete. Piggy will not restart it, because that would repeat what you have already been shown. Ask again shortly.',
|
||||
code: rateLimited ? 'inference_rate_limited' : 'inference_failed',
|
||||
});
|
||||
return;
|
||||
}
|
||||
emit({
|
||||
type: 'error',
|
||||
message: rateLimited
|
||||
? `Prime Inference is rate limiting us, so this question was never answered. Piggy asked ${attempts} time${attempts === 1 ? '' : 's'} and was turned away each time. Wait a few seconds and ask again; the refused attempts generated nothing, so none of this was charged to you.`
|
||||
: 'Piggy could not finish this answer.',
|
||||
code: rateLimited ? 'inference_rate_limited' : 'inference_failed',
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* The harness's vocabulary, narrowed to PIG's.
|
||||
*
|
||||
@@ -594,11 +987,53 @@ function translateSessionEvent(
|
||||
): void {
|
||||
switch (event.type) {
|
||||
case 'message_update': {
|
||||
/*
|
||||
* Nothing more of this turn may reach the transcript once a replay has
|
||||
* been refused.
|
||||
*
|
||||
* Aborting the session should be enough, and the whole reason that guard
|
||||
* exists is that a harness which cannot unwind is precisely the fault
|
||||
* being defended against — the stall watchdog already races `prompt()`
|
||||
* for the same reason. A harness that carried on regardless would stream
|
||||
* the replacement answer over the top of the one the reader already has,
|
||||
* which is the duplication the abort was meant to prevent. Dropping the
|
||||
* deltas here makes exactly-once a property of PIG's own code rather than
|
||||
* a favour from somebody else's.
|
||||
*/
|
||||
if (state.replayRefused) return;
|
||||
const streamed = event.assistantMessageEvent;
|
||||
if (streamed.type === 'text_delta') emit({ type: 'content_delta', delta: streamed.delta });
|
||||
if (streamed.type === 'thinking_delta') emit({ type: 'reasoning_delta', delta: streamed.delta });
|
||||
// Recorded before either is written, because it is what makes a retry
|
||||
// unsafe: from here on the reader has part of a turn in front of them,
|
||||
// and starting it again would show them a second copy. Reasoning counts
|
||||
// as well as answer text — it is a rendered panel, not a private
|
||||
// scratchpad, and a restarted turn would repeat that too.
|
||||
if (streamed.type === 'text_delta') {
|
||||
state.delivered = true;
|
||||
emit({ type: 'content_delta', delta: streamed.delta });
|
||||
}
|
||||
if (streamed.type === 'thinking_delta') {
|
||||
state.delivered = true;
|
||||
emit({ type: 'reasoning_delta', delta: streamed.delta });
|
||||
}
|
||||
return;
|
||||
}
|
||||
/*
|
||||
* The harness is about to restart a turn that failed.
|
||||
*
|
||||
* Announced, backed off and then re-driven by `AgentSession`, which
|
||||
* discards the errored assistant message and continues. It is counted here
|
||||
* rather than dropped with the rest of the harness's vocabulary because a
|
||||
* turn that needed two goes and a turn that needed one look identical in
|
||||
* the ledger otherwise, and "how often is this happening" is the first
|
||||
* question an operator asks about a rate limit.
|
||||
*/
|
||||
case 'auto_retry_start':
|
||||
state.retries += 1;
|
||||
state.retryReason = event.errorMessage;
|
||||
console.warn(
|
||||
`[piggy] retrying a chat turn after ${event.errorMessage} (attempt ${event.attempt} of ${event.maxAttempts}, in ${event.delayMs}ms)`,
|
||||
);
|
||||
return;
|
||||
case 'tool_execution_start':
|
||||
emit({ type: 'tool_call', id: event.toolCallId, name: event.toolName, arguments: event.args });
|
||||
return;
|
||||
@@ -635,7 +1070,21 @@ function translateSessionEvent(
|
||||
state.inputTokens += assistant.inputTokens;
|
||||
state.outputTokens += assistant.outputTokens;
|
||||
state.stopReason = assistant.stopReason ?? state.stopReason;
|
||||
if (assistant.errorMessage) state.errorMessage = assistant.errorMessage;
|
||||
// Set on a fault and CLEARED on the next model call that is not one.
|
||||
//
|
||||
// It used to latch, and that was the second half of the production
|
||||
// failure: the harness retries a 429 of its own accord and often
|
||||
// succeeds, but the errored `turn_end` had already stamped this field, so
|
||||
// a turn that recovered and streamed a perfectly good answer was still
|
||||
// closed as `inference_failed` with the 429 in the ledger. The only way a
|
||||
// model call can follow an errored one at all is that something retried
|
||||
// it, so the later reading is the one that describes how the turn ended.
|
||||
//
|
||||
// Unless the retry was one this server refused. Then the replacement
|
||||
// answer was never delivered, whatever the harness did with it, and
|
||||
// letting its success clear the fault would report a turn as finished on
|
||||
// the strength of an answer nobody was shown.
|
||||
if (!state.replayRefused) state.errorMessage = assistant.errorMessage;
|
||||
return;
|
||||
}
|
||||
default:
|
||||
@@ -741,7 +1190,20 @@ class ApprovalRegistry {
|
||||
open(
|
||||
conversationId: string,
|
||||
emit: (event: PiggyChatEvent) => void,
|
||||
): { propose: PigWriteToolDeps['propose']; cancelAll: () => void } {
|
||||
): {
|
||||
propose: PigWriteToolDeps['propose'];
|
||||
cancelAll: () => void;
|
||||
/**
|
||||
* True while this turn is waiting on a human.
|
||||
*
|
||||
* Published because the stall watchdog has to be able to tell a turn parked
|
||||
* on `propose()` from a turn nobody is answering: the parked one emits
|
||||
* nothing for up to five minutes by design, and killing it would break the
|
||||
* write flow entirely. `owned` is the honest source — it holds exactly the
|
||||
* cards this turn has raised and not yet settled.
|
||||
*/
|
||||
hasPending: () => boolean;
|
||||
} {
|
||||
const owned = new Set<string>();
|
||||
|
||||
const propose: PigWriteToolDeps['propose'] = (draft) =>
|
||||
@@ -770,7 +1232,7 @@ class ApprovalRegistry {
|
||||
for (const key of [...owned]) this.pending.get(key)?.settle('reject', 'cancelled');
|
||||
};
|
||||
|
||||
return { propose, cancelAll };
|
||||
return { propose, cancelAll, hasPending: () => owned.size > 0 };
|
||||
}
|
||||
|
||||
/** False when nothing is pending: an unknown id, or one already settled. */
|
||||
@@ -871,6 +1333,20 @@ interface ChatRunOutcome {
|
||||
modelCalls?: number;
|
||||
/** The ceiling this turn passed, if it passed one. */
|
||||
breach?: PiggyTurnBreach;
|
||||
/** The silence that ended this turn, if one did. */
|
||||
stall?: TurnStall;
|
||||
/**
|
||||
* Times the turn had to be started again, the first go included.
|
||||
*
|
||||
* Written for every turn rather than only the failed ones, because the
|
||||
* question an operator has is "how often is the endpoint making us retry",
|
||||
* and a column that only ever appears on failures cannot answer it: a day
|
||||
* where every turn needed two attempts and succeeded looks, in that ledger,
|
||||
* exactly like a day where none of them did.
|
||||
*/
|
||||
attempts?: number;
|
||||
/** How the endpoint described the fault behind the last of them. */
|
||||
retryReason?: string;
|
||||
/** The loop kept going after the ceiling and had to be aborted. */
|
||||
overran?: boolean;
|
||||
}
|
||||
@@ -954,6 +1430,14 @@ async function finishChatRun(
|
||||
approvalsRequested: outcome.approvalsRequested,
|
||||
approvalsApplied: outcome.approvalsApplied,
|
||||
modelCalls: outcome.modelCalls ?? 0,
|
||||
// What it took to get an answer at all. `attempts: 1` is the healthy
|
||||
// reading and the common one; anything above it is the endpoint
|
||||
// making the product slower, which is a trend rather than an
|
||||
// incident and so belongs in a column rather than in a log line.
|
||||
inference: {
|
||||
attempts: outcome.attempts ?? 1,
|
||||
...(outcome.retryReason ? { retryReason: outcome.retryReason } : {}),
|
||||
},
|
||||
...(outcome.breach
|
||||
? {
|
||||
limit: {
|
||||
@@ -965,6 +1449,20 @@ async function finishChatRun(
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
// A stalled turn closes as `failed`, not as `aborted`: unlike a
|
||||
// ceiling, which is PIG stopping a turn that was working, this is the
|
||||
// upstream not answering, and an operator watching the failure rate
|
||||
// should see it. `stall` says which silence it was, so it can still be
|
||||
// told from a fault without reading a log.
|
||||
...(outcome.stall
|
||||
? {
|
||||
stall: {
|
||||
phase: outcome.stall.phase,
|
||||
waitedMs: outcome.stall.waitedMs,
|
||||
ceilingMs: outcome.stall.ceilingMs,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
},
|
||||
inputTokens: outcome.inputTokens ?? null,
|
||||
outputTokens: outcome.outputTokens ?? null,
|
||||
|
||||
Reference in New Issue
Block a user