Capture harness, fixture verification, CI, and the public README
The site does no live inference. Rollouts are captured once against spark-1 and replayed at their recorded wall-clock — a public demo with no auth cannot hold an API key, and a recorded run can be scrubbed, permalinked, blind-compared and verified in ways a live one cannot. What stops it being a video is that the browser re-derives every number from the recorded moves. verify_fixtures.py is the Python half of that: it replays every committed fixture through the engine and reproduces its own rewards. All 16 land at delta 0.0. A fixture that cannot be regenerated is a claim with no receipt. First real measurement, thinking off, 8 seeds: solved 0/8. The model repeats guesses it has already played, invents words (trape, slith, postt, boomy), and contradicts its own feedback — consistency 0.09 to 0.17. That is the published failure taxonomy showing up in our own data on the first run, and it is why `consistency` is a reward component rather than a footnote. A capture failure is recorded as a turn with a null reply, never dropped. A capture that silently discarded failed turns would be reporting a better model than the one that ran. CI gates both halves and four things that fail silently in production: the word lists must rebuild byte-identically, the prerendered routes must carry their own baked og tags (crawlers do not run JS, so without them every shared link previews as the homepage), no blob: URL may reach the bundle (the site's CSP has no worker-src, so it falls back to default-src 'self' and a blob worker is blocked with no error), and the conformance digest must match across languages. Co-Authored-By: Claude Opus 5 (1M context) <noreply@anthropic.com> Claude-Session: https://claude.ai/code/session_019mt6sHQHEnEYrJZvoMCJSB
This commit is contained in:
+10
-41
@@ -15,7 +15,7 @@
|
||||
* site is arguing.
|
||||
*/
|
||||
|
||||
import { Component, type ReactNode } from 'react';
|
||||
import type { ReactNode } from 'react';
|
||||
import {
|
||||
createBrowserRouter,
|
||||
isRouteErrorResponse,
|
||||
@@ -27,6 +27,7 @@ import {
|
||||
type RouteObject,
|
||||
} from 'react-router-dom';
|
||||
import { getDemo, loadDemoModule } from '@/lib/demo-kit/registry';
|
||||
import { DemoErrorBoundary } from '@/components/demo/DemoErrorBoundary';
|
||||
import { SiteFooter } from '@/components/site/SiteFooter';
|
||||
import { SiteHeader } from '@/components/site/SiteHeader';
|
||||
import { SkipLink } from '@/components/site/SkipLink';
|
||||
@@ -132,7 +133,7 @@ function RootErrorBoundary(): ReactNode {
|
||||
);
|
||||
}
|
||||
|
||||
function DemoErrorBoundary(): ReactNode {
|
||||
function DemoRouteError(): ReactNode {
|
||||
const error = useRouteError();
|
||||
if (isRouteErrorResponse(error) && error.status === 404) {
|
||||
return (
|
||||
@@ -153,42 +154,6 @@ function DemoErrorBoundary(): ReactNode {
|
||||
);
|
||||
}
|
||||
|
||||
/**
|
||||
* Catches errors thrown while a demo's own components RENDER.
|
||||
*
|
||||
* The route error boundary above only sees loader and lazy-import failures; a
|
||||
* demo whose `Surface` throws on a malformed board state would still white-page
|
||||
* the app without this.
|
||||
*/
|
||||
export class DemoRenderBoundary extends Component<
|
||||
{ children: ReactNode },
|
||||
{ error: Error | null }
|
||||
> {
|
||||
override state: { error: Error | null } = { error: null };
|
||||
|
||||
static getDerivedStateFromError(error: unknown): { error: Error } {
|
||||
return { error: error instanceof Error ? error : new Error(String(error)) };
|
||||
}
|
||||
|
||||
override componentDidCatch(error: unknown): void {
|
||||
console.error('[demo] render failed', error);
|
||||
}
|
||||
|
||||
override render(): ReactNode {
|
||||
const { error } = this.state;
|
||||
if (error) {
|
||||
return (
|
||||
<ErrorCard
|
||||
heading="This demo failed to render"
|
||||
body="Only this demo is affected. The recorded runs and the environment source in the repository are unaffected by a bug in the viewer."
|
||||
detail={error.message}
|
||||
/>
|
||||
);
|
||||
}
|
||||
return this.props.children;
|
||||
}
|
||||
}
|
||||
|
||||
/** Shown while a lazy route's chunk is in flight. */
|
||||
function RouteFallback(): ReactNode {
|
||||
return (
|
||||
@@ -244,14 +209,18 @@ export const routes: RouteObject[] = [
|
||||
{
|
||||
path: 'demos/:slug',
|
||||
loader: demoLoader,
|
||||
errorElement: <DemoErrorBoundary />,
|
||||
errorElement: <DemoRouteError />,
|
||||
// Two boundaries, because they catch different things. `errorElement`
|
||||
// above catches a loader or chunk failure; this one catches a demo whose
|
||||
// own Surface throws while rendering a board state, which the router
|
||||
// never sees.
|
||||
lazy: async () => {
|
||||
const { default: DemoPage } = await import('@/pages/DemoPage');
|
||||
return {
|
||||
Component: () => (
|
||||
<DemoRenderBoundary>
|
||||
<DemoErrorBoundary>
|
||||
<DemoPage />
|
||||
</DemoRenderBoundary>
|
||||
</DemoErrorBoundary>
|
||||
),
|
||||
};
|
||||
},
|
||||
|
||||
Reference in New Issue
Block a user