diff --git a/apps/web/src/components/motion/StageRail.tsx b/apps/web/src/components/motion/StageRail.tsx index 975c390..9ace0ed 100644 --- a/apps/web/src/components/motion/StageRail.tsx +++ b/apps/web/src/components/motion/StageRail.tsx @@ -18,12 +18,17 @@ import { DEMAND_OPEN_STAGES, DEMAND_STAGE_LABELS, type DemandStage } from '@pig/core'; import { cn } from '@/components/ui'; +/** Zero takes the plural, as it does in English: "no templates", "0 templates". */ +function noun(count: number, [one, many]: readonly [string, string]): string { + return count === 1 ? one : many; +} + export function StageRail({ counts, coverage, stages = DEMAND_OPEN_STAGES, - countLabel = 'engagements', - coverageLabel = 'templates', + countNoun = ['engagement', 'engagements'], + coverageNoun = ['template', 'templates'], activeStage, onSelect, className, @@ -33,8 +38,9 @@ export function StageRail({ /** The second figure, if the caller has one — library cover, typically. */ coverage?: Partial>; stages?: readonly DemandStage[]; - countLabel?: string; - coverageLabel?: string; + /** Singular and plural, because "1 engagements" is read aloud by a screen reader. */ + countNoun?: readonly [one: string, many: string]; + coverageNoun?: readonly [one: string, many: string]; activeStage?: DemandStage | null; /** Omit to render a read-only rail: a non-interactive button is a trap. */ onSelect?: (stage: DemandStage) => void; @@ -61,7 +67,7 @@ export function StageRail({ // surface — it is where the motion stops repeating — so it is // called out rather than shown as another grey zero. - {covered === 0 ? `No ${coverageLabel}` : `${covered} ${coverageLabel}`} + {covered === 0 ? `No ${coverageNoun[1]}` : `${covered} ${noun(covered, coverageNoun)}`} )} @@ -79,13 +85,13 @@ export function StageRail({ type="button" onClick={() => onSelect(stage)} aria-pressed={active} - aria-label={`${DEMAND_STAGE_LABELS[stage]}: ${count} ${countLabel}`} + aria-label={`${DEMAND_STAGE_LABELS[stage]}: ${count} ${noun(count, countNoun)}`} className={cn(shape, 'transition-colors hover:bg-surface-2')} > {body} ) : ( -
+
{body}
)} diff --git a/apps/web/src/pages/Engagement.tsx b/apps/web/src/pages/Engagement.tsx index 94c9f70..0e15455 100644 --- a/apps/web/src/pages/Engagement.tsx +++ b/apps/web/src/pages/Engagement.tsx @@ -314,7 +314,7 @@ export function Engagement() { setStageFilter((current) => (current === stage ? null : stage))} /> diff --git a/apps/web/src/pages/MotionLibrary.tsx b/apps/web/src/pages/MotionLibrary.tsx index 9069ba6..4619889 100644 --- a/apps/web/src/pages/MotionLibrary.tsx +++ b/apps/web/src/pages/MotionLibrary.tsx @@ -180,7 +180,7 @@ export function MotionLibrary() { setParam(setParams, STAGE_PARAM, stage === filters.stage ? '' : stage) diff --git a/apps/web/vite.config.ts b/apps/web/vite.config.ts index 7969447..d0b2ceb 100644 --- a/apps/web/vite.config.ts +++ b/apps/web/vite.config.ts @@ -44,6 +44,14 @@ export default defineConfig({ react: ['react', 'react-dom', 'react-router-dom'], supabase: ['@supabase/supabase-js'], query: ['@tanstack/react-query'], + /* + * Named explicitly, or the object form of `manualChunks` hoists the + * markdown parser into the entry chunk even though the only importers + * are lazy routes — measured at 327.70 kB gzip entry with it hoisted + * against 314 kB without. As its own chunk it is fetched when a + * Motion page is opened and never on any other route. + */ + markdown: ['react-markdown', 'remark-gfm'], }, }, }, diff --git a/packages/db/src/seed/index.ts b/packages/db/src/seed/index.ts index 42b2a90..6c94d9d 100644 --- a/packages/db/src/seed/index.ts +++ b/packages/db/src/seed/index.ts @@ -350,9 +350,16 @@ async function seed() { // ------------------------------------------------ the Motion starter library const motion = await seedMotionLibrary(db); console.log( - ` ${motion.total} Motion starter template(s) (${motion.added} new) — authored content, ` + - 'shared and system-owned, version 1 of their lineages.', + ` ${motion.total} Motion starter template(s) (${motion.added} new, ${motion.refreshed} ` + + 'refreshed) — authored content, shared and system-owned, version 1 of their lineages.', ); + if (motion.held.length) { + console.log( + ` ${motion.held.length} starter template(s) have newer wording here and were LEFT ALONE, ` + + `because they are in use or have been adopted: ${motion.held.join(', ')}. ` + + 'Publish a new version rather than editing one an engagement was cut from.', + ); + } for (const bad of motion.rejected) { console.error(` SKIPPED a starter template with a value outside the ontology: ${bad}`); } diff --git a/packages/db/src/seed/motion/case-study-frame.json b/packages/db/src/seed/motion/case-study-frame.json index cd31613..d25064f 100644 --- a/packages/db/src/seed/motion/case-study-frame.json +++ b/packages/db/src/seed/motion/case-study-frame.json @@ -4,7 +4,7 @@ "title": "Case Study Frame", "summary": "The structure every completed deployment gets written into, with the evidence rules that keep it honest, a fully worked fictional example, and the during-engagement harvest checklist that makes it writable at all.", "stage": "expansion", - "body": "*Playbook stage: expansion — but the harvest starts at qualification.* This is the expansion-stage artifact of the **Strategic Deployment Playbook**, and the finished document is consumed back at qualification and proposal. The harvest checklist below runs across every stage, so open this file in week one, not at the end.\n\n## What this document is\n\nA case study is not a testimonial and not a press release. It is a short evidence document that lets a stranger in a similar seat decide, in four minutes, whether the same motion would work for them.\n\nThe discipline that makes it worth writing is simple and unforgiving: a case study without a baseline is marketing, not evidence. If you cannot say what the number was before, no reader can tell whether the number after is good. Half-written case studies are usually the ones where nobody measured the \"before\", and by the time anyone noticed, the old system had been switched off.\n\nWrite to the same standard you would apply to a customer's own eval report. Every number traces to a measurement someone can point at: a run, a dataset, a date, a person who ran it. Where a number cannot be sourced, it does not appear.\n\n## The ten sections\n\n| Section | The question it answers | What a weak version looks like |\n| --- | --- | --- |\n| The customer, in one line | Who is this, and why is their situation legible to the reader? | Adjectives instead of a sector, a scale and a constraint |\n| The workflow and what it cost | What specific loop was slow, expensive or unreliable, and by how much? | \"Manual processes were inefficient\" |\n| Why it was not a prompting problem | What did they already try, and why did it plateau? | Silence, which reads as \"they never tried the cheap thing\" |\n| What we built | Eval, verifier, environment, recipe, in that order | A product feature list |\n| How we know the verifier holds | Judge agreement with humans, reward hacking found, what the verifier misses | An assertion that the task is \"mostly verifiable\" |\n| The numbers | Baseline, result, held-out set, who measured | A single percentage with no denominator |\n| What surprised us | The finding that changed the plan mid-engagement | A generic anecdote with no date attached to it |\n| What the customer owns now | Assets, weights, environments, and their ongoing loop | Vague partnership language |\n| What came next commercially | The expansion that followed, and on what trigger | Omitted, which wastes the artefact |\n| Approvals | Who at the customer signed off on these words | Missing, which makes the whole thing unpublishable |\n\nThe ordering is deliberate. Eval before environment, always. If the eval is written after the environment, it will quietly be shaped to flatter the environment, and the reader who matters most — a sceptical head of ML at the next account — will spot it. The verifier section sits between the build and the numbers for the same reason: a number produced by an unexamined judge is not a measurement, and that reader will ask about the judge before they ask about the score.\n\n## Evidence discipline\n\nFour rules, non-negotiable.\n\n**One: every number traces to a run.** Keep the run identifier, the dataset version, the date, and the name of the person who executed it. Label customer-run and our-run measurements separately and never blur them. At least one headline row must be a customer-run measurement; ours may support it but may not substitute for it.\n\n**Two: the held-out set must be described, not just named.** Five fields, every time: size, sampling period, selection method, freeze date, and written confirmation that it was excluded from training and from verifier development. A held-out set assembled after the fact is not held out. State the sampling method even when it is boring: \"{{n}} tickets sampled uniformly from {{month}}, frozen on {{date}}, excluded from training and from verifier development per {{name}}'s written confirmation\".\n\n**Three: comparisons must survive a change of definition.** If the label space, the taxonomy or the population changed during the engagement — and on a good engagement it usually does — the baseline is re-run on the new definition and both figures are reported. Merging the two classes an old system confused will raise its successor's agreement score on arithmetic alone. Say which definition each number was measured under, or the strongest number in the document becomes the easiest one to dismiss.\n\n**Four: the customer approves the words.** Not the gist, the words. Send the exact paragraphs, with the numbers in them, to the named approver and get written confirmation. Anonymised case studies still need this — \"a top-five logistics platform\" is identifiable to anyone in the sector, and the legal exposure is identical.\n\nEngagements that missed their exit criteria get written up on this same template, internal-only, and the harvest is mandatory regardless of outcome. A library containing only wins is a selection bias, and the sceptical reader invoked above is precisely the person who infers one. The failure write-ups are what make the published ones credible to our own people.\n\nWhere a number is genuinely unavailable at writing time, leave a marked placeholder in braces rather than an estimate. A published placeholder is embarrassing; a published guess is a credibility event.\n\n## Worked example, illustrative and fictional\n\nThe following is a fictional example. Verrata Freight does not exist, and no part of it describes a real Prime Intellect customer, deployment or result. Every figure in it is invented to show the shape filled in.\n\n**The customer, in one line.** Verrata Freight is a fictional mid-market freight brokerage, roughly 400 staff, running about 9,000 shipment exception cases a week across three regions.\n\n**The workflow and what it cost.** When a shipment goes off-plan — missed pickup, temperature excursion, customs hold — an operations coordinator must read the carrier's notes and the shipment record, decide on one of eleven remedies, notify the customer, and file the claim. Median handling was 14 minutes on TMS timestamps. Two of the eleven remedies carried financial exposure and were misapplied often enough that the finance team ran a weekly manual audit costing about 30 hours.\n\n**Why it was not a prompting problem.** They had already built a prompted assistant on a frontier model and iterated on it for five months, reaching an informal 71% agreement with their senior coordinators on the original eleven-way taxonomy. It stalled there. The failures were not phrasing failures: the model could not sequence a multi-step lookup across the carrier portal and their TMS, and it had no way to learn from the coordinators' corrections, which were being logged and thrown away.\n\n**What we built.** First an eval: 620 dev-split exceptions used for iteration, each labelled with the remedy the senior coordinator actually applied and the financial outcome ninety days later; then a disjoint 300-case held-out set drawn from the following quarter, sampled uniformly by week, frozen on {{date}} before any training began. Then a verifier: the remedy code is checkable exactly, and the claim filing is checkable against schema plus the carrier's acceptance response, so roughly 80% of the trajectory is programmatically verifiable and only the customer-facing message needed a rubric judge. Then an environment: a sandboxed replica of the TMS and carrier portal with recorded responses, so the agent could take real multi-step actions without touching production. Then the recipe: supervised fine-tuning of a {{base model class}} on 4,100 coordinator trajectories drawn only from cases predating both eval sets, then RL against a composite reward — exact remedy-code match at 0.6, claim schema and carrier acceptance at 0.25, rubric judge on the customer message at 0.15, with a hard zero on any trajectory that filed a claim the sandbox rejected. Because the message score carried the least weight, we ran a regression check on it explicitly: message quality on the held-out set was flat within the judge's noise band while remedy accuracy rose. The whole run was {{n}} GPU-hours over eleven days, of which the environment build was seven.\n\n**How we know the verifier holds.** The rubric judge was validated against two of Verrata's senior coordinators on 200 held-out messages neither had seen: judge-to-human agreement 91%, human-to-human 93%, so the judge is roughly as reliable as the raters it replaces and no more. Two reward-hacking behaviours showed up in RL. The policy learned to pad messages with hedging clauses the rubric read as thoroughness, patched with a length penalty and a rubric revision; and under uncertainty it drifted towards the single most common remedy, which the aggregate reward was indifferent to and per-class recall caught. One thing the verifier does not cover at all is escalation tone — whether a message that should sound apologetic does — so 50 cases were read by hand by Verrata's ops lead and signed off separately.\n\n**The numbers.** During the engagement the remedy taxonomy changed from eleven codes to ten (see below), so agreement figures are reported under both.\n\n| Measure | Baseline | Result | Measured on | Measured by |\n| --- | --- | --- | --- | --- |\n| Remedy agreement, ten-way taxonomy | 74%, their prompted system re-run unchanged in {{month}} | 89% | The frozen 300, held out from training and verifier development | Verrata's ops analytics team |\n| Remedy agreement, original eleven-way taxonomy | 71%, their own informal figure over five months of iteration | not measured; the eleven-way taxonomy was retired before training | — | Verrata |\n| High-exposure remedy misapplication | 6.2% | 1.4% | The same frozen 300; the two collapsed codes were not the two exposure-bearing remedies, so this denominator is unchanged across both taxonomies | Verrata, audited by finance |\n| Median handling time, assisted | 14 min | 6 min | One region, all exception types, weeks 1–3 after cutover, 2,300 cases; both figures from TMS timestamps, the baseline from the same region's preceding eight weeks | Verrata's TMS |\n\nThe handling-time row has an obvious confound: the taxonomy was simplified in the same window, which would shorten handling on its own. A second region ran unassisted on the new taxonomy through the same three weeks and held at 13 minutes, which is the best available control and is worth exactly as much as the reader thinks it is.\n\n**What surprised us.** The largest single gain came before any training. Building the eval exposed that two of the eleven remedy codes were used inconsistently by the coordinators themselves; agreement between two senior coordinators on the same case was only 84%. On {{date}} we collapsed those two codes into one with a follow-up field, re-scored the prompted system against the collapsed taxonomy on the same frozen 300, and it came back at 74%. Three of the eighteen points we eventually reported were a definition fix, not a model, and saying so costs nothing next to being caught not saying it.\n\n**What the customer owns now.** The environment, the verifier and its rubric, both eval sets, the trained weights, and a retraining loop that ingests coordinator corrections weekly instead of discarding them. They run the eval themselves on a schedule and it gates their own deploys.\n\n**What came next commercially.** The exception environment became the template for two adjacent workflows, carrier onboarding checks and rate-quote validation, contracted as a second phase. The trigger was their own eval showing the same plateau pattern in a neighbouring queue.\n\n**Approvals.** {{Name}}, VP Operations, approved v4 of this text on {{date}}; legal cleared the named use on {{date}}. Run identifiers r-2291 and r-2317, dataset v3, held on the demand record in PIG.\n\n## The harvest checklist\n\nCase studies die in reconstruction. Six weeks after go-live nobody can find the pre-POC baseline, the person who ran it has changed teams, and the surprising finding has been smoothed into \"it went well\". Capture as you go.\n\nThe rule of thumb: at every stage, ask \"if we win, what will I wish I had written down today?\" and write it down today. Three items carry most of the mortality. The baseline has a shelf life measured in days, because the moment a POC starts the old system stops being run cleanly enough to measure. The dated definition-change log is the only thing that later lets you re-run a baseline under the definition you actually reported. And the run of record — identifier, dataset version, operator — is what separates a number you can publish from a number you merely remember. The register below is the full set.\n\nKeep the harvest on the demand record in PIG rather than in someone's notebook, or in the shared deal folder until demand attachments exist. Wherever it lives, it has to survive an account handover.\n\n## Before you publish\n\nConfirm four things. The baseline is stated, sourced, and measured under the same definition as the result. The held-out set carries all five of its fields: size, sampling period, selection method, freeze date, and written confirmation of exclusion from training and from verifier development. The named approver at the customer has confirmed the exact wording in writing. And legal has cleared any named use of the customer's brand against whatever publicity or confidentiality terms the executed agreement actually carries — read it, because some agreements prohibit naming outright, and that is a fact you need in week one rather than at publication. If any of the four is missing, the document is an internal reference note rather than a case study, and it should be labelled as such.", + "body": "*Playbook stage: expansion — but the harvest starts at qualification.* This is the expansion-stage artefact of the **Strategic Deployment Playbook**, and the finished document is consumed back at qualification and proposal. The harvest checklist below runs across every stage, so open this file in week one, not at the end.\n\n## What this document is\n\nA case study is not a testimonial and not a press release. It is a short evidence document that lets a stranger in a similar seat decide, in four minutes, whether the same motion would work for them.\n\nThe discipline that makes it worth writing is simple and unforgiving: a case study without a baseline is marketing, not evidence. If you cannot say what the number was before, no reader can tell whether the number after is good. Half-written case studies are usually the ones where nobody measured the \"before\", and by the time anyone noticed, the old system had been switched off.\n\nWrite to the same standard you would apply to a customer's own eval report. Every number traces to a measurement someone can point at: a run, a dataset, a date, a person who ran it. Where a number cannot be sourced, it does not appear.\n\n## The ten sections\n\n| Section | The question it answers | What a weak version looks like |\n| --- | --- | --- |\n| The customer, in one line | Who is this, and why is their situation legible to the reader? | Adjectives instead of a sector, a scale and a constraint |\n| The workflow and what it cost | What specific loop was slow, expensive or unreliable, and by how much? | \"Manual processes were inefficient\" |\n| Why it was not a prompting problem | What did they already try, and why did it plateau? | Silence, which reads as \"they never tried the cheap thing\" |\n| What we built | Eval, verifier, environment, recipe, in that order | A product feature list |\n| How we know the verifier holds | Judge agreement with humans, reward hacking found, what the verifier misses | An assertion that the task is \"mostly verifiable\" |\n| The numbers | Baseline, result, held-out set, who measured | A single percentage with no denominator |\n| What surprised us | The finding that changed the plan mid-engagement | A generic anecdote with no date attached to it |\n| What the customer owns now | Assets, weights, environments, and their ongoing loop | Vague partnership language |\n| What came next commercially | The expansion that followed, and on what trigger | Omitted, which wastes the artefact |\n| Approvals | Who at the customer signed off on these words | Missing, which makes the whole thing unpublishable |\n\nThe ordering is deliberate. Eval before environment, always. If the eval is written after the environment, it will quietly be shaped to flatter the environment, and the reader who matters most — a sceptical head of ML at the next account — will spot it. The verifier section sits between the build and the numbers for the same reason: a number produced by an unexamined judge is not a measurement, and that reader will ask about the judge before they ask about the score.\n\n## Evidence discipline\n\nFour rules, non-negotiable.\n\n**One: every number traces to a run.** Keep the run identifier, the dataset version, the date, and the name of the person who executed it. Label customer-run and our-run measurements separately and never blur them. At least one headline row must be a customer-run measurement; ours may support it but may not substitute for it.\n\n**Two: the held-out set must be described, not just named.** Five fields, every time: size, sampling period, selection method, freeze date, and written confirmation that it was excluded from training and from verifier development. A held-out set assembled after the fact is not held out. State the sampling method even when it is boring: \"{{n}} tickets sampled uniformly from {{month}}, frozen on {{date}}, excluded from training and from verifier development per {{name}}'s written confirmation\".\n\n**Three: comparisons must survive a change of definition.** If the label space, the taxonomy or the population changed during the engagement — and on a good engagement it usually does — the baseline is re-run on the new definition and both figures are reported. Merging the two classes an old system confused will raise its successor's agreement score on arithmetic alone. Say which definition each number was measured under, or the strongest number in the document becomes the easiest one to dismiss.\n\n**Four: the customer approves the words.** Not the gist, the words. Send the exact paragraphs, with the numbers in them, to the named approver and get written confirmation. Anonymised case studies still need this — \"a top-five logistics platform\" is identifiable to anyone in the sector, and the legal exposure is identical.\n\nEngagements that missed their exit criteria get written up on this same template, internal-only, and the harvest is mandatory regardless of outcome. A library containing only wins is a selection bias, and the sceptical reader invoked above is precisely the person who infers one. The failure write-ups are what make the published ones credible to our own people.\n\nWhere a number is genuinely unavailable at writing time, leave a marked placeholder in braces rather than an estimate. A published placeholder is embarrassing; a published guess is a credibility event.\n\n## Worked example, illustrative and fictional\n\nThe following is a fictional example. Verrata Freight does not exist, and no part of it describes a real Prime Intellect customer, deployment or result. Every figure in it is invented to show the shape filled in.\n\n**The customer, in one line.** Verrata Freight is a fictional mid-market freight brokerage, roughly 400 staff, running about 9,000 shipment exception cases a week across three regions.\n\n**The workflow and what it cost.** When a shipment goes off-plan — missed pickup, temperature excursion, customs hold — an operations coordinator must read the carrier's notes and the shipment record, decide on one of eleven remedies, notify the customer, and file the claim. Median handling was 14 minutes on TMS timestamps. Two of the eleven remedies carried financial exposure and were misapplied often enough that the finance team ran a weekly manual audit costing about 30 hours.\n\n**Why it was not a prompting problem.** They had already built a prompted assistant on a frontier model and iterated on it for five months, reaching an informal 71% agreement with their senior coordinators on the original eleven-way taxonomy. It stalled there. The failures were not phrasing failures: the model could not sequence a multi-step lookup across the carrier portal and their TMS, and it had no way to learn from the coordinators' corrections, which were being logged and thrown away.\n\n**What we built.** First an eval: 620 dev-split exceptions used for iteration, each labelled with the remedy the senior coordinator actually applied and the financial outcome ninety days later; then a disjoint 300-case held-out set drawn from the following quarter, sampled uniformly by week, frozen on {{date}} before any training began. Then a verifier: the remedy code is checkable exactly, and the claim filing is checkable against schema plus the carrier's acceptance response, so roughly 80% of the trajectory is programmatically verifiable and only the customer-facing message needed a rubric judge. Then an environment: a sandboxed replica of the TMS and carrier portal with recorded responses, so the agent could take real multi-step actions without touching production. Then the recipe: supervised fine-tuning of a {{base model class}} on 4,100 coordinator trajectories drawn only from cases predating both eval sets, then RL against a composite reward — exact remedy-code match at 0.6, claim schema and carrier acceptance at 0.25, rubric judge on the customer message at 0.15, with a hard zero on any trajectory that filed a claim the sandbox rejected. Because the message score carried the least weight, we ran a regression check on it explicitly: message quality on the held-out set was flat within the judge's noise band while remedy accuracy rose. The whole run was {{n}} GPU-hours over eleven days, of which the environment build was seven.\n\n**How we know the verifier holds.** The rubric judge was validated against two of Verrata's senior coordinators on 200 held-out messages neither had seen: judge-to-human agreement 91%, human-to-human 93%, so the judge is roughly as reliable as the raters it replaces and no more. Two reward-hacking behaviours showed up in RL. The policy learned to pad messages with hedging clauses the rubric read as thoroughness, patched with a length penalty and a rubric revision; and under uncertainty it drifted towards the single most common remedy, which the aggregate reward was indifferent to and per-class recall caught. One thing the verifier does not cover at all is escalation tone — whether a message that should sound apologetic does — so 50 cases were read by hand by Verrata's ops lead and signed off separately.\n\n**The numbers.** During the engagement the remedy taxonomy changed from eleven codes to ten (see below), so agreement figures are reported under both.\n\n| Measure | Baseline | Result | Measured on | Measured by |\n| --- | --- | --- | --- | --- |\n| Remedy agreement, ten-way taxonomy | 74%, their prompted system re-run unchanged in {{month}} | 89% | The frozen 300, held out from training and verifier development | Verrata's ops analytics team |\n| Remedy agreement, original eleven-way taxonomy | 71%, their own informal figure over five months of iteration | not measured; the eleven-way taxonomy was retired before training | — | Verrata |\n| High-exposure remedy misapplication | 6.2% | 1.4% | The same frozen 300; the two collapsed codes were not the two exposure-bearing remedies, so this denominator is unchanged across both taxonomies | Verrata, audited by finance |\n| Median handling time, assisted | 14 min | 6 min | One region, all exception types, weeks 1–3 after cutover, 2,300 cases; both figures from TMS timestamps, the baseline from the same region's preceding eight weeks | Verrata's TMS |\n\nThe handling-time row has an obvious confound: the taxonomy was simplified in the same window, which would shorten handling on its own. A second region ran unassisted on the new taxonomy through the same three weeks and held at 13 minutes, which is the best available control and is worth exactly as much as the reader thinks it is.\n\n**What surprised us.** The largest single gain came before any training. Building the eval exposed that two of the eleven remedy codes were used inconsistently by the coordinators themselves; agreement between two senior coordinators on the same case was only 84%. On {{date}} we collapsed those two codes into one with a follow-up field, re-scored the prompted system against the collapsed taxonomy on the same frozen 300, and it came back at 74%. Three of the eighteen points we eventually reported were a definition fix, not a model, and saying so costs nothing next to being caught not saying it.\n\n**What the customer owns now.** The environment, the verifier and its rubric, both eval sets, the trained weights, and a retraining loop that ingests coordinator corrections weekly instead of discarding them. They run the eval themselves on a schedule and it gates their own deploys.\n\n**What came next commercially.** The exception environment became the template for two adjacent workflows, carrier onboarding checks and rate-quote validation, contracted as a second phase. The trigger was their own eval showing the same plateau pattern in a neighbouring queue.\n\n**Approvals.** {{Name}}, VP Operations, approved v4 of this text on {{date}}; legal cleared the named use on {{date}}. Run identifiers r-2291 and r-2317, dataset v3, held on the demand record in PIG.\n\n## The harvest checklist\n\nCase studies die in reconstruction. Six weeks after go-live nobody can find the pre-POC baseline, the person who ran it has changed teams, and the surprising finding has been smoothed into \"it went well\". Capture as you go.\n\nThe rule of thumb: at every stage, ask \"if we win, what will I wish I had written down today?\" and write it down today. Three items carry most of the mortality. The baseline has a shelf life measured in days, because the moment a POC starts the old system stops being run cleanly enough to measure. The dated definition-change log is the only thing that later lets you re-run a baseline under the definition you actually reported. And the run of record — identifier, dataset version, operator — is what separates a number you can publish from a number you merely remember. The register below is the full set.\n\nKeep the harvest on the demand record in PIG rather than in someone's notebook, or in the shared deal folder until demand attachments exist. Wherever it lives, it has to survive an account handover.\n\n## Before you publish\n\nConfirm four things. The baseline is stated, sourced, and measured under the same definition as the result. The held-out set carries all five of its fields: size, sampling period, selection method, freeze date, and written confirmation of exclusion from training and from verifier development. The named approver at the customer has confirmed the exact wording in writing. And legal has cleared any named use of the customer's brand against whatever publicity or confidentiality terms the executed agreement actually carries — read it, because some agreements prohibit naming outright, and that is a fact you need in week one rather than at publication. If any of the four is missing, the document is an internal reference note rather than a case study, and it should be labelled as such.", "fields": { "sections": [ { diff --git a/packages/db/src/seed/motion/index.ts b/packages/db/src/seed/motion/index.ts index 9c51217..677d1f1 100644 --- a/packages/db/src/seed/motion/index.ts +++ b/packages/db/src/seed/motion/index.ts @@ -19,12 +19,22 @@ * against this to confirm the attribute syntax is understood by each, since * the server runs the TypeScript directly. * - * Idempotency is `onConflictDoNothing({ target: [slug, version] })`, which - * works **only** because of `motion_templates_slug_version_key`. AGENTS.md §5 - * is blunt about what that clause does without a constraint to fire on — it - * silently duplicated seed data here twice — so `assertNoDuplicates` below - * checks the outcome rather than trusting the schema, and CI counts this table - * in its idempotency gate alongside `contacts`. + * Idempotency is keyed on `(slug, version)`, which is enforceable **only** + * because of `motion_templates_slug_version_key`. AGENTS.md §5 is blunt about + * what `onConflictDoNothing` does without a constraint to fire on — it silently + * duplicated seed data here twice — so `assertNoDuplicates` below checks the + * outcome rather than trusting the schema, and CI counts this table in its + * idempotency gate alongside `contacts`. + * + * A second run REFRESHES a starter row rather than skipping it, but only while + * that row is still ours: `is_system`, nobody has forked or instantiated it + * (`usage_count = 0`) and it has no owner. Insert-only was the first version and + * is wrong for the one thing this library is for — the content improves, and a + * deployment seeded in August would otherwise be frozen on August's wording for + * ever, with no upgrade path short of hand-editing production rows. The + * pristine test is what keeps that from becoming a write over somebody's work: + * the moment a template has been used, §7a says it is never edited in place, + * and this respects that with the same condition the API enforces. */ import { DEMAND_STAGES, isMotionKind, type DemandStage, type MotionKind } from '@pig/core'; import { and, eq, inArray } from 'drizzle-orm'; @@ -89,6 +99,29 @@ const isDemandStage = (value: string): value is DemandStage => * inside `fields` means promotion carries it forward with the rest of the * structured payload without any special handling in the promote path. */ +/* + * Key order is not preserved by `jsonb`: Postgres stores an object with its + * keys sorted by length and then bytewise, so the value that comes back is + * rarely the value that went in. A plain `JSON.stringify` comparison therefore + * reported every starter template as changed on every run, and the seed + * rewrote nine rows each time while claiming to be idempotent. Found by + * running it twice and reading the count, not by reading the code. + */ +function canonical(value: unknown): string { + const order = (node: unknown): unknown => { + if (Array.isArray(node)) return node.map(order); + if (node && typeof node === 'object') { + return Object.fromEntries( + Object.keys(node as Record) + .sort() + .map((key) => [key, order((node as Record)[key])]), + ); + } + return node; + }; + return JSON.stringify(order(value)); +} + function toRow( template: StarterTemplate, kind: MotionKind, @@ -144,9 +177,17 @@ async function assertNoDuplicates(db: Database, slugs: readonly string[]): Promi export async function seedMotionLibrary( db: Database, -): Promise<{ total: number; added: number; rejected: readonly string[] }> { +): Promise<{ + total: number; + added: number; + refreshed: number; + held: readonly string[]; + rejected: readonly string[]; +}> { const rejected: string[] = []; + const held: string[] = []; let added = 0; + let refreshed = 0; for (const template of STARTER_LIBRARY) { /* @@ -161,12 +202,64 @@ export async function seedMotionLibrary( continue; } - const [created] = await db - .insert(motionTemplates) - .values(toRow(template, template.kind, template.stage)) - .onConflictDoNothing({ target: [motionTemplates.slug, motionTemplates.version] }) - .returning({ id: motionTemplates.id }); - if (created) added += 1; + const row = toRow(template, template.kind, template.stage); + + const [existing] = await db + .select({ + id: motionTemplates.id, + isSystem: motionTemplates.isSystem, + usageCount: motionTemplates.usageCount, + ownerUserId: motionTemplates.ownerUserId, + title: motionTemplates.title, + summary: motionTemplates.summary, + body: motionTemplates.body, + fields: motionTemplates.fields, + kind: motionTemplates.kind, + stage: motionTemplates.stage, + }) + .from(motionTemplates) + .where(and(eq(motionTemplates.slug, template.slug), eq(motionTemplates.version, 1))) + .limit(1); + + if (!existing) { + const [created] = await db + .insert(motionTemplates) + .values(row) + .onConflictDoNothing({ target: [motionTemplates.slug, motionTemplates.version] }) + .returning({ id: motionTemplates.id }); + if (created) added += 1; + continue; + } + + const pristine = + existing.isSystem && existing.usageCount === 0 && existing.ownerUserId === null; + const changed = + existing.title !== row.title || + existing.summary !== row.summary || + existing.body !== row.body || + existing.kind !== row.kind || + existing.stage !== row.stage || + canonical(existing.fields) !== canonical(row.fields); + + if (pristine && changed) { + await db + .update(motionTemplates) + .set({ + title: row.title, + summary: row.summary, + body: row.body, + fields: row.fields, + kind: row.kind, + stage: row.stage, + updatedAt: new Date(), + }) + .where(eq(motionTemplates.id, existing.id)); + refreshed += 1; + } else if (changed) { + // Reported rather than forced: the row has been used or adopted, so + // overwriting it would edit a template a live engagement was cut from. + held.push(template.slug); + } } const total = await assertNoDuplicates( @@ -174,5 +267,5 @@ export async function seedMotionLibrary( STARTER_LIBRARY.map((template) => template.slug), ); - return { total, added, rejected }; + return { total, added, refreshed, held, rejected }; } diff --git a/packages/db/src/seed/motion/proposal-blocks.json b/packages/db/src/seed/motion/proposal-blocks.json index 69ea8b7..344916b 100644 --- a/packages/db/src/seed/motion/proposal-blocks.json +++ b/packages/db/src/seed/motion/proposal-blocks.json @@ -4,7 +4,7 @@ "title": "Proposal Blocks", "summary": "A library of finished, editable proposal paragraphs a forward deployed lead assembles into a customer proposal or memo, with guidance on which blocks to use, in what order, and which to leave out.", "stage": "proposal", - "body": "*Playbook stage: proposal.* This is the proposal-stage artifact of the **Strategic Deployment Playbook**. You arrive with a reviewed scope and a measured baseline; the threshold written here is the one **Post-Training POC Structure** then runs against. The number in the commercial block is built with **Pricing and Packaging Inputs**, and the data-handling block is what the legal and procurement stages reuse.\n\n# Proposal Blocks\n\nA proposal for a post-training engagement is not a document you write once and reuse. It is an argument you assemble, in a specific order, for one named buying group. This library exists so that the argument's load-bearing paragraphs are already written to a standard, and the forward deployed lead spends their time on the two things a template cannot do: getting the customer's own language right, and getting the numbers right.\n\n## How to assemble\n\nPick blocks; do not pick all of them. A proposal long enough to need a contents page is being read by nobody except procurement. The default spine for a first POC is:\n\n| Order | Block | Why it sits here |\n| --- | --- | --- |\n| 1 | Problem restated | Earns the right to be read. If this is wrong, nothing downstream lands. |\n| 2 | Why post-training | Pre-empts the cheapest internal objection before a sceptic raises it in a room you are not in. |\n| 3 | What we will build | Converts an abstract capability into four named artefacts. |\n| 4 | Success and measurement | Defines the number that ends the POC. |\n| 5 | Measuring it honestly | Makes the number credible rather than assertable, and says how the model is stopped from gaming it. |\n| 6 | What you own | The differentiator. Do not bury it on page five. |\n| 7 | Data handling and where it runs | Unblocks security review, which is usually the true critical path. |\n| 8 | What we need from you | The block most often cut and the most common cause of slippage. |\n| 9 | Commercial structure | After value, never before. |\n| 10 | Expansion path | Signals that the POC is a first step in a system, not a one-off. |\n| 11 | Timeline and decision | Creates a real deadline out of real constraints. |\n\nBlock 2 and the prompting objection block are the same argument aimed at different readers. Use one or the other, never both. Objection blocks and the scope-boundary block are inserted only when the specific objection has actually been voiced, or when the champion tells you it will be voiced by someone else. An unprompted rebuttal reads as defensiveness and teaches the reader an objection they had not thought of.\n\n## The rule about the first block\n\nThe problem restatement is the highest-leverage paragraph in the document, and it is the one most people write last and fastest. It does three jobs at once. It proves you listened, which is the only credential that matters at this stage. It fixes the scope in writing, so that in week five nobody can quietly redefine what the POC was for. And it decides who can forward the document: a restatement written in the customer's internal vocabulary will be pasted into their Slack and read by people you will never meet, whereas one written in vendor vocabulary dies in the champion's inbox.\n\nWrite it using words you heard on a call. Their name for the workflow, their name for the queue, their internal metric, their acronym. If you find yourself writing \"unlock efficiency\" or \"scale operations\", you did not take good enough notes and you should go back and ask.\n\n## Worked example\n\nThe blocks are dense with placeholders, which makes it easy to fill them in mechanically and end up with a form rather than a paragraph. Here is the problem-restated block with every token filled, for an invented customer used only for illustration. No part of this is a real engagement, a real number, or a real quote.\n\n> Meridian Freight runs exception triage across roughly 4,000 shipments a week. Today GPT-class prompting handles the first pass, and the Ops Desk reviews the output. The review is not a formality: roughly one in three of those outputs is corrected or rewritten before it goes out, and the corrections cluster in cases where a customs hold has to be reconciled against the carrier's own status feed. That correction work is currently absorbed by four senior coordinators, which means the cost of the failure mode is real but is not sitting in anyone's model spend. You have told us the constraint is not throughput but trust: \"the desk re-checks everything anyway, so the model saves us nothing on the cases that matter\". This proposal addresses that specific gap and nothing wider.\n\nNote what the filling-in did. \"Exception triage\" and \"Ops Desk\" are their words, not ours. The quote is verbatim from a call. The failure category is specific enough that a coordinator reading it would recognise their own week. If your filled-in version reads more generically than this, the problem is your discovery, not the block.\n\n## The rule about numbers\n\nEvery number in a proposal is either measured, quoted, or a placeholder. There is no fourth category, and a number that entered the document as an illustration will be quoted back to you as a commitment. Keep placeholders in {{braces}} until the moment they are real, and where you must show a range, say in the sentence that it is an assumption and name what it depends on. A proposal that says \"we expect {{15-25}} points of pass-rate improvement, an assumption based on your current failure distribution which we have not yet seen\" is stronger than one asserting a confident number, because it tells the reader you know which numbers you have earned.\n\nThe same rule binds words that are numbers in disguise. \"Most\", \"usually\" and \"far more often\" are quantities, and if you cannot source them, attribute them: \"in the engagements we have run\" is honest, and it is what a technical reader will assume you meant anyway.\n\n## The rule about obligations\n\nIn the engagements we have run, POCs slip for customer-side reasons more often than for technical ones: data access that took six weeks to approve, a subject-matter expert who was never actually allocated, an environment that could not be reproduced outside a laptop. The \"what we need from you\" block converts those from surprises into agreed line items with names and dates against them. Put a person's name next to each obligation before you send the document. If the champion cannot supply a name, you have found the real risk in the deal and you have found it in week zero rather than week six.\n\n## Editing rules for every block\n\nReplace every {{placeholder}}, and check for nested or duplicated tokens before you run find-and-replace. A single unreplaced brace destroys the impression that the document was written for this customer, and it is the single most common self-inflicted wound in proposal work. Where two placeholders in one block mean different things, they carry different names — {{signature_deadline}} and {{engineer_start}} are both dates and are not interchangeable.\n\nCut any sentence that would be equally true of a different customer. Read the finished document and strike anything that survives the swap test: replace the customer's name with a competitor's and see if the sentence still reads. If it does, it is not doing work. This applies hardest to the sentences you are proudest of; a flywheel metaphor that survives the swap is decoration, and the arithmetic underneath it is what you meant to say.\n\nKeep the verb tense honest. \"We will build\" for work in scope; \"we would expect\" for work beyond the POC. Blurring the two is how a proposal becomes a dispute.\n\nNever soften the success criterion to make signature easier. A vague criterion buys a fast signature and then costs a renewal, because there is no moment at which anyone can say the thing worked. The criterion is the product of the proposal; the rest is context around it.\n\n## What a proposal is not\n\nIt is not a technical design document. Depth belongs in the appendix or in a follow-up architecture session; the body should be readable by a CFO who will approve the spend and has no interest in reward shaping. The one piece of method that does belong in the body is how the measurement is protected, because that is the part a technical evaluator will test you on and the part a commercial reader is quietly relying on.\n\nIt is not a security questionnaire. The data-handling block exists to get the right people into the room early, not to pre-answer their forms.\n\nIt is not the place to make the case for AI in general. If the customer needs that case made, they are not at proposal stage, and sending one anyway is how deals stall at procurement with everyone politely confused about what was being bought.", + "body": "*Playbook stage: proposal.* This is the proposal-stage artefact of the **Strategic Deployment Playbook**. You arrive with a reviewed scope and a measured baseline; the threshold written here is the one **Post-Training POC Structure** then runs against. The number in the commercial block is built with **Pricing and Packaging Inputs**, and the data-handling block is what the legal and procurement stages reuse.\n\n# Proposal Blocks\n\nA proposal for a post-training engagement is not a document you write once and reuse. It is an argument you assemble, in a specific order, for one named buying group. This library exists so that the argument's load-bearing paragraphs are already written to a standard, and the forward deployed lead spends their time on the two things a template cannot do: getting the customer's own language right, and getting the numbers right.\n\n## How to assemble\n\nPick blocks; do not pick all of them. A proposal long enough to need a contents page is being read by nobody except procurement. The default spine for a first POC is:\n\n| Order | Block | Why it sits here |\n| --- | --- | --- |\n| 1 | Problem restated | Earns the right to be read. If this is wrong, nothing downstream lands. |\n| 2 | Why post-training | Pre-empts the cheapest internal objection before a sceptic raises it in a room you are not in. |\n| 3 | What we will build | Converts an abstract capability into four named artefacts. |\n| 4 | Success and measurement | Defines the number that ends the POC. |\n| 5 | Measuring it honestly | Makes the number credible rather than assertable, and says how the model is stopped from gaming it. |\n| 6 | What you own | The differentiator. Do not bury it on page five. |\n| 7 | Data handling and where it runs | Unblocks security review, which is usually the true critical path. |\n| 8 | What we need from you | The block most often cut and the most common cause of slippage. |\n| 9 | Commercial structure | After value, never before. |\n| 10 | Expansion path | Signals that the POC is a first step in a system, not a one-off. |\n| 11 | Timeline and decision | Creates a real deadline out of real constraints. |\n\nBlock 2 and the prompting objection block are the same argument aimed at different readers. Use one or the other, never both. Objection blocks and the scope-boundary block are inserted only when the specific objection has actually been voiced, or when the champion tells you it will be voiced by someone else. An unprompted rebuttal reads as defensiveness and teaches the reader an objection they had not thought of.\n\n## The rule about the first block\n\nThe problem restatement is the highest-leverage paragraph in the document, and it is the one most people write last and fastest. It does three jobs at once. It proves you listened, which is the only credential that matters at this stage. It fixes the scope in writing, so that in week five nobody can quietly redefine what the POC was for. And it decides who can forward the document: a restatement written in the customer's internal vocabulary will be pasted into their Slack and read by people you will never meet, whereas one written in vendor vocabulary dies in the champion's inbox.\n\nWrite it using words you heard on a call. Their name for the workflow, their name for the queue, their internal metric, their acronym. If you find yourself writing \"unlock efficiency\" or \"scale operations\", you did not take good enough notes and you should go back and ask.\n\n## Worked example\n\nThe blocks are dense with placeholders, which makes it easy to fill them in mechanically and end up with a form rather than a paragraph. Here is the problem-restated block with every token filled, for an invented customer used only for illustration. No part of this is a real engagement, a real number, or a real quote.\n\n> Meridian Freight runs exception triage across roughly 4,000 shipments a week. Today GPT-class prompting handles the first pass, and the Ops Desk reviews the output. The review is not a formality: roughly one in three of those outputs is corrected or rewritten before it goes out, and the corrections cluster in cases where a customs hold has to be reconciled against the carrier's own status feed. That correction work is currently absorbed by four senior coordinators, which means the cost of the failure mode is real but is not sitting in anyone's model spend. You have told us the constraint is not throughput but trust: \"the desk re-checks everything anyway, so the model saves us nothing on the cases that matter\". This proposal addresses that specific gap and nothing wider.\n\nNote what the filling-in did. \"Exception triage\" and \"Ops Desk\" are their words, not ours. The quote is verbatim from a call. The failure category is specific enough that a coordinator reading it would recognise their own week. If your filled-in version reads more generically than this, the problem is your discovery, not the block.\n\n## The rule about numbers\n\nEvery number in a proposal is either measured, quoted, or a placeholder. There is no fourth category, and a number that entered the document as an illustration will be quoted back to you as a commitment. Keep placeholders in {{braces}} until the moment they are real, and where you must show a range, say in the sentence that it is an assumption and name what it depends on. A proposal that says \"we expect {{15-25}} points of pass-rate improvement, an assumption based on your current failure distribution which we have not yet seen\" is stronger than one asserting a confident number, because it tells the reader you know which numbers you have earned.\n\nThe same rule binds words that are numbers in disguise. \"Most\", \"usually\" and \"far more often\" are quantities, and if you cannot source them, attribute them: \"in the engagements we have run\" is honest, and it is what a technical reader will assume you meant anyway.\n\n## The rule about obligations\n\nIn the engagements we have run, POCs slip for customer-side reasons more often than for technical ones: data access that took six weeks to approve, a subject-matter expert who was never actually allocated, an environment that could not be reproduced outside a laptop. The \"what we need from you\" block converts those from surprises into agreed line items with names and dates against them. Put a person's name next to each obligation before you send the document. If the champion cannot supply a name, you have found the real risk in the deal and you have found it in week zero rather than week six.\n\n## Editing rules for every block\n\nReplace every {{placeholder}}, and check for nested or duplicated tokens before you run find-and-replace. A single unreplaced brace destroys the impression that the document was written for this customer, and it is the single most common self-inflicted wound in proposal work. Where two placeholders in one block mean different things, they carry different names — {{signature_deadline}} and {{engineer_start}} are both dates and are not interchangeable.\n\nCut any sentence that would be equally true of a different customer. Read the finished document and strike anything that survives the swap test: replace the customer's name with a competitor's and see if the sentence still reads. If it does, it is not doing work. This applies hardest to the sentences you are proudest of; a flywheel metaphor that survives the swap is decoration, and the arithmetic underneath it is what you meant to say.\n\nKeep the verb tense honest. \"We will build\" for work in scope; \"we would expect\" for work beyond the POC. Blurring the two is how a proposal becomes a dispute.\n\nNever soften the success criterion to make signature easier. A vague criterion buys a fast signature and then costs a renewal, because there is no moment at which anyone can say the thing worked. The criterion is the product of the proposal; the rest is context around it.\n\n## What a proposal is not\n\nIt is not a technical design document. Depth belongs in the appendix or in a follow-up architecture session; the body should be readable by a CFO who will approve the spend and has no interest in reward shaping. The one piece of method that does belong in the body is how the measurement is protected, because that is the part a technical evaluator will test you on and the part a commercial reader is quietly relying on.\n\nIt is not a security questionnaire. The data-handling block exists to get the right people into the room early, not to pre-answer their forms.\n\nIt is not the place to make the case for AI in general. If the customer needs that case made, they are not at proposal stage, and sending one anyway is how deals stall at procurement with everyone politely confused about what was being bought.", "fields": { "blocks": [ { diff --git a/packages/db/src/seed/motion/reference-architectures.json b/packages/db/src/seed/motion/reference-architectures.json index 27b0387..6bfd64e 100644 --- a/packages/db/src/seed/motion/reference-architectures.json +++ b/packages/db/src/seed/motion/reference-architectures.json @@ -4,7 +4,7 @@ "title": "Reference Architectures", "summary": "The four shapes a Prime Intellect Lab deployment actually takes, drawn end to end, for choosing and defending a technical design during scoping.", "stage": "scoping", - "body": "*Playbook stage: scoping.* This is the scoping-stage artifact of the **Strategic Deployment Playbook**: it produces the environment specification the stage exits on. Its companion is the eval-design section of **Post-Training POC Structure**, and its data-path sections are what the legal stage reviews.\n\nFour shapes cover the deployments this playbook is written for. Knowing which one you are in before the scoping call ends is what separates a week-four training run from a week-four discovery that no eval exists, the traces are outcome-only, and the environment cannot be reset. Each shape below has a diagram, a component table with an owner per box, an account of where customer data crosses a boundary, and the two or three decisions that redraw the picture. The failure modes are failures of the architecture rather than of execution: they occur even when everyone does their job well. Every value in double braces is an assumption to be replaced with a figure you can defend in the room; the Prime Intellect component names track the current public stack and should be checked before the deck goes out.\n\n## 1. Eval-first\n\n**Fits when** the customer cannot yet tell you what \"better\" means, has no clean traces, or has a stakeholder who does not believe a model can be improved at all. This is the cheapest engagement and it is usually the correct first one. It is also the only architecture that produces something valuable even if you never train.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n L[Prod logs]\n S[Scrub and sample]\n X[Expert panel]\n G[Adjudicated labels]\n Gc[Calibration half]\n Gh[Held-out half]\n end\n subgraph PI[Prime Intellect Lab]\n E[Eval set]\n M[Model under test]\n V[verifiers lib]\n R[Scorecard]\n F[Eval frozen]\n end\n L --> S --> E\n X --> G\n G --> Gc --> V\n G --> Gh\n E --> M --> V --> R\n R --> D{\"Agreement vs target\"}\n D -- below --> V\n D -- met --> Ha{\"Holds on held-out\"}\n Gh --> Ha\n Ha -- no --> V\n Ha -- yes --> F\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Prod logs and sampler | Customer | The eval must be drawn from real traffic, not from the tasks the team finds interesting. Stratify by the segments the business already reports on. |\n| Scrub and sample | Customer | Redaction happens before egress so the DPA conversation stays narrow. |\n| Expert panel and adjudicated labels | Customer, facilitated by PI | Human adjudication is the only thing that makes a verifier trustworthy. Size the set at {{adjudicated_label_count}} items, or by the operational test: keep labelling until measured agreement is stable across two disjoint halves. |\n| Calibration and held-out halves | Customer | The adjudicated set is split on arrival. The verifier is tuned against the calibration half only; the held-out half is never seen during tuning and exists to catch a verifier fitted to its own reference. |\n| Eval set | Joint, hosted in Lab | Versioned and frozen, so a score in month one is comparable to a score in month six. |\n| Verifier | Prime Intellect | The `verifiers` library, programmatic where possible and model-judge where not. The statistic is Cohen's kappa against the calibration half, item by item; acceptance is kappa at or above {{agreement_target}} on the held-out half, and {{agreement_target}} is not a free parameter: the library floor set in the Strategic Deployment Playbook is Cohen's kappa 0.7 against adjudicated labels, or within 0.10 of a measured human ceiling that is itself below 0.8. Recalibrate on any of three triggers: the judge model version changes, the eval set gains items, or held-out kappa drops below target on a scheduled re-measure. |\n| Disagreement adjudication | Joint | When judge and adjudicated label disagree, a named human decides which is wrong. The adjudicated label is often the error, and relabelling it is a legitimate outcome provided the change is versioned with a reason. |\n| Scorecard | Prime Intellect | Per-segment, not a single aggregate. A single number hides the segment you are actually failing. |\n\n**Data path.** Logs are sampled and redacted inside the customer estate; only the redacted eval set and the adjudicated labels cross into Lab. Results return as scores and, where permitted, sampled failure traces. If the customer will not export even redacted text, the harness runs inside their VPC — the diagram survives, the hosting moves.\n\n**Decisions that change the diagram.** Whether the verifier can be programmatic (a diff, a schema check, a unit test, a lookup against a system of record) or must be a model judge — this determines whether you need the gold-label loop at all, and it is the single largest cost driver. Whether the eval runs on the customer side, which extends the build from {{eval_build_weeks_lab}} weeks to {{eval_build_weeks_customer}}. And whether you evaluate the model or the whole product surface including retrieval and prompts, since only the latter predicts what the business will feel.\n\n**Failure mode of the architecture.** The eval becomes the deliverable and nothing is ever trained. A frozen scorecard is comfortable: it makes the team look rigorous, generates a quarterly artefact, and demands nothing. Write the training trigger into the scope document — a named threshold and a named owner — or this engagement terminates itself politely.\n\n## 2. SFT on traces\n\n**Fits when** the customer already has a large body of expert behaviour recorded: resolved support tickets with the actual resolution, reviewed legal redlines, closed engineering PRs, annotated claims decisions. The behaviour is demonstrable but not currently reproducible by a general model.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n T[Trace store]\n P[Filter and scrub]\n Rv[SME review]\n end\n subgraph PI[Prime Intellect Lab]\n Dx[Eval id subtract]\n D[SFT dataset]\n B[Base model]\n Tn[SFT run]\n Ck[Checkpoints]\n Ev[Eval harness]\n In[Inference]\n end\n T --> P --> Rv --> Dx --> D\n Ev -. eval ids .-> Dx\n B --> Tn\n D --> Tn --> Ck --> Ev\n Ev -- regress --> D\n Ev -- pass --> In\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Trace store | Customer | The asset. Ask for row counts and a schema in discovery, not adjectives. |\n| Filter and scrub | Customer | Traces are kept on a stated outcome rule — the ticket closed without reopening, the PR merged without revert, the redline survived counterparty review — not on volume. Assume roughly the top {{trace_keep_fraction}} of the corpus survives the rule and size the engagement on that, not on the row count. |\n| Eval id subtraction | Prime Intellect | The eval set and the training corpus come from the same records in most customers. Every eval item's source id is subtracted from the corpus before the first run, or a pass on the harness is a pass on memorised rows. |\n| SME review | Customer | Someone must confirm that a trace records good behaviour and not a tolerated one. |\n| SFT dataset | Joint | Versioned with the filter rules and the subtraction manifest attached, so a regression can be traced to a data decision. |\n| Base model, SFT run, checkpoints | Prime Intellect | Training runs on Prime Intellect compute through the managed `prime-rl` stack; the customer never operates a scheduler, a cluster, or a checkpoint store. |\n| Eval harness | Prime Intellect | Architecture 1's `verifiers` harness, reused — and it must export its item ids into the subtraction step above. If it does not exist, build it first. |\n| Inference endpoint | Prime Intellect or customer VPC | Deployment target decided in scoping, because it changes the DPA. |\n\n**Data path.** The heaviest crossing of the four: the training corpus leaves the customer boundary and the resulting weights encode it. Before that, the eval set is subtracted from the corpus by source id, and the manifest of what was removed ships with the dataset version. Establish early who owns the checkpoint, whether it may touch any other customer (no), and what deletion means. Legal is the long pole here, which is exactly why legal sits second in the pipeline.\n\n**Decisions that change the diagram.** Whether traces contain the reasoning or only the outcome — outcome-only corpora teach answers rather than method and cap the ceiling hard. Whether you adapt an open frontier model or the customer's incumbent. And whether inference lands in Lab or the customer VPC, which decides if the arrow at the right edge crosses back.\n\n**Failure mode of the architecture.** SFT reproduces the average of the corpus, including its bad habits, and there is no mechanism in the diagram that can exceed the demonstrations. If the customer wants behaviour that is better than their current experts rather than cheaper, this shape structurally cannot deliver it and you must move to architecture 3.\n\n## 3. RL in a constructed environment\n\n**Fits when** the outcome is checkable, the task has room above human demonstration, and the workflow can be replayed against a replica of the customer's systems. This is the flagship, and the honest version of the diagram has a human in it.\n\n```mermaid\nflowchart TD\n subgraph Cust[Customer]\n W[Workflow spec]\n Sd[Task seeds]\n Sy[Systems replica]\n H[Human raters]\n end\n subgraph PI[Prime Intellect Lab]\n En[Environment]\n Hb[Environments Hub]\n Po[Policy model]\n Ro[Rollouts]\n Ve[verifiers]\n Rw[Reward]\n Op[prime-rl trainer]\n Sm[Sampled traces]\n Hd[Held-out eval]\n Sh[Ship candidate]\n end\n W --> En\n Sy --> En\n Sd --> En\n En --> Hb --> Ro\n Po --> Ro --> Ve --> Rw --> Op --> Po\n Ro --> Sm --> H --> Ve\n Po --> Hd --> Gt{\"Gain over target\"}\n Gt -- no --> En\n Gt -- yes --> Sh\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Workflow spec | Joint | The written definition of a completed task. Until it exists there is nothing for a verifier to check, which is why projects stall here rather than in training. |\n| Task seeds | Customer | Real starting states. Synthetic seeds produce a policy that wins in a world that does not exist. |\n| Systems replica | Customer, sandboxed by PI | The environment must be steppable and resettable; production systems are neither. |\n| Environment | Prime Intellect | Deterministic reset, bounded episode, observable state. Built against the environments interface and versioned on the Environments Hub, so a rollout six weeks later is reproducible against a named environment version. |\n| Verifier and reward | Prime Intellect, calibrated by customer | The reward function is signed off by the business owner who will be held to the outcome metric, and they sign off having read ten scored rollouts — five at the top of the reward range and five at the bottom — not the specification. |\n| Human raters | Customer | Ongoing, not one-off: they score sampled trajectories the verifier passed, which is the only place two distinct hazards surface. Reward hacking is the policy satisfying the reward rule without doing the task; judge gaming is the policy learning the stylistic tells of the judge model. The first is caught by tightening the rule, the second only by rotating or re-anchoring the judge. |\n| RL trainer and compute | Prime Intellect | Rollout generation and the policy update run on Prime Intellect compute through the managed `prime-rl` trainer; the customer supplies seeds and raters, not a cluster. |\n| Held-out eval | Joint | Frozen before training starts, or the result is unfalsifiable. Held by the customer and answered within the scored-submission budget set in Post-Training POC Structure. Ship gate is a gain of at least {{target_delta}} on the primary metric, measured on this set and nowhere else. |\n\n**Data path.** Task seeds and a systems replica cross into Lab; rollouts generated inside are synthetic. The legal claim worth making is narrow and true: the volume of customer data crossing is far smaller than an SFT corpus, and no raw customer text is encoded into weights. The claim not to make is that nothing sensitive crosses. The seeds are real starting states and the replica's fixture data is real unless someone does the work — insist it is redacted or synthesised, and expect that fixture set to be the thing legal actually reviews.\n\n**Decisions that change the diagram.** Whether the verifier is programmatic or a model judge, since a judge inserts a second model into the reward path that can itself be gamed. Whether the environment stubs the customer's systems or replays them. And the episode horizon: beyond roughly {{horizon_tool_calls}} tool calls the outcome signal is too sparse to train on alone, which forces intermediate rewards and adds a step-rubric build to the environment work.\n\n**Failure mode of the architecture.** The reward is optimised and the business outcome is not. The loop is closed and self-consistent, which means it will happily converge on a verifier artefact. The human-rater arrow is the only defence in the diagram and it is the first thing cut when the timeline slips.\n\n## 4. Agent workflow with tool use\n\n**Fits when** the unit of work is a multi-step job against real tools — a migration, a reconciliation, a research task — and the customer has, or will expose, MCP servers over their systems.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n Sy[Systems of record]\n Rp[Redacted replica]\n Mc[MCP servers]\n Ap[Policy and approvals]\n Hu[Approval gate]\n end\n subgraph PI[Prime Intellect Lab]\n Sb[Sandbox]\n Ag[Agent policy]\n Rc[Trajectory log]\n St[Step verifier]\n Hd[Frozen task set]\n Oc[Outcome check]\n Lp[SFT or RL loop]\n end\n Sy -. clone .-> Rp\n Sy -- read only --> Mc\n Rp -- writes --> Mc\n Mc --> Sb\n Ap --> Hu\n Hu --> Sb\n Sb --> Ag --> Rc\n Ag -- write calls --> Hu\n Rc --> St --> Lp\n Hd --> Oc\n Rc --> Oc --> Lp\n Lp --> Ag\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| MCP servers | Customer, specified by PI | The tool surface is the action space, and the specification bar is concrete: one verb per tool, typed arguments with no free-text catch-all, idempotent wherever the underlying system allows it, and errors that name the corrective action rather than the exception class. |\n| Redacted replica | Customer | Reads may come from the live systems of record; writes must not. The replica is optional only while every exposed tool is read-only, and becomes mandatory the moment one is not. |\n| Policy and approvals | Customer | Which actions are irreversible, which need a human. Draw this before the first rollout. |\n| Approval gate | Customer | A human in the write path. If it exists only in production, the policy was never trained against it and will meet it for the first time on a live task. |\n| Sandbox | Prime Intellect | Isolation, reset, and per-episode credential scoping, on Prime Intellect sandboxes. It isolates the agent from live writes, not from live reads — that is what the replica is for. |\n| Trajectory log | Prime Intellect | The primary asset of the engagement; it is also the future SFT corpus. |\n| Step verifier | Prime Intellect | Credit assignment over long horizons. The recommended method is a rubric judge scoring each tool call against the workflow spec: did this call advance a named precondition of the goal. It rewards looking busy, so cap episode length and score cost per completed task alongside the rubric. |\n| Frozen task set | Joint | A held-out set of whole jobs with known-good endpoints, frozen before the first training run. Without it the outcome check drifts with the training data and the improvement curve means nothing. |\n| Outcome check | Joint | The only measure the customer's business recognises, run against the frozen task set. |\n\n**Data path.** Reads flow from the systems of record through MCP tools into the sandbox on every episode; writes are pointed at the redacted replica, and any write the approvals policy marks irreversible passes the human gate before it reaches a tool. This is still the hardest architecture to reason about for compliance: scope credentials per episode, log every call, and start read-only. The trajectory log will contain production data by construction — treat it with the same controls as the source systems.\n\n**Decisions that change the diagram.** Read-only versus write-capable tools, which decides whether the replica node is present at all. Whether evaluation is per-step or per-outcome, which decides whether the step verifier or the frozen task set carries the training signal. And whether the approval gate sits inside the training loop or only in production, because a gate that exists only in production is untrained-for.\n\n**Failure mode of the architecture.** Evaluation quietly becomes impossible. Long-horizon agentic tasks have many acceptable trajectories and few checkable endpoints, so teams substitute plausibility for correctness and ship an agent nobody can prove is improving. If you cannot state the outcome check in one sentence before the build starts, this is architecture 1 wearing a costume.\n", + "body": "*Playbook stage: scoping.* This is the scoping-stage artefact of the **Strategic Deployment Playbook**: it produces the environment specification the stage exits on. Its companion is the eval-design section of **Post-Training POC Structure**, and its data-path sections are what the legal stage reviews.\n\nFour shapes cover the deployments this playbook is written for. Knowing which one you are in before the scoping call ends is what separates a week-four training run from a week-four discovery that no eval exists, the traces are outcome-only, and the environment cannot be reset. Each shape below has a diagram, a component table with an owner per box, an account of where customer data crosses a boundary, and the two or three decisions that redraw the picture. The failure modes are failures of the architecture rather than of execution: they occur even when everyone does their job well. Every value in double braces is an assumption to be replaced with a figure you can defend in the room; the Prime Intellect component names track the current public stack and should be checked before the deck goes out.\n\n## 1. Eval-first\n\n**Fits when** the customer cannot yet tell you what \"better\" means, has no clean traces, or has a stakeholder who does not believe a model can be improved at all. This is the cheapest engagement and it is usually the correct first one. It is also the only architecture that produces something valuable even if you never train.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n L[Prod logs]\n S[Scrub and sample]\n X[Expert panel]\n G[Adjudicated labels]\n Gc[Calibration half]\n Gh[Held-out half]\n end\n subgraph PI[Prime Intellect Lab]\n E[Eval set]\n M[Model under test]\n V[verifiers lib]\n R[Scorecard]\n F[Eval frozen]\n end\n L --> S --> E\n X --> G\n G --> Gc --> V\n G --> Gh\n E --> M --> V --> R\n R --> D{\"Agreement vs target\"}\n D -- below --> V\n D -- met --> Ha{\"Holds on held-out\"}\n Gh --> Ha\n Ha -- no --> V\n Ha -- yes --> F\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Prod logs and sampler | Customer | The eval must be drawn from real traffic, not from the tasks the team finds interesting. Stratify by the segments the business already reports on. |\n| Scrub and sample | Customer | Redaction happens before egress so the DPA conversation stays narrow. |\n| Expert panel and adjudicated labels | Customer, facilitated by PI | Human adjudication is the only thing that makes a verifier trustworthy. Size the set at {{adjudicated_label_count}} items, or by the operational test: keep labelling until measured agreement is stable across two disjoint halves. |\n| Calibration and held-out halves | Customer | The adjudicated set is split on arrival. The verifier is tuned against the calibration half only; the held-out half is never seen during tuning and exists to catch a verifier fitted to its own reference. |\n| Eval set | Joint, hosted in Lab | Versioned and frozen, so a score in month one is comparable to a score in month six. |\n| Verifier | Prime Intellect | The `verifiers` library, programmatic where possible and model-judge where not. The statistic is Cohen's kappa against the calibration half, item by item; acceptance is kappa at or above {{agreement_target}} on the held-out half, and {{agreement_target}} is not a free parameter: the library floor set in the Strategic Deployment Playbook is Cohen's kappa 0.7 against adjudicated labels, or within 0.10 of a measured human ceiling that is itself below 0.8. Recalibrate on any of three triggers: the judge model version changes, the eval set gains items, or held-out kappa drops below target on a scheduled re-measure. |\n| Disagreement adjudication | Joint | When judge and adjudicated label disagree, a named human decides which is wrong. The adjudicated label is often the error, and relabelling it is a legitimate outcome provided the change is versioned with a reason. |\n| Scorecard | Prime Intellect | Per-segment, not a single aggregate. A single number hides the segment you are actually failing. |\n\n**Data path.** Logs are sampled and redacted inside the customer estate; only the redacted eval set and the adjudicated labels cross into Lab. Results return as scores and, where permitted, sampled failure traces. If the customer will not export even redacted text, the harness runs inside their VPC — the diagram survives, the hosting moves.\n\n**Decisions that change the diagram.** Whether the verifier can be programmatic (a diff, a schema check, a unit test, a lookup against a system of record) or must be a model judge — this determines whether you need the gold-label loop at all, and it is the single largest cost driver. Whether the eval runs on the customer side, which extends the build from {{eval_build_weeks_lab}} weeks to {{eval_build_weeks_customer}}. And whether you evaluate the model or the whole product surface including retrieval and prompts, since only the latter predicts what the business will feel.\n\n**Failure mode of the architecture.** The eval becomes the deliverable and nothing is ever trained. A frozen scorecard is comfortable: it makes the team look rigorous, generates a quarterly artefact, and demands nothing. Write the training trigger into the scope document — a named threshold and a named owner — or this engagement terminates itself politely.\n\n## 2. SFT on traces\n\n**Fits when** the customer already has a large body of expert behaviour recorded: resolved support tickets with the actual resolution, reviewed legal redlines, closed engineering PRs, annotated claims decisions. The behaviour is demonstrable but not currently reproducible by a general model.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n T[Trace store]\n P[Filter and scrub]\n Rv[SME review]\n end\n subgraph PI[Prime Intellect Lab]\n Dx[Eval id subtract]\n D[SFT dataset]\n B[Base model]\n Tn[SFT run]\n Ck[Checkpoints]\n Ev[Eval harness]\n In[Inference]\n end\n T --> P --> Rv --> Dx --> D\n Ev -. eval ids .-> Dx\n B --> Tn\n D --> Tn --> Ck --> Ev\n Ev -- regress --> D\n Ev -- pass --> In\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Trace store | Customer | The asset. Ask for row counts and a schema in discovery, not adjectives. |\n| Filter and scrub | Customer | Traces are kept on a stated outcome rule — the ticket closed without reopening, the PR merged without revert, the redline survived counterparty review — not on volume. Assume roughly the top {{trace_keep_fraction}} of the corpus survives the rule and size the engagement on that, not on the row count. |\n| Eval id subtraction | Prime Intellect | The eval set and the training corpus come from the same records in most customers. Every eval item's source id is subtracted from the corpus before the first run, or a pass on the harness is a pass on memorised rows. |\n| SME review | Customer | Someone must confirm that a trace records good behaviour and not a tolerated one. |\n| SFT dataset | Joint | Versioned with the filter rules and the subtraction manifest attached, so a regression can be traced to a data decision. |\n| Base model, SFT run, checkpoints | Prime Intellect | Training runs on Prime Intellect compute through the managed `prime-rl` stack; the customer never operates a scheduler, a cluster, or a checkpoint store. |\n| Eval harness | Prime Intellect | Architecture 1's `verifiers` harness, reused — and it must export its item ids into the subtraction step above. If it does not exist, build it first. |\n| Inference endpoint | Prime Intellect or customer VPC | Deployment target decided in scoping, because it changes the DPA. |\n\n**Data path.** The heaviest crossing of the four: the training corpus leaves the customer boundary and the resulting weights encode it. Before that, the eval set is subtracted from the corpus by source id, and the manifest of what was removed ships with the dataset version. Establish early who owns the checkpoint, whether it may touch any other customer (no), and what deletion means. Legal is the long pole here, which is exactly why legal sits second in the pipeline.\n\n**Decisions that change the diagram.** Whether traces contain the reasoning or only the outcome — outcome-only corpora teach answers rather than method and cap the ceiling hard. Whether you adapt an open frontier model or the customer's incumbent. And whether inference lands in Lab or the customer VPC, which decides if the arrow at the right edge crosses back.\n\n**Failure mode of the architecture.** SFT reproduces the average of the corpus, including its bad habits, and there is no mechanism in the diagram that can exceed the demonstrations. If the customer wants behaviour that is better than their current experts rather than cheaper, this shape structurally cannot deliver it and you must move to architecture 3.\n\n## 3. RL in a constructed environment\n\n**Fits when** the outcome is checkable, the task has room above human demonstration, and the workflow can be replayed against a replica of the customer's systems. This is the flagship, and the honest version of the diagram has a human in it.\n\n```mermaid\nflowchart TD\n subgraph Cust[Customer]\n W[Workflow spec]\n Sd[Task seeds]\n Sy[Systems replica]\n H[Human raters]\n end\n subgraph PI[Prime Intellect Lab]\n En[Environment]\n Hb[Environments Hub]\n Po[Policy model]\n Ro[Rollouts]\n Ve[verifiers]\n Rw[Reward]\n Op[prime-rl trainer]\n Sm[Sampled traces]\n Hd[Held-out eval]\n Sh[Ship candidate]\n end\n W --> En\n Sy --> En\n Sd --> En\n En --> Hb --> Ro\n Po --> Ro --> Ve --> Rw --> Op --> Po\n Ro --> Sm --> H --> Ve\n Po --> Hd --> Gt{\"Gain over target\"}\n Gt -- no --> En\n Gt -- yes --> Sh\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| Workflow spec | Joint | The written definition of a completed task. Until it exists there is nothing for a verifier to check, which is why projects stall here rather than in training. |\n| Task seeds | Customer | Real starting states. Synthetic seeds produce a policy that wins in a world that does not exist. |\n| Systems replica | Customer, sandboxed by PI | The environment must be steppable and resettable; production systems are neither. |\n| Environment | Prime Intellect | Deterministic reset, bounded episode, observable state. Built against the environments interface and versioned on the Environments Hub, so a rollout six weeks later is reproducible against a named environment version. |\n| Verifier and reward | Prime Intellect, calibrated by customer | The reward function is signed off by the business owner who will be held to the outcome metric, and they sign off having read ten scored rollouts — five at the top of the reward range and five at the bottom — not the specification. |\n| Human raters | Customer | Ongoing, not one-off: they score sampled trajectories the verifier passed, which is the only place two distinct hazards surface. Reward hacking is the policy satisfying the reward rule without doing the task; judge gaming is the policy learning the stylistic tells of the judge model. The first is caught by tightening the rule, the second only by rotating or re-anchoring the judge. |\n| RL trainer and compute | Prime Intellect | Rollout generation and the policy update run on Prime Intellect compute through the managed `prime-rl` trainer; the customer supplies seeds and raters, not a cluster. |\n| Held-out eval | Joint | Frozen before training starts, or the result is unfalsifiable. Held by the customer and answered within the scored-submission budget set in Post-Training POC Structure. Ship gate is a gain of at least {{target_delta}} on the primary metric, measured on this set and nowhere else. |\n\n**Data path.** Task seeds and a systems replica cross into Lab; rollouts generated inside are synthetic. The legal claim worth making is narrow and true: the volume of customer data crossing is far smaller than an SFT corpus, and no raw customer text is encoded into weights. The claim not to make is that nothing sensitive crosses. The seeds are real starting states and the replica's fixture data is real unless someone does the work — insist it is redacted or synthesised, and expect that fixture set to be the thing legal actually reviews.\n\n**Decisions that change the diagram.** Whether the verifier is programmatic or a model judge, since a judge inserts a second model into the reward path that can itself be gamed. Whether the environment stubs the customer's systems or replays them. And the episode horizon: beyond roughly {{horizon_tool_calls}} tool calls the outcome signal is too sparse to train on alone, which forces intermediate rewards and adds a step-rubric build to the environment work.\n\n**Failure mode of the architecture.** The reward is optimised and the business outcome is not. The loop is closed and self-consistent, which means it will happily converge on a verifier artefact. The human-rater arrow is the only defence in the diagram and it is the first thing cut when the timeline slips.\n\n## 4. Agent workflow with tool use\n\n**Fits when** the unit of work is a multi-step job against real tools — a migration, a reconciliation, a research task — and the customer has, or will expose, MCP servers over their systems.\n\n```mermaid\nflowchart LR\n subgraph Cust[Customer]\n Sy[Systems of record]\n Rp[Redacted replica]\n Mc[MCP servers]\n Ap[Policy and approvals]\n Hu[Approval gate]\n end\n subgraph PI[Prime Intellect Lab]\n Sb[Sandbox]\n Ag[Agent policy]\n Rc[Trajectory log]\n St[Step verifier]\n Hd[Frozen task set]\n Oc[Outcome check]\n Lp[SFT or RL loop]\n end\n Sy -. clone .-> Rp\n Sy -- read only --> Mc\n Rp -- writes --> Mc\n Mc --> Sb\n Ap --> Hu\n Hu --> Sb\n Sb --> Ag --> Rc\n Ag -- write calls --> Hu\n Rc --> St --> Lp\n Hd --> Oc\n Rc --> Oc --> Lp\n Lp --> Ag\n```\n\n| Component | Who runs it | Why it is there |\n| --- | --- | --- |\n| MCP servers | Customer, specified by PI | The tool surface is the action space, and the specification bar is concrete: one verb per tool, typed arguments with no free-text catch-all, idempotent wherever the underlying system allows it, and errors that name the corrective action rather than the exception class. |\n| Redacted replica | Customer | Reads may come from the live systems of record; writes must not. The replica is optional only while every exposed tool is read-only, and becomes mandatory the moment one is not. |\n| Policy and approvals | Customer | Which actions are irreversible, which need a human. Draw this before the first rollout. |\n| Approval gate | Customer | A human in the write path. If it exists only in production, the policy was never trained against it and will meet it for the first time on a live task. |\n| Sandbox | Prime Intellect | Isolation, reset, and per-episode credential scoping, on Prime Intellect sandboxes. It isolates the agent from live writes, not from live reads — that is what the replica is for. |\n| Trajectory log | Prime Intellect | The primary asset of the engagement; it is also the future SFT corpus. |\n| Step verifier | Prime Intellect | Credit assignment over long horizons. The recommended method is a rubric judge scoring each tool call against the workflow spec: did this call advance a named precondition of the goal. It rewards looking busy, so cap episode length and score cost per completed task alongside the rubric. |\n| Frozen task set | Joint | A held-out set of whole jobs with known-good endpoints, frozen before the first training run. Without it the outcome check drifts with the training data and the improvement curve means nothing. |\n| Outcome check | Joint | The only measure the customer's business recognises, run against the frozen task set. |\n\n**Data path.** Reads flow from the systems of record through MCP tools into the sandbox on every episode; writes are pointed at the redacted replica, and any write the approvals policy marks irreversible passes the human gate before it reaches a tool. This is still the hardest architecture to reason about for compliance: scope credentials per episode, log every call, and start read-only. The trajectory log will contain production data by construction — treat it with the same controls as the source systems.\n\n**Decisions that change the diagram.** Read-only versus write-capable tools, which decides whether the replica node is present at all. Whether evaluation is per-step or per-outcome, which decides whether the step verifier or the frozen task set carries the training signal. And whether the approval gate sits inside the training loop or only in production, because a gate that exists only in production is untrained-for.\n\n**Failure mode of the architecture.** Evaluation quietly becomes impossible. Long-horizon agentic tasks have many acceptable trajectories and few checkable endpoints, so teams substitute plausibility for correctness and ship an agent nobody can prove is improving. If you cannot state the outcome check in one sentence before the build starts, this is architecture 1 wearing a costume.\n", "fields": { "architectures": [ { diff --git a/packages/db/src/seed/motion/strategic-deployment-playbook.json b/packages/db/src/seed/motion/strategic-deployment-playbook.json index 6437bbd..4491094 100644 --- a/packages/db/src/seed/motion/strategic-deployment-playbook.json +++ b/packages/db/src/seed/motion/strategic-deployment-playbook.json @@ -2,9 +2,9 @@ "kind": "playbook", "slug": "strategic-deployment-playbook", "title": "Strategic Deployment Playbook", - "summary": "The stage-by-stage spine of a forward deployed engagement, from first technical discovery to expansion, naming the artifact, the checkable exit test and the silent failure mode at each step.", + "summary": "The stage-by-stage spine of a forward deployed engagement, from first technical discovery to expansion, naming the artefact, the checkable exit test and the silent failure mode at each step.", "stage": "qualification", - "body": "# What this playbook is\n\nThis is the spine of the Motion library. Every other artifact hangs off a stage in it. If you read one file before walking into a new logo, read this one, then open the artifact named for the stage you are actually in.\n\nTwo conventions before the content. Any number written in double braces, like {{10}}, is a stated planning assumption rather than an observed figure from a real deal; nothing in this file is a claim about what past customers have done. And the stage tag on this document is a filing artifact, not its scope: the playbook governs all eight stages and is tagged at qualification only so it surfaces before the first call.\n\nThe pipeline is PIG's: qualification, legal, scoping, proposal, procurement, poc, deployment, expansion. Legal sits second on purpose. In AI infrastructure the MSA and DPA are not the paperwork at the end of a handshake; they are the gate that decides whether the customer can hand you a single trace of real data. A scoping conversation held before the DPA is signed is a conversation about hypothetical workflows, and hypothetical workflows do not become environments.\n\nTwo rules apply everywhere. First, no stage exits on a feeling. Every exit criterion below is written so a third party who was not in the room could look at an artifact and say yes or no, which means every exit names the artifact and where it lives. Second, the forward deployed lead owns the artifact, not the meeting. Meetings that do not produce or advance a named artifact are how deals die politely.\n\n# The library\n\nEight templates hang off the stages below. This is the whole list, and every artifact named anywhere in this file is one of these eight or is explicitly flagged as not yet written.\n\n| Template | Stage it serves | What it is used for |\n| --- | --- | --- |\n| **First Technical Discovery** | qualification | The question set for the first deep technical call. Its seven artefacts are what the qualification exit summary is written from. |\n| **Trainability and Deal Qualification Scorecard** | qualification | Scored at the end of discovery, before legal. It is the gate that stops an unqualified deal consuming counsel's time at stage two. |\n| **Reference Architectures** | scoping | Decides which of the four deployment shapes this is, and turns the workflow into an environment specification. |\n| **Technical Narratives for Applied AI Deals** | scoping, usable at any stage | Seven spoken explanations for a deal stalled on a misconception rather than on price. Its own table maps each narrative to a stage. |\n| **Proposal Blocks** | proposal | The customer-facing paragraphs: the single success threshold, and the data-handling block that legal and procurement both reuse. |\n| **Pricing and Packaging Inputs** | proposal | Builds and defends the number, and separates the compute line from the engagement fee so procurement can route each. |\n| **Post-Training POC Structure** | poc | Owns the falsifiable hypothesis, the eval and verifier method, the week-by-week plan, the kill gates and the weekly scorecard. |\n| **Case Study Frame** | expansion | The write-up structure and the during-engagement harvest that makes it writable. Produced at expansion, consumed back at qualification and proposal. |\n\nThree stages have no template of their own. **Legal** is served for now by the data-handling block in **Proposal Blocks** and the data-path sections of **Reference Architectures**; **procurement** by the budget-source and compute-line material in **Pricing and Packaging Inputs**; **deployment** by narrative 7, \"Prototype to production\", in **Technical Narratives for Applied AI Deals**. Each is a real gap. Until it is filled, do not go looking for a template that does not exist.\n\n# The stages\n\n## Qualification\n\nYou enter when there is a named workflow, not a named budget. Budget without a workflow is a compute conversation and belongs to a different motion.\n\nYour job is to find out whether this customer has something trainable. That means three things exist: a repeated task with a decision in it, a record of how that task went (traces, logs, tickets, adjudications, anything with an outcome attached), and a human today who can tell you whether an output was right. If any of the three is missing you are looking at a prompting problem, and you should say so early — you will be invited back.\n\nRun the **First Technical Discovery**. It is built to force numbers out of people: volume per week, current accuracy and how it is measured, cost per task, who reviews the output. Answers like \"a lot\" and \"pretty good\" are the finding, not a failed question.\n\nApplied Research does not attend qualification. Bringing a researcher to a first call spends your scarcest internal currency on a meeting you can run alone.\n\nExit: a written qualification summary saved on the demand in PIG naming the workflow, the weekly volume, the current quality measure, the internal owner of that workflow, and the product line you expect to sell against. Assume {{10}} days.\n\n## Legal\n\nYou enter on an observable event, not on warmth: a named person on the customer side has been asked for a sample of real traces and has not refused, so the request now sits with their legal or security function rather than with your champion's goodwill. MSA and DPA start at that moment, in parallel with everything else, because the review clock is external to you and cannot be compressed later.\n\nYour job is not to negotiate the contract. It is to make sure the paper being drafted permits the technical thing you intend to do. That list is short and specific: can traces leave the customer's environment, can outputs be retained for evaluation, can the resulting weights be used only by them, where does inference run, and what is the retention window on sandbox artifacts. Bring the data-handling block from **Proposal Blocks**, backed by the data-path section of whichever shape in **Reference Architectures** you are proposing, to the first legal call and get each line initialled by whoever owns it on their side.\n\nExit: MSA and DPA executed, or a signed data-handling addendum sufficient for a limited scoping data pull, with the permitted-use lines identified against that block. Assume {{30}} days. This is the assumption most worth re-checking per account, because it is set by the depth of the customer's legal queue, not by anything you control.\n\n## Scoping\n\nYou enter with executed paper and access to at least a sample of real traces.\n\nYou are converting a workflow into two things: an environment specification and an evaluation a sceptic would accept. Use **Reference Architectures** for the first and the eval-design section of **Post-Training POC Structure** for the second. Run them in that order but iterate them together — an environment whose success signal you cannot compute is not an environment, it is a simulator.\n\n**Verifier quality is the decision the deal turns on, so measure it rather than asserting it.** Deterministic checks beat model judges. Model judges beat nothing, but only after you have a number. The measurement method — sample size and stratification, the adjudication rule, the chance-corrected statistic and its confidence interval — is owned by the verifier section of **Post-Training POC Structure** and is not restated here. Run it as written there, on at least {{200}} items stratified so the minority outcome is at least {{30}} of them. What this playbook owns is the decision rule on top of it. Report the two customer adjudicators' agreement with each other first: if the humans cannot agree — below roughly {{0.6}} on Cohen's kappa as a working line — there is no ground truth to verify against, and what the customer has is a labelling engagement, not a training one. Say that out loud at scoping. Only once the humans agree do you measure the judge against the adjudicated label. **The floor for the whole library is Cohen's kappa {{0.7}} against adjudicated labels, or within {{0.10}} of the measured human ceiling where that ceiling is itself below {{0.8}}.** Every other template defers to that number. Below it the judge is a triage filter, not a reward signal, and the proposal must say so in those words.\n\n**Trainability triage decides whether the deal is even scopeable.** Three failures kill post-training work technically and none of them announce themselves. The first is reward hacking: the harder case is not a signal you cannot compute but one you can, which the policy then exploits. Before the POC, spend one day trying to maximise the verifier without doing the task. If you can beat it in a day, so will the policy, and you have found the real scope of work. The second is contamination: when eval traces and training traces come from the same customer log dump, near-duplicate tickets land on both sides of the split and the POC reports a number nobody can reproduce in production. Split by time period, never at random, and write the cut-off date into the scope. The third is method choice, which sets the trace count and therefore the price: assume the ranges in the data table of **Post-Training POC Structure**, which is the authority — {{1,000}} to {{20,000}} traces for supervised fine-tuning, where quality dominates volume, {{2,000}}-plus preference pairs for preference optimisation, and a working programmatic verifier rather than a trace count for RL. If the customer's available data does not clear the threshold for any of the three, the honest scope is a data collection engagement first.\n\n**Make the baseline fair before you make it flattering.** The oldest dishonest number in this business is a delta measured against the customer's unoptimised prompt. The baseline is the strongest cheap alternative, not the incumbent's current state: a prompted frontier model given the same context the trained model will get, tuned by us for at least a day. If the delta over that baseline is not worth the engagement fee, say so at scoping and keep the relationship.\n\nApplied Research enters here, once, on a structured hand-in. See the interface section below.\n\nExit: an environment spec and an eval spec, each reviewed by Applied Research, plus a baseline number for the current system measured on that eval, with the verifier agreement number and the train/eval cut-off date recorded alongside it. No baseline, no scope. Assume {{21}} days.\n\n## Proposal\n\nYou enter with a reviewed scope and a measured baseline.\n\nWrite the proposal from **Proposal Blocks**, and build the number with **Pricing and Packaging Inputs**. It contains exactly one success threshold, expressed as a delta from the measured baseline on the agreed eval, and it names who signs off that the threshold was met. If the customer wants three thresholds, you have three POCs or you have none.\n\nPrice against the post-training and evaluation product lines for the POC itself, with the compute line separated so procurement can route it. Separating compute from the engagement fee is the difference between one approval and two.\n\nExit: the threshold accepted in writing — the economic buyer's reply on the proposal thread, or their name and the date logged against the threshold line on the demand in PIG. A verbal yes is not an exit, because a third party cannot check it. Assume {{10}} days.\n\n## Procurement\n\nYou enter with the threshold accepted in writing. This stage is logistical and entirely about being early, but it is not the generic B2B version of early, because three things on this purchase order are unfamiliar to most procurement functions.\n\nThe compute line has its own approval path and you should ask which one it takes on day one: whether it lands as operating expense or as a capacity commitment, because a commitment usually needs a second signature and sometimes a finance review the engagement fee does not trigger. Ask early whether this procurement team has ever bought GPU-hours before; a unit they cannot forecast is a unit they escalate. And expect questions that only appear on AI vendor questionnaires — model provenance and licence lineage, where weights are stored and who can extract them, whether any training data crosses a border, and export-control exposure on the compute region. The data-handling block in **Proposal Blocks** answers most of these already; reuse it rather than re-answering, and add any new question to it afterwards.\n\nAlso ask for the vendor onboarding form, the approver chain, and whether the purchase needs a competitive quote. None of that is hard; all of it is slow if requested late.\n\nExit: PO issued or order form countersigned. Assume {{25}} days.\n\n## POC\n\nYou enter with a signed order and a kick-off date.\n\nKeep the POC measured against the thing you agreed and nothing else. Scope drift during a POC almost always arrives as generosity — the customer asks for one more case type, you say yes, and the eval no longer matches the proposal.\n\nRun the weekly scorecard from **Post-Training POC Structure** and send it to the sponsor whether the news is good or bad. A weekly number that sometimes moves down is more credible than a monthly deck that only moves up.\n\nExit: final scorecard, signed by the named sign-off, stating met or not met against the single threshold. Assume {{56}} days for the eight-week shape in **Post-Training POC Structure**, or {{28}} for its compressed four-week variant; those two are the only shapes, and the three things the compressed one may not drop are listed there.\n\n## Deployment\n\nYou enter when the threshold is met and there is a production owner on the customer side. If nobody owns it in production, you have a successful POC and no deployment.\n\nWork a production readiness checklist. No Motion template owns deployment yet, so build it from narrative 7, \"Prototype to production\", in **Technical Narratives for Applied AI Deals**: inference path and latency budget, fallback behaviour when the model is unavailable, monitoring on the eval in production, the retraining trigger, and who gets paged. The retraining trigger matters most — a model that is never refreshed decays into the reason they churn.\n\nExit: the traffic percentage recorded on the production readiness checklist is actually being served, evidenced by a screenshot or dashboard link of the split; the eval has been running on live traffic for at least {{7}} days; and the alert has a named on-call owner in the customer's own rota, not a shared mailbox. Assume {{40}} days.\n\n## Expansion\n\nYou enter with a model in production and at least one month of live eval data.\n\nUse **Case Study Frame**. The second workflow is chosen for shared verifier infrastructure, and this is the whole point of the stage rather than a phrase in it. A shared verifier means the second workflow's outcome is judged by the same mechanism as the first: the same adjudication rubric, the same held-out construction, often the same deterministic checks with different thresholds. That halves the next scope for concrete reasons. The verifier agreement number is already measured, so scoping does not have to stop for a labelling exercise. The held-out set already exists and its time-based cut-off is already defended. The customer's two adjudicators are already trained and have already argued with each other, which is the expensive part nobody budgets for. What does not carry over is the eval itself — the task set, the reward weights, the held-out split and the baseline are rebuilt for every workflow, and **Pricing and Packaging Inputs** re-costs them at close to full price each time. The saving is in the adjudication apparatus and the environment scaffolding, never in the eval. In practice this means expanding along a judgement type rather than along a department — a second workflow judged by the same quality bar is cheaper to scope than a larger workflow judged by a new one, even when the larger one is worth more.\n\nThe case study is the second output, and its approval cycle should start now rather than at renewal, because communications approval is the same external queue problem as legal.\n\nExit: a second scoped workflow entered as its own demand in PIG, or a signed expansion order. Assume {{60}} days.\n\n# The stall diagnostic\n\nDeals rarely die of \"no\". They die of ambiguity nobody is impolite enough to name. The terse symptom-and-move pair for each stage is in the structured fields; what follows is why each move works, and for the three that are hardest to compose in the moment, the actual words.\n\nAt qualification, the symptom is warmth without numbers. Asking for one week of real traces under NDA works because it converts an opinion about interest into an act that requires someone's approval; whichever way it goes, you learn the true state in days rather than months. A refusal is not a setback, it is the qualification answer.\n\nAt legal, redlines returning slower each round means the MSA has been queued behind bigger contracts. The limited scoping addendum works because it decouples the two clocks: technical work resumes on a narrow permission while the full agreement continues at its own pace, and progress on the technical side is what keeps the champion's attention.\n\nAt scoping, when every workflow sounds equally attractive, nobody will choose because business value is the axis on which everything is defensible. Ranking by verifier cost works because it is an axis the customer has never ranked on and cannot argue with from habit; presenting the ranking invites the correction that reveals the real priority.\n\nAt proposal, silence after issue is almost never an unread document. It is a threshold someone cannot defend internally and will not say so. Send one line:\n\n> Quick check before I chase anything else — is the threshold the wrong number? Happy to re-cut it against a different slice of the eval if that is the sticking point.\n\nThat works because it makes the easy reply the informative one. \"It is with the team\" costs them nothing; \"the number is wrong\" is the thing you need and this phrasing makes it polite to say.\n\nAt procurement, the champion going quiet usually means they do not know how to drive their own purchasing process and are embarrassed. Approach procurement directly with the questionnaire as the pretext:\n\n> I have your security questionnaire ready to return and wanted to send it to the right owner rather than sit in your champion's inbox. Could you point me at the vendor onboarding form and the approver chain for a purchase with a compute line in it?\n\nThat works because a questionnaire creates a ticket in procurement's system, whereas vendor mail creates a favour in someone's inbox, and ticket-shaped requests get answered faster than favour-shaped ones.\n\nAt POC, a scorecard that slips a week and then two is a lead protecting a bad number. Publish it anyway with the missing week marked blank; the gap is the message, and a visible gap forces the conversation that a delayed deck postpones.\n\nAt deployment, go-live dates that move without an engineering name attached mean the work has no owner. Escalate to the sponsor with a dated go-live in writing:\n\n> We are holding the {{15 March}} go-live on the plan, but with no platform owner named it will slip visibly rather than quietly. Who should I be working with, or should I move the date?\n\nThat works because it makes the absence of a name into a scheduling problem the sponsor owns, rather than an engineering problem nobody owns.\n\nAt expansion, praise without a second workflow means the value is felt but has never been stated in front of anyone who allocates budget. Bringing the case study draft works because approving it obliges the sponsor to defend the value internally, and that argument almost always names the next workflow.\n\n# The Applied Research interface\n\nResearch time is the scarcest thing in this motion. Protect it with a contract in both directions. What a scope must contain, and what you owe back, are listed in the structured fields; below is one filled-in hand-in so a lead can pattern-match rather than interpret.\n\nThe example is illustrative and entirely fictional. No part of it describes a real customer, deal or measurement.\n\n**Workflow.** Triage of inbound warranty claims for a mid-market appliance retailer. Input: a claim record — free-text customer description, purchase date, product SKU, prior claim history, and up to three photographs. Output: one of four dispositions (approve, deny, request more evidence, escalate to human specialist), plus a one-sentence reason. Real-shaped example in: *\"Dishwasher leaking from the base, bought 14 months ago, no receipt but I have the delivery email\"* with SKU and two photos attached. Expected out: *request more evidence — proof of purchase absent and the 12-month base warranty has expired, so the extended-cover check needs the delivery confirmation.*\n\n**Verifier.** Deterministic checks for the mechanical part of the decision (warranty window arithmetic from purchase date, SKU cover class lookup, presence of required evidence fields), plus a model judge for whether the stated reason matches the disposition. Cost per call: {{0.004}} USD, against a human review cost of {{4.10}} USD. Agreement: two customer adjudicators reached Cohen's kappa {{0.71}} with each other on a {{200}}-item stratified sample; the judge reached {{0.74}} against the adjudicated label. Above the triage-filter line, usable as a reward signal, and the proposal states both numbers.\n\n**Traces.** {{18,000}} adjudicated claims over {{14}} months, each with the specialist's final disposition attached. Split by time: everything before {{1 October}} trains, everything after is held out. Provenance: the customer's own claims system export, no third-party data.\n\n**Baseline.** A prompted frontier model with the same claim record and the same cover-class lookup, tuned by us for two days: {{0.62}} agreement with the adjudicated label on the held-out slice. The trained model must beat that, not the incumbent rules engine.\n\n**Deployment constraint.** {{800}} ms at p95 for the disposition, inference inside the customer's cloud tenancy, no claim text leaving it. Peak {{4,000}} claims per hour on Mondays.\n\n**Known failure modes.** Photographs of a different appliance than the SKU; customers describing a symptom that is covered while requesting a remedy that is not; two claims filed on one incident.\n\nA hand-in in this shape is actionable. One missing the verifier is not a scope.\n\n# The promotion ritual\n\nAt close — won or lost — the deployment pays rent by making the next one cheaper. This ritual gets the same treatment as a stage, because a ritual nobody can check is exactly the unchecked completion this playbook exists to prevent.\n\nOwner: the forward deployed lead who ran the deal, personally, not the account team.\n\nArtifact: a promotion note linked from the closed demand in PIG that lists each of the seven triggers in the structured fields and records, for each, either what was promoted and where it went, or the words \"not applicable\" and why. A note that says nothing was promotable is a valid note; a missing note is not.\n\nDeadline: within two weeks of close, while you still remember what was hard.\n\nHow it dies: the lead moves to the next logo in week one, the note is never written, and the second deployment costs as much as the first — which is the same failure as an unowned production model, one layer up.\n\nPromote the environment spec into the shared environment catalogue, wherever the team keeps it, whenever the workflow generalises beyond one customer. Promote the eval and its verifier agreement number into the shared eval library whenever a second customer would recognise the task. Promote any new question that changed your mind during discovery into the **First Technical Discovery**. Promote a legal term you fought for into the data-handling block of **Proposal Blocks**. Promote a stall you had not seen before into this playbook. On a loss, promote the reason as a disqualifying question — a loss that becomes a question at qualification is a cheaper loss than the same deal run twice.", + "body": "# What this playbook is\n\nThis is the spine of the Motion library. Every other artefact hangs off a stage in it. If you read one file before walking into a new logo, read this one, then open the artefact named for the stage you are actually in.\n\nTwo conventions before the content. Any number written in double braces, like {{10}}, is a stated planning assumption rather than an observed figure from a real deal; nothing in this file is a claim about what past customers have done. And the stage tag on this document is a filing artefact, not its scope: the playbook governs all eight stages and is tagged at qualification only so it surfaces before the first call.\n\nThe pipeline is PIG's: qualification, legal, scoping, proposal, procurement, poc, deployment, expansion. Legal sits second on purpose. In AI infrastructure the MSA and DPA are not the paperwork at the end of a handshake; they are the gate that decides whether the customer can hand you a single trace of real data. A scoping conversation held before the DPA is signed is a conversation about hypothetical workflows, and hypothetical workflows do not become environments.\n\nTwo rules apply everywhere. First, no stage exits on a feeling. Every exit criterion below is written so a third party who was not in the room could look at an artefact and say yes or no, which means every exit names the artefact and where it lives. Second, the forward deployed lead owns the artefact, not the meeting. Meetings that do not produce or advance a named artefact are how deals die politely.\n\n# The library\n\nEight templates hang off the stages below. This is the whole list, and every artefact named anywhere in this file is one of these eight or is explicitly flagged as not yet written.\n\n| Template | Stage it serves | What it is used for |\n| --- | --- | --- |\n| **First Technical Discovery** | qualification | The question set for the first deep technical call. Its seven artefacts are what the qualification exit summary is written from. |\n| **Trainability and Deal Qualification Scorecard** | qualification | Scored at the end of discovery, before legal. It is the gate that stops an unqualified deal consuming counsel's time at stage two. |\n| **Reference Architectures** | scoping | Decides which of the four deployment shapes this is, and turns the workflow into an environment specification. |\n| **Technical Narratives for Applied AI Deals** | scoping, usable at any stage | Seven spoken explanations for a deal stalled on a misconception rather than on price. Its own table maps each narrative to a stage. |\n| **Proposal Blocks** | proposal | The customer-facing paragraphs: the single success threshold, and the data-handling block that legal and procurement both reuse. |\n| **Pricing and Packaging Inputs** | proposal | Builds and defends the number, and separates the compute line from the engagement fee so procurement can route each. |\n| **Post-Training POC Structure** | poc | Owns the falsifiable hypothesis, the eval and verifier method, the week-by-week plan, the kill gates and the weekly scorecard. |\n| **Case Study Frame** | expansion | The write-up structure and the during-engagement harvest that makes it writable. Produced at expansion, consumed back at qualification and proposal. |\n\nThree stages have no template of their own. **Legal** is served for now by the data-handling block in **Proposal Blocks** and the data-path sections of **Reference Architectures**; **procurement** by the budget-source and compute-line material in **Pricing and Packaging Inputs**; **deployment** by narrative 7, \"Prototype to production\", in **Technical Narratives for Applied AI Deals**. Each is a real gap. Until it is filled, do not go looking for a template that does not exist.\n\n# The stages\n\n## Qualification\n\nYou enter when there is a named workflow, not a named budget. Budget without a workflow is a compute conversation and belongs to a different motion.\n\nYour job is to find out whether this customer has something trainable. That means three things exist: a repeated task with a decision in it, a record of how that task went (traces, logs, tickets, adjudications, anything with an outcome attached), and a human today who can tell you whether an output was right. If any of the three is missing you are looking at a prompting problem, and you should say so early — you will be invited back.\n\nRun the **First Technical Discovery**. It is built to force numbers out of people: volume per week, current accuracy and how it is measured, cost per task, who reviews the output. Answers like \"a lot\" and \"pretty good\" are the finding, not a failed question.\n\nApplied Research does not attend qualification. Bringing a researcher to a first call spends your scarcest internal currency on a meeting you can run alone.\n\nExit: a written qualification summary saved on the demand in PIG naming the workflow, the weekly volume, the current quality measure, the internal owner of that workflow, and the product line you expect to sell against. Assume {{10}} days.\n\n## Legal\n\nYou enter on an observable event, not on warmth: a named person on the customer side has been asked for a sample of real traces and has not refused, so the request now sits with their legal or security function rather than with your champion's goodwill. MSA and DPA start at that moment, in parallel with everything else, because the review clock is external to you and cannot be compressed later.\n\nYour job is not to negotiate the contract. It is to make sure the paper being drafted permits the technical thing you intend to do. That list is short and specific: can traces leave the customer's environment, can outputs be retained for evaluation, can the resulting weights be used only by them, where does inference run, and what is the retention window on sandbox artefacts. Bring the data-handling block from **Proposal Blocks**, backed by the data-path section of whichever shape in **Reference Architectures** you are proposing, to the first legal call and get each line initialled by whoever owns it on their side.\n\nExit: MSA and DPA executed, or a signed data-handling addendum sufficient for a limited scoping data pull, with the permitted-use lines identified against that block. Assume {{30}} days. This is the assumption most worth re-checking per account, because it is set by the depth of the customer's legal queue, not by anything you control.\n\n## Scoping\n\nYou enter with executed paper and access to at least a sample of real traces.\n\nYou are converting a workflow into two things: an environment specification and an evaluation a sceptic would accept. Use **Reference Architectures** for the first and the eval-design section of **Post-Training POC Structure** for the second. Run them in that order but iterate them together — an environment whose success signal you cannot compute is not an environment, it is a simulator.\n\n**Verifier quality is the decision the deal turns on, so measure it rather than asserting it.** Deterministic checks beat model judges. Model judges beat nothing, but only after you have a number. The measurement method — sample size and stratification, the adjudication rule, the chance-corrected statistic and its confidence interval — is owned by the verifier section of **Post-Training POC Structure** and is not restated here. Run it as written there, on at least {{200}} items stratified so the minority outcome is at least {{30}} of them. What this playbook owns is the decision rule on top of it. Report the two customer adjudicators' agreement with each other first: if the humans cannot agree — below roughly {{0.6}} on Cohen's kappa as a working line — there is no ground truth to verify against, and what the customer has is a labelling engagement, not a training one. Say that out loud at scoping. Only once the humans agree do you measure the judge against the adjudicated label. **The floor for the whole library is Cohen's kappa {{0.7}} against adjudicated labels, or within {{0.10}} of the measured human ceiling where that ceiling is itself below {{0.8}}.** Every other template defers to that number. Below it the judge is a triage filter, not a reward signal, and the proposal must say so in those words.\n\n**Trainability triage decides whether the deal is even scopeable.** Three failures kill post-training work technically and none of them announce themselves. The first is reward hacking: the harder case is not a signal you cannot compute but one you can, which the policy then exploits. Before the POC, spend one day trying to maximise the verifier without doing the task. If you can beat it in a day, so will the policy, and you have found the real scope of work. The second is contamination: when eval traces and training traces come from the same customer log dump, near-duplicate tickets land on both sides of the split and the POC reports a number nobody can reproduce in production. Split by time period, never at random, and write the cut-off date into the scope. The third is method choice, which sets the trace count and therefore the price: assume the ranges in the data table of **Post-Training POC Structure**, which is the authority — {{1,000}} to {{20,000}} traces for supervised fine-tuning, where quality dominates volume, {{2,000}}-plus preference pairs for preference optimisation, and a working programmatic verifier rather than a trace count for RL. If the customer's available data does not clear the threshold for any of the three, the honest scope is a data collection engagement first.\n\n**Make the baseline fair before you make it flattering.** The oldest dishonest number in this business is a delta measured against the customer's unoptimised prompt. The baseline is the strongest cheap alternative, not the incumbent's current state: a prompted frontier model given the same context the trained model will get, tuned by us for at least a day. If the delta over that baseline is not worth the engagement fee, say so at scoping and keep the relationship.\n\nApplied Research enters here, once, on a structured hand-in. See the interface section below.\n\nExit: an environment spec and an eval spec, each reviewed by Applied Research, plus a baseline number for the current system measured on that eval, with the verifier agreement number and the train/eval cut-off date recorded alongside it. No baseline, no scope. Assume {{21}} days.\n\n## Proposal\n\nYou enter with a reviewed scope and a measured baseline.\n\nWrite the proposal from **Proposal Blocks**, and build the number with **Pricing and Packaging Inputs**. It contains exactly one success threshold, expressed as a delta from the measured baseline on the agreed eval, and it names who signs off that the threshold was met. If the customer wants three thresholds, you have three POCs or you have none.\n\nPrice against the post-training and evaluation product lines for the POC itself, with the compute line separated so procurement can route it. Separating compute from the engagement fee is the difference between one approval and two.\n\nExit: the threshold accepted in writing — the economic buyer's reply on the proposal thread, or their name and the date logged against the threshold line on the demand in PIG. A verbal yes is not an exit, because a third party cannot check it. Assume {{10}} days.\n\n## Procurement\n\nYou enter with the threshold accepted in writing. This stage is logistical and entirely about being early, but it is not the generic B2B version of early, because three things on this purchase order are unfamiliar to most procurement functions.\n\nThe compute line has its own approval path and you should ask which one it takes on day one: whether it lands as operating expense or as a capacity commitment, because a commitment usually needs a second signature and sometimes a finance review the engagement fee does not trigger. Ask early whether this procurement team has ever bought GPU-hours before; a unit they cannot forecast is a unit they escalate. And expect questions that only appear on AI vendor questionnaires — model provenance and licence lineage, where weights are stored and who can extract them, whether any training data crosses a border, and export-control exposure on the compute region. The data-handling block in **Proposal Blocks** answers most of these already; reuse it rather than re-answering, and add any new question to it afterwards.\n\nAlso ask for the vendor onboarding form, the approver chain, and whether the purchase needs a competitive quote. None of that is hard; all of it is slow if requested late.\n\nExit: PO issued or order form countersigned. Assume {{25}} days.\n\n## POC\n\nYou enter with a signed order and a kick-off date.\n\nKeep the POC measured against the thing you agreed and nothing else. Scope drift during a POC almost always arrives as generosity — the customer asks for one more case type, you say yes, and the eval no longer matches the proposal.\n\nRun the weekly scorecard from **Post-Training POC Structure** and send it to the sponsor whether the news is good or bad. A weekly number that sometimes moves down is more credible than a monthly deck that only moves up.\n\nExit: final scorecard, signed by the named sign-off, stating met or not met against the single threshold. Assume {{56}} days for the eight-week shape in **Post-Training POC Structure**, or {{28}} for its compressed four-week variant; those two are the only shapes, and the three things the compressed one may not drop are listed there.\n\n## Deployment\n\nYou enter when the threshold is met and there is a production owner on the customer side. If nobody owns it in production, you have a successful POC and no deployment.\n\nWork a production readiness checklist. No Motion template owns deployment yet, so build it from narrative 7, \"Prototype to production\", in **Technical Narratives for Applied AI Deals**: inference path and latency budget, fallback behaviour when the model is unavailable, monitoring on the eval in production, the retraining trigger, and who gets paged. The retraining trigger matters most — a model that is never refreshed decays into the reason they churn.\n\nExit: the traffic percentage recorded on the production readiness checklist is actually being served, evidenced by a screenshot or dashboard link of the split; the eval has been running on live traffic for at least {{7}} days; and the alert has a named on-call owner in the customer's own rota, not a shared mailbox. Assume {{40}} days.\n\n## Expansion\n\nYou enter with a model in production and at least one month of live eval data.\n\nUse **Case Study Frame**. The second workflow is chosen for shared verifier infrastructure, and this is the whole point of the stage rather than a phrase in it. A shared verifier means the second workflow's outcome is judged by the same mechanism as the first: the same adjudication rubric, the same held-out construction, often the same deterministic checks with different thresholds. That halves the next scope for concrete reasons. The verifier agreement number is already measured, so scoping does not have to stop for a labelling exercise. The held-out set already exists and its time-based cut-off is already defended. The customer's two adjudicators are already trained and have already argued with each other, which is the expensive part nobody budgets for. What does not carry over is the eval itself — the task set, the reward weights, the held-out split and the baseline are rebuilt for every workflow, and **Pricing and Packaging Inputs** re-costs them at close to full price each time. The saving is in the adjudication apparatus and the environment scaffolding, never in the eval. In practice this means expanding along a judgement type rather than along a department — a second workflow judged by the same quality bar is cheaper to scope than a larger workflow judged by a new one, even when the larger one is worth more.\n\nThe case study is the second output, and its approval cycle should start now rather than at renewal, because communications approval is the same external queue problem as legal.\n\nExit: a second scoped workflow entered as its own demand in PIG, or a signed expansion order. Assume {{60}} days.\n\n# The stall diagnostic\n\nDeals rarely die of \"no\". They die of ambiguity nobody is impolite enough to name. The terse symptom-and-move pair for each stage is in the structured fields; what follows is why each move works, and for the three that are hardest to compose in the moment, the actual words.\n\nAt qualification, the symptom is warmth without numbers. Asking for one week of real traces under NDA works because it converts an opinion about interest into an act that requires someone's approval; whichever way it goes, you learn the true state in days rather than months. A refusal is not a setback, it is the qualification answer.\n\nAt legal, redlines returning slower each round means the MSA has been queued behind bigger contracts. The limited scoping addendum works because it decouples the two clocks: technical work resumes on a narrow permission while the full agreement continues at its own pace, and progress on the technical side is what keeps the champion's attention.\n\nAt scoping, when every workflow sounds equally attractive, nobody will choose because business value is the axis on which everything is defensible. Ranking by verifier cost works because it is an axis the customer has never ranked on and cannot argue with from habit; presenting the ranking invites the correction that reveals the real priority.\n\nAt proposal, silence after issue is almost never an unread document. It is a threshold someone cannot defend internally and will not say so. Send one line:\n\n> Quick check before I chase anything else — is the threshold the wrong number? Happy to re-cut it against a different slice of the eval if that is the sticking point.\n\nThat works because it makes the easy reply the informative one. \"It is with the team\" costs them nothing; \"the number is wrong\" is the thing you need and this phrasing makes it polite to say.\n\nAt procurement, the champion going quiet usually means they do not know how to drive their own purchasing process and are embarrassed. Approach procurement directly with the questionnaire as the pretext:\n\n> I have your security questionnaire ready to return and wanted to send it to the right owner rather than sit in your champion's inbox. Could you point me at the vendor onboarding form and the approver chain for a purchase with a compute line in it?\n\nThat works because a questionnaire creates a ticket in procurement's system, whereas vendor mail creates a favour in someone's inbox, and ticket-shaped requests get answered faster than favour-shaped ones.\n\nAt POC, a scorecard that slips a week and then two is a lead protecting a bad number. Publish it anyway with the missing week marked blank; the gap is the message, and a visible gap forces the conversation that a delayed deck postpones.\n\nAt deployment, go-live dates that move without an engineering name attached mean the work has no owner. Escalate to the sponsor with a dated go-live in writing:\n\n> We are holding the {{15 March}} go-live on the plan, but with no platform owner named it will slip visibly rather than quietly. Who should I be working with, or should I move the date?\n\nThat works because it makes the absence of a name into a scheduling problem the sponsor owns, rather than an engineering problem nobody owns.\n\nAt expansion, praise without a second workflow means the value is felt but has never been stated in front of anyone who allocates budget. Bringing the case study draft works because approving it obliges the sponsor to defend the value internally, and that argument almost always names the next workflow.\n\n# The Applied Research interface\n\nResearch time is the scarcest thing in this motion. Protect it with a contract in both directions. What a scope must contain, and what you owe back, are listed in the structured fields; below is one filled-in hand-in so a lead can pattern-match rather than interpret.\n\nThe example is illustrative and entirely fictional. No part of it describes a real customer, deal or measurement.\n\n**Workflow.** Triage of inbound warranty claims for a mid-market appliance retailer. Input: a claim record — free-text customer description, purchase date, product SKU, prior claim history, and up to three photographs. Output: one of four dispositions (approve, deny, request more evidence, escalate to human specialist), plus a one-sentence reason. Real-shaped example in: *\"Dishwasher leaking from the base, bought 14 months ago, no receipt but I have the delivery email\"* with SKU and two photos attached. Expected out: *request more evidence — proof of purchase absent and the 12-month base warranty has expired, so the extended-cover check needs the delivery confirmation.*\n\n**Verifier.** Deterministic checks for the mechanical part of the decision (warranty window arithmetic from purchase date, SKU cover class lookup, presence of required evidence fields), plus a model judge for whether the stated reason matches the disposition. Cost per call: {{0.004}} USD, against a human review cost of {{4.10}} USD. Agreement: two customer adjudicators reached Cohen's kappa {{0.71}} with each other on a {{200}}-item stratified sample; the judge reached {{0.74}} against the adjudicated label. Above the triage-filter line, usable as a reward signal, and the proposal states both numbers.\n\n**Traces.** {{18,000}} adjudicated claims over {{14}} months, each with the specialist's final disposition attached. Split by time: everything before {{1 October}} trains, everything after is held out. Provenance: the customer's own claims system export, no third-party data.\n\n**Baseline.** A prompted frontier model with the same claim record and the same cover-class lookup, tuned by us for two days: {{0.62}} agreement with the adjudicated label on the held-out slice. The trained model must beat that, not the incumbent rules engine.\n\n**Deployment constraint.** {{800}} ms at p95 for the disposition, inference inside the customer's cloud tenancy, no claim text leaving it. Peak {{4,000}} claims per hour on Mondays.\n\n**Known failure modes.** Photographs of a different appliance than the SKU; customers describing a symptom that is covered while requesting a remedy that is not; two claims filed on one incident.\n\nA hand-in in this shape is actionable. One missing the verifier is not a scope.\n\n# The promotion ritual\n\nAt close — won or lost — the deployment pays rent by making the next one cheaper. This ritual gets the same treatment as a stage, because a ritual nobody can check is exactly the unchecked completion this playbook exists to prevent.\n\nOwner: the forward deployed lead who ran the deal, personally, not the account team.\n\nArtefact: a promotion note linked from the closed demand in PIG that lists each of the seven triggers in the structured fields and records, for each, either what was promoted and where it went, or the words \"not applicable\" and why. A note that says nothing was promotable is a valid note; a missing note is not.\n\nDeadline: within two weeks of close, while you still remember what was hard.\n\nHow it dies: the lead moves to the next logo in week one, the note is never written, and the second deployment costs as much as the first — which is the same failure as an unowned production model, one layer up.\n\nPromote the environment spec into the shared environment catalogue, wherever the team keeps it, whenever the workflow generalises beyond one customer. Promote the eval and its verifier agreement number into the shared eval library whenever a second customer would recognise the task. Promote any new question that changed your mind during discovery into the **First Technical Discovery**. Promote a legal term you fought for into the data-handling block of **Proposal Blocks**. Promote a stall you had not seen before into this playbook. On a loss, promote the reason as a disqualifying question — a loss that becomes a question at qualification is a cheaper loss than the same deal run twice.", "fields": { "stages": [ { @@ -213,5 +213,5 @@ } ] }, - "notes": "This template exists to stop the most expensive failure in forward deployed work: a stage that ends on a good feeling rather than an artifact. Every exit criterion here names an artifact and where it lives, so a third party can check it without having been in the room. If you cannot name which artifact the last meeting advanced, the deal is already stalled and you have not noticed. Numbers written in double braces are stated planning assumptions, not observed figures from real deals or commitments to a customer. The stage tag on this document is a filing convention so the playbook surfaces before the first call; its scope is all eight stages. This file also holds the library's shared vocabulary: verifier (the whole scoring mechanism), judge (a model-based verifier), adjudicator and rater (humans), adjudicated label, environment, held-out set, dev split, baseline, POC, demand record in PIG, forward deployed lead, Applied Research. Where a template disagrees with the verifier floor or the POC shape stated here, this file governs." + "notes": "This template exists to stop the most expensive failure in forward deployed work: a stage that ends on a good feeling rather than an artefact. Every exit criterion here names an artefact and where it lives, so a third party can check it without having been in the room. If you cannot name which artefact the last meeting advanced, the deal is already stalled and you have not noticed. Numbers written in double braces are stated planning assumptions, not observed figures from real deals or commitments to a customer. The stage tag on this document is a filing convention so the playbook surfaces before the first call; its scope is all eight stages. This file also holds the library's shared vocabulary: verifier (the whole scoring mechanism), judge (a model-based verifier), adjudicator and rater (humans), adjudicated label, environment, held-out set, dev split, baseline, POC, demand record in PIG, forward deployed lead, Applied Research. Where a template disagrees with the verifier floor or the POC shape stated here, this file governs." } diff --git a/packages/db/src/seed/motion/technical-discovery.json b/packages/db/src/seed/motion/technical-discovery.json index d84a0e4..601760f 100644 --- a/packages/db/src/seed/motion/technical-discovery.json +++ b/packages/db/src/seed/motion/technical-discovery.json @@ -4,7 +4,7 @@ "title": "First Technical Discovery", "summary": "The question set and run-of-show for the first deep technical conversation with a strategic customer, used to decide whether a real workflow is trainable before anyone writes a proposal.", "stage": "qualification", - "body": "*Playbook stage: qualification.* This is the first artifact in the **Strategic Deployment Playbook** — the call it runs produces the numbers the qualification exit summary is written from, and the **Trainability and Deal Qualification Scorecard** is scored from these notes immediately afterwards.\n\n## What this call is for\n\nA first technical discovery with a strategic customer has exactly one job: decide, with evidence, whether a real workflow inside this company can be turned into a trainable target — and if it can, learn enough to scope an environment, an eval and a verifier without a second round of archaeology.\n\nIt is not a demo. It is not a capabilities tour. If you spend the hour explaining what we do, you will leave with enthusiasm and no scope, and the proposal you write two weeks later will be fiction assembled from vibes.\n\nRun it with two people from your side: the forward deployed lead, and a forward deployed engineer who can smell an unlearnable task from across the table. Applied Research does not attend a first call — it is the scarcest internal currency in this motion, and the **Strategic Deployment Playbook** spends it once, at scoping, on a structured hand-in. On their side you want the person who owns the metric and at least one person who actually does or maintains the workflow. If the room contains only executives, you will get the strategy, not the truth. If it contains only engineers, you will get the truth and no budget.\n\nNinety minutes. Twenty on the workflow, fifteen on the failed AI attempt, twenty on data, traces and whether the environment can be built, ten on verification, ten on the incumbent model and its economics, ten on the org and the counterfactual, and five held back so the last question is never rushed.\n\n## The one thing you are listening for\n\nEverything below is a way of answering a single question: **is there a repeatable task, with recoverable traces, whose outcome can be checked by something other than a human's opinion?**\n\nThose three properties — repeatable, traced, checkable — are the whole test of the task. Miss any one of them and you do not have an RL problem yet. You may still have a valuable engagement (evaluations, inference, a data programme that creates the traces), but you do not have a post-training engagement, and pretending otherwise is how a POC becomes an eight-month apology.\n\nThere is a fourth question, and it is not about the task but about their estate: can the environment actually be built? A task can be perfectly trainable and still out of reach, because the tools only exist behind a UI, because every attempt sends a real email or moves real money, because there is no staging tenant with data resembling production, or because no outsider has ever held a credential against one. Ask this in the room, not in the scoping week. It moves the timeline more than anything else in this document.\n\n## Trainable versus a prompting problem\n\nThe most valuable outcome of a discovery call is often a fast, honest disqualification. Say it in the room. A customer who hears \"this is a prompting problem, here is how I would solve it in a fortnight, come back to us when you hit the ceiling\" will trust the eventual yes far more than one who was sold a training run.\n\n| Signals this is trainable | Signals this is a prompting problem |\n| --- | --- |\n| The task runs thousands of times a month with the same shape | It runs a handful of times, each one bespoke |\n| Failures are systematic and repeat: the same tool misused, the same edge case dropped | Failures are scattered and each has a different explanation |\n| Success is checkable by code, a test suite, a downstream system, or a reconciled ground truth | Success is checkable only by asking a senior person how they feel about it |\n| Multiple steps, tool calls, and decisions that depend on earlier results | One shot in, one answer out, no branching |\n| Historical traces exist with the actions taken, not just the final answer | Only final artefacts survive; the reasoning path was never recorded |\n| The frontier model plateaus with a good prompt, few-shot examples and a retrieval layer already in place | Nobody has tried a decent system prompt yet |\n| The domain has private knowledge, conventions or tools that are not on the public internet | The knowledge is public and the model already has it |\n| Latency or unit cost forces a smaller model than the workflow needs | Cost and latency are comfortable at the frontier |\n\nThe last two are the ones customers underrate. Post-training earns its keep either when the behaviour is not in any base model, or when the required quality has to fit in a cost and latency envelope the frontier model cannot meet. Both are quantifiable. Ask for the numbers.\n\nTwo traps in the other direction. First, \"we already tried fine-tuning and it didn't work\" is not a disqualification — ask what they trained on, how many examples, whether it was SFT on final answers only, and whether they had a reward signal at all. Second, a plateau on a *prompted* frontier model with a good scaffold is the most encouraging signal in this entire document. It means the easy headroom is gone and the remaining gap is a training gap.\n\n## What to write down before you leave\n\nYou need seven artefacts from this call. If you cannot produce them from your notes, the call was not finished and you should book the follow-up before you get on the plane.\n\n1. **The task, in one sentence, with an actor and a boundary.** \"Given a customer's ticket and access to {{internal tools}}, produce a resolution that passes {{check}} without escalation.\" If the sentence needs an \"and also\", you have two tasks. Pick one.\n2. **The current baseline number.** Whatever the customer's own metric is, measured today, by them, on their data. Not a benchmark. If it does not exist, creating it is the first deliverable of the engagement and you should say so out loud.\n3. **The verifier sketch.** What checks the outcome, how often it is wrong, and in which direction it is wrong. A verifier that is 90% accurate but biased towards passing is far more dangerous than one that is 80% accurate and symmetric.\n4. **The environment sketch.** Which systems have to be callable outside their UI, which of them have a sandbox, roughly how many tool calls one episode takes, and what an attempt breaks in the real world if it fails halfway. If every call has an irreversible side effect and there is no staging tenant, building one is the first phase of work and the proposal has to price it as weeks.\n5. **The data path.** Where the traces live, who owns them, whether they can leave the customer's VPC, and what the legal route looks like. The DPA determines the architecture, so it has to be started before scoping, not after the proposal.\n6. **The economics.** Current model, current spend and current latency, the fully loaded human cost of doing the work today, and the envelope the workflow actually needs. Inference spend on its own is rarely the ROI argument for a post-training deal; headcount, vendor invoices and the cost of a failure that reached a customer usually are.\n7. **The disqualifier you did not find.** Write down what would have killed it. If nothing would have, you did not push hard enough.\n\n## How to ask\n\nAsk for a file, a number, a name or a screen share. A question a customer can answer \"yes\" to without thinking has taught you nothing. \"Do you have data?\" is worthless. \"Can you open the table and tell me the row count and the earliest timestamp?\" is a discovery call.\n\nWhen you hear a round number, ask how it was measured. When you hear \"roughly\", ask who would know exactly. When you hear a passive verb — \"it gets reviewed\", \"it's approved\" — ask who does it, because the passive voice is where accountability goes to hide and it is usually where your veto risk lives.\n\nThere is one moment where silence earns its keep: after you ask how often their existing check gets it wrong. The first answer is the SLA number. The second, arriving a long fifteen seconds later from the engineer at the end of the table, is what they have actually seen in production, and it is the one that sizes the verifier-measurement work.\n\n## Reading the room afterwards\n\nScore the opportunity on the three properties before you leave the car park: repeatable (yes/no), traced (yes/partial/no), checkable (automatic/proxy/human-only). Then apply the rule rather than the impression. This is the in-the-room read, not the qualification decision: the formal score is the twelve-dimension **Trainability and Deal Qualification Scorecard**, run from these notes within the day, and where the two disagree the Scorecard governs.\n\n- Repeatable no ends the conversation regardless of the other two. A bespoke task is not a training target at any budget, and saying so today costs you nothing.\n- Three yeses is a scoping conversation this quarter.\n- Traced partial with an automatic verifier is still strong, because traces can be generated once the environment exists. But that generation is the first phase of the work, not a preliminary to it: scope it in weeks, price it, and say so before the proposal rather than after someone has read the timeline.\n- Checkable proxy with traces is a scoping conversation with a verifier-measurement phase in front of it. The proxy gets measured against human judgement before anything is trained against it.\n- Human-only checking with no proxy is a research problem you should not sell as a POC.\n- Anything carrying two nos is a fast disqualification, and you say it in the room today.\n\nAnd record the counterfactual honestly. A customer who would be fine doing nothing for another year is not a customer with a deadline, whatever the enthusiasm in the room suggested. The forcing function — a launch, a contract, a cost line under scrutiny, a competitor — is what converts a good technical fit into a signed order form.", + "body": "*Playbook stage: qualification.* This is the first artefact in the **Strategic Deployment Playbook** — the call it runs produces the numbers the qualification exit summary is written from, and the **Trainability and Deal Qualification Scorecard** is scored from these notes immediately afterwards.\n\n## What this call is for\n\nA first technical discovery with a strategic customer has exactly one job: decide, with evidence, whether a real workflow inside this company can be turned into a trainable target — and if it can, learn enough to scope an environment, an eval and a verifier without a second round of archaeology.\n\nIt is not a demo. It is not a capabilities tour. If you spend the hour explaining what we do, you will leave with enthusiasm and no scope, and the proposal you write two weeks later will be fiction assembled from vibes.\n\nRun it with two people from your side: the forward deployed lead, and a forward deployed engineer who can smell an unlearnable task from across the table. Applied Research does not attend a first call — it is the scarcest internal currency in this motion, and the **Strategic Deployment Playbook** spends it once, at scoping, on a structured hand-in. On their side you want the person who owns the metric and at least one person who actually does or maintains the workflow. If the room contains only executives, you will get the strategy, not the truth. If it contains only engineers, you will get the truth and no budget.\n\nNinety minutes. Twenty on the workflow, fifteen on the failed AI attempt, twenty on data, traces and whether the environment can be built, ten on verification, ten on the incumbent model and its economics, ten on the org and the counterfactual, and five held back so the last question is never rushed.\n\n## The one thing you are listening for\n\nEverything below is a way of answering a single question: **is there a repeatable task, with recoverable traces, whose outcome can be checked by something other than a human's opinion?**\n\nThose three properties — repeatable, traced, checkable — are the whole test of the task. Miss any one of them and you do not have an RL problem yet. You may still have a valuable engagement (evaluations, inference, a data programme that creates the traces), but you do not have a post-training engagement, and pretending otherwise is how a POC becomes an eight-month apology.\n\nThere is a fourth question, and it is not about the task but about their estate: can the environment actually be built? A task can be perfectly trainable and still out of reach, because the tools only exist behind a UI, because every attempt sends a real email or moves real money, because there is no staging tenant with data resembling production, or because no outsider has ever held a credential against one. Ask this in the room, not in the scoping week. It moves the timeline more than anything else in this document.\n\n## Trainable versus a prompting problem\n\nThe most valuable outcome of a discovery call is often a fast, honest disqualification. Say it in the room. A customer who hears \"this is a prompting problem, here is how I would solve it in a fortnight, come back to us when you hit the ceiling\" will trust the eventual yes far more than one who was sold a training run.\n\n| Signals this is trainable | Signals this is a prompting problem |\n| --- | --- |\n| The task runs thousands of times a month with the same shape | It runs a handful of times, each one bespoke |\n| Failures are systematic and repeat: the same tool misused, the same edge case dropped | Failures are scattered and each has a different explanation |\n| Success is checkable by code, a test suite, a downstream system, or a reconciled ground truth | Success is checkable only by asking a senior person how they feel about it |\n| Multiple steps, tool calls, and decisions that depend on earlier results | One shot in, one answer out, no branching |\n| Historical traces exist with the actions taken, not just the final answer | Only final artefacts survive; the reasoning path was never recorded |\n| The frontier model plateaus with a good prompt, few-shot examples and a retrieval layer already in place | Nobody has tried a decent system prompt yet |\n| The domain has private knowledge, conventions or tools that are not on the public internet | The knowledge is public and the model already has it |\n| Latency or unit cost forces a smaller model than the workflow needs | Cost and latency are comfortable at the frontier |\n\nThe last two are the ones customers underrate. Post-training earns its keep either when the behaviour is not in any base model, or when the required quality has to fit in a cost and latency envelope the frontier model cannot meet. Both are quantifiable. Ask for the numbers.\n\nTwo traps in the other direction. First, \"we already tried fine-tuning and it didn't work\" is not a disqualification — ask what they trained on, how many examples, whether it was SFT on final answers only, and whether they had a reward signal at all. Second, a plateau on a *prompted* frontier model with a good scaffold is the most encouraging signal in this entire document. It means the easy headroom is gone and the remaining gap is a training gap.\n\n## What to write down before you leave\n\nYou need seven artefacts from this call. If you cannot produce them from your notes, the call was not finished and you should book the follow-up before you get on the plane.\n\n1. **The task, in one sentence, with an actor and a boundary.** \"Given a customer's ticket and access to {{internal tools}}, produce a resolution that passes {{check}} without escalation.\" If the sentence needs an \"and also\", you have two tasks. Pick one.\n2. **The current baseline number.** Whatever the customer's own metric is, measured today, by them, on their data. Not a benchmark. If it does not exist, creating it is the first deliverable of the engagement and you should say so out loud.\n3. **The verifier sketch.** What checks the outcome, how often it is wrong, and in which direction it is wrong. A verifier that is 90% accurate but biased towards passing is far more dangerous than one that is 80% accurate and symmetric.\n4. **The environment sketch.** Which systems have to be callable outside their UI, which of them have a sandbox, roughly how many tool calls one episode takes, and what an attempt breaks in the real world if it fails halfway. If every call has an irreversible side effect and there is no staging tenant, building one is the first phase of work and the proposal has to price it as weeks.\n5. **The data path.** Where the traces live, who owns them, whether they can leave the customer's VPC, and what the legal route looks like. The DPA determines the architecture, so it has to be started before scoping, not after the proposal.\n6. **The economics.** Current model, current spend and current latency, the fully loaded human cost of doing the work today, and the envelope the workflow actually needs. Inference spend on its own is rarely the ROI argument for a post-training deal; headcount, vendor invoices and the cost of a failure that reached a customer usually are.\n7. **The disqualifier you did not find.** Write down what would have killed it. If nothing would have, you did not push hard enough.\n\n## How to ask\n\nAsk for a file, a number, a name or a screen share. A question a customer can answer \"yes\" to without thinking has taught you nothing. \"Do you have data?\" is worthless. \"Can you open the table and tell me the row count and the earliest timestamp?\" is a discovery call.\n\nWhen you hear a round number, ask how it was measured. When you hear \"roughly\", ask who would know exactly. When you hear a passive verb — \"it gets reviewed\", \"it's approved\" — ask who does it, because the passive voice is where accountability goes to hide and it is usually where your veto risk lives.\n\nThere is one moment where silence earns its keep: after you ask how often their existing check gets it wrong. The first answer is the SLA number. The second, arriving a long fifteen seconds later from the engineer at the end of the table, is what they have actually seen in production, and it is the one that sizes the verifier-measurement work.\n\n## Reading the room afterwards\n\nScore the opportunity on the three properties before you leave the car park: repeatable (yes/no), traced (yes/partial/no), checkable (automatic/proxy/human-only). Then apply the rule rather than the impression. This is the in-the-room read, not the qualification decision: the formal score is the twelve-dimension **Trainability and Deal Qualification Scorecard**, run from these notes within the day, and where the two disagree the Scorecard governs.\n\n- Repeatable no ends the conversation regardless of the other two. A bespoke task is not a training target at any budget, and saying so today costs you nothing.\n- Three yeses is a scoping conversation this quarter.\n- Traced partial with an automatic verifier is still strong, because traces can be generated once the environment exists. But that generation is the first phase of the work, not a preliminary to it: scope it in weeks, price it, and say so before the proposal rather than after someone has read the timeline.\n- Checkable proxy with traces is a scoping conversation with a verifier-measurement phase in front of it. The proxy gets measured against human judgement before anything is trained against it.\n- Human-only checking with no proxy is a research problem you should not sell as a POC.\n- Anything carrying two nos is a fast disqualification, and you say it in the room today.\n\nAnd record the counterfactual honestly. A customer who would be fine doing nothing for another year is not a customer with a deadline, whatever the enthusiasm in the room suggested. The forcing function — a launch, a contract, a cost line under scrutiny, a competitor — is what converts a good technical fit into a signed order form.", "fields": { "sections": [ { diff --git a/scripts/motion-overflow-check.mjs b/scripts/motion-overflow-check.mjs index a3ccdc5..163091b 100644 --- a/scripts/motion-overflow-check.mjs +++ b/scripts/motion-overflow-check.mjs @@ -49,12 +49,31 @@ for (const theme of ['light', 'dark']) { isMobile: vp.name === 'mobile', hasTouch: vp.name === 'mobile', }); - // The sidebar and the dock are per-device states in localStorage; left to - // whatever a human last set, the measurement is not reproducible. + /* + * The sidebar and the dock are per-device states in localStorage; left to + * whatever a human last set, the measurement is not reproducible. The key + * names are `pig.sidebarOpen` and `pig.piggyDockOpen` — an earlier version + * of this file invented `pig.sidebar` and `pig.piggy.dock`, which set two + * keys nothing reads and left the real state untouched, so the run looked + * pinned and was not. + */ await ctx.addInitScript(` - localStorage.setItem('pig.sidebar', 'expanded'); - localStorage.setItem('pig.piggy.dock', 'closed'); + localStorage.setItem('pig.sidebarOpen', 'true'); + localStorage.setItem('pig.piggyDockOpen', 'false'); + localStorage.setItem('pig.themeMode', '${theme}'); `); + /* + * colorScheme alone only sets prefers-color-scheme, and the appearance + * preference is stored SERVER-side and adopted after hydration — so the + * dark pass rendered light a beat after first paint and measured the wrong + * theme. Rewriting the profile response is what screenshots.mjs does, and + * for the same reason. + */ + await ctx.route('**/api/me/profile', async (route) => { + const response = await route.fetch(); + const body = await response.json().catch(() => ({})); + await route.fulfill({ json: { ...body, themeMode: theme } }); + }); for (const route of ROUTES) { const page = await ctx.newPage(); @@ -90,11 +109,27 @@ for (const theme of ['light', 'dark']) { .split(' ') .slice(0, 4) .join('.')}`); - return { overflow, wide, text: document.body.innerText.slice(0, 120) }; + return { + overflow, + wide, + theme: doc.getAttribute('data-theme'), + chars: document.body.innerText.trim().length, + text: document.body.innerText.slice(0, 120), + }; }); const label = `${route.slug} ${vp.name} ${theme}`; - if (measured.overflow !== 0) { + if (measured.theme !== theme) { + // Without this the dark pass silently measured the light palette, and a + // dark-only overflow — a wider border, a different font fallback — + // would have gone unseen while the run reported ok. + failures++; + console.log(`FAIL ${label}: rendered data-theme=${measured.theme}, not ${theme}`); + } else if (measured.chars < 120) { + // A page that throws inside its own body still measures zero overflow. + failures++; + console.log(`FAIL ${label}: rendered only ${measured.chars} characters — the page is blank`); + } else if (measured.overflow !== 0) { failures++; console.log(`FAIL ${label}: overflows by ${measured.overflow}px — ${measured.wide.join(', ')}`); } else if (problems.length) {