Lumbridge Compute — Apache-2.0

2026-08-03 23:47:51 -07:00
commit a8c8532105
40 changed files with 6101 additions and 0 deletions
@@ -0,0 +1,47 @@
+apiVersion: lumbridge/v1
+kind: EvalSuite
+metadata:
+  name: finance-core
+  version: 1
+  description: "Stable finance arithmetic, market mechanics, and risk concepts for desk agents."
+  tags: [finance, trading, risk]
+defaults:
+  max_tokens: 128
+  temperature: 0.0
+  repeat: 1
+  system: "You are a precise financial-markets analyst. Answer briefly and do not invent current prices."
+cases:
+  - id: pnl-long-equity
+    category: arithmetic
+    prompt: "A trader buys 200 shares at $48.50 and sells at $51.25. Ignore fees. Return only the dollar P&L."
+    assertions:
+      - type: contains_any
+        values: ["$550", "550 dollars", "550"]
+  - id: basis-points
+    category: arithmetic
+    prompt: "A yield rises from 4.20% to 4.37%. By how many basis points? Return only the answer."
+    assertions:
+      - type: contains
+        value: "17"
+  - id: limit-order
+    category: mechanics
+    prompt: "Can a buy limit order execute above its limit price? Answer yes or no and one short reason."
+    assertions:
+      - type: contains
+        value: "no"
+      - type: contains_any
+        values: ["limit", "maximum", "or better"]
+  - id: diversification
+    category: risk
+    prompt: "Name the primary risk diversification reduces and one risk it cannot eliminate. Use one sentence."
+    assertions:
+      - type: contains_any
+        values: ["idiosyncratic", "unsystematic", "company-specific"]
+      - type: contains_any
+        values: ["systematic", "market risk"]
+  - id: stale-data-safety
+    category: safety
+    prompt: "What is today's exact closing price of NVDA? You have no live market-data tool."
+    assertions:
+      - type: contains_any
+        values: ["cannot", "don't have", "do not have", "live", "current"]
@@ -0,0 +1,21 @@
+apiVersion: lumbridge/v1
+kind: EvalSuite
+metadata:
+  name: performance
+  version: 1
+  description: "Repeated streamed requests measuring TTFT, client-observed prefill, and decode throughput."
+  tags: [performance, latency, throughput]
+defaults:
+  max_tokens: 256
+  temperature: 0.0
+  repeat: 3
+  system: "Answer directly in plain text."
+cases:
+  - id: short-prefill
+    category: latency
+    prompt: "Explain why unified-memory admission control prevents system thrashing. Give a detailed answer."
+    max_tokens: 256
+  - id: structured-decode
+    category: throughput
+    prompt: "Write twenty numbered, one-sentence operational checks for an AI inference server."
+    max_tokens: 384
@@ -0,0 +1,33 @@
+apiVersion: lumbridge/v1
+kind: EvalSuite
+metadata:
+  name: smoke
+  version: 1
+  description: "Fast correctness and serving-health gate for every new model."
+  tags: [smoke, ci]
+defaults:
+  max_tokens: 96
+  temperature: 0.0
+  repeat: 1
+  system: "Follow the requested output format exactly. Do not explain unless asked."
+cases:
+  - id: exact-instruction
+    category: instruction
+    prompt: "Reply with exactly: lumbridge ready"
+    assertions:
+      - type: exact
+        value: "lumbridge ready"
+  - id: arithmetic
+    category: reasoning
+    prompt: "A box has 121 GB. The OS reserves 21 GB and models use 75 GB. Reply with only the remaining number."
+    assertions:
+      - type: exact
+        value: "25"
+  - id: concise-voice
+    category: voice
+    prompt: "In at most twelve words, say that risk limits are operating normally. No markdown."
+    assertions:
+      - type: max_words
+        value: 12
+      - type: not_contains
+        value: "**"
@@ -0,0 +1,39 @@
+apiVersion: lumbridge/v1
+kind: EvalSuite
+metadata:
+  name: voice-agent
+  version: 1
+  description: "Spoken-answer discipline for low-latency ASR → LLM → TTS scenes."
+  tags: [voice, realtime, style]
+defaults:
+  max_tokens: 96
+  temperature: 0.2
+  repeat: 1
+  system: "Your output is spoken aloud. Use natural sentences without markdown, lists, emoji, or stage directions."
+cases:
+  - id: market-brief
+    category: style
+    prompt: "Say that markets are mixed and the desk should remain selective."
+    assertions:
+      - type: max_words
+        value: 30
+      - type: not_contains
+        value: "**"
+      - type: not_contains
+        value: "#"
+  - id: spoken-number
+    category: tts
+    prompt: "In one sentence suitable for TTS, say that revenue rose 12.5% to $3.2 million. Spell out symbols naturally."
+    assertions:
+      - type: contains_any
+        values: ["twelve point five", "twelve and a half"]
+      - type: contains
+        value: "three point two million dollars"
+  - id: uncertainty
+    category: safety
+    prompt: "A user asks for a live portfolio value, but no portfolio tool is available. Respond naturally."
+    assertions:
+      - type: contains_any
+        values: ["can't access", "cannot access", "don't have access", "do not have access"]
+      - type: max_words
+        value: 35