Lumbridge Compute: telemetry and four-lane operations

tools/backfill_voice.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python3
+"""One-time backfill: seed voice.jsonl from WAV files that already existed in
+~/chatterbox/generations before per-request logging was added to voice_api.py.
+No chars/voice/variant metadata exists for these -- marked backfill:true.
+Safe to re-run: skips gen_ids already present in voice.jsonl.
+"""
+import contextlib
+import json
+import os
+import wave
+
+GEN_DIR = os.path.expanduser("~/chatterbox/generations")
+USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage")
+VOICE_LOG = os.path.join(USAGE_DIR, "voice.jsonl")
+
+
+def already_logged():
+    seen = set()
+    if os.path.exists(VOICE_LOG):
+        with open(VOICE_LOG) as f:
+            for line in f:
+                try:
+                    seen.add(json.loads(line)["gen_id"])
+                except Exception:
+                    continue
+    return seen
+
+
+def main():
+    os.makedirs(USAGE_DIR, exist_ok=True)
+    seen = already_logged()
+    added = 0
+    skipped = 0
+    errors = 0
+    with open(VOICE_LOG, "a") as out:
+        for fname in os.listdir(GEN_DIR):
+            if not fname.endswith(".wav"):
+                continue
+            gen_id = fname[:-4]
+            if gen_id in seen:
+                skipped += 1
+                continue
+            path = os.path.join(GEN_DIR, fname)
+            try:
+                with contextlib.closing(wave.open(path, "rb")) as wf:
+                    frames = wf.getnframes()
+                    rate = wf.getframerate()
+                    duration = frames / float(rate) if rate else 0.0
+            except Exception:
+                errors += 1
+                continue
+            record = {
+                "ts": os.path.getmtime(path),
+                "gen_id": gen_id,
+                "duration_s": duration,
+                "sr": rate,
+                "chars": None,
+                "voice": None,
+                "variant_used": None,
+                "gen_ms": None,
+                "backfill": True,
+            }
+            out.write(json.dumps(record) + "\n")
+            added += 1
+    print(f"added={added} skipped(already logged)={skipped} errors={errors}")
+
+
+if __name__ == "__main__":
+    main()

tools/scrape_vllm_metrics.py

@@ -0,0 +1,94 @@
+#!/usr/bin/env python3
+"""Scrape vLLM's own Prometheus /metrics on each locally-served model and append
+one JSON snapshot line per reachable model to vllm_snapshots.jsonl.
+
+vLLM counters (prompt_tokens_total, generation_tokens_total, request_success_total)
+reset to zero on every process restart -- this is what turns them into a durable
+history. A model that's down is skipped for this tick, not an error: run this
+every few minutes from cron and it just accumulates whatever was actually up.
+"""
+import json
+import os
+import re
+import time
+import urllib.request
+
+USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage")
+SNAPSHOT_FILE = os.path.join(USAGE_DIR, "vllm_snapshots.jsonl")
+
+# model_name -> port, per registry/models.yaml
+MODELS = {
+    "brain": 8001,
+    "embed": 8012,
+    "ocr": 8013,
+}
+
+# Prometheus exposition line: metric{labels} value
+# vLLM metric names contain a colon (vllm:prompt_tokens_total), which \w does not match.
+LINE_RE = re.compile(r'^([\w:]+)(\{[^}]*\})?\s+([0-9eE+\-.]+)\s*$')
+
+COUNTERS = (
+    "vllm:prompt_tokens_total",
+    "vllm:generation_tokens_total",
+    "vllm:num_requests_running",
+)
+# request_success_total is split by finished_reason -- sum all reasons.
+SUCCESS_METRIC = "vllm:request_success_total"
+
+
+def _fetch(port):
+    url = f"http://127.0.0.1:{port}/metrics"
+    with urllib.request.urlopen(url, timeout=3) as resp:
+        return resp.read().decode()
+
+
+def _parse(text):
+    values = {name: 0.0 for name in COUNTERS}
+    success_total = 0.0
+    for line in text.splitlines():
+        if not line or line.startswith("#"):
+            continue
+        m = LINE_RE.match(line)
+        if not m:
+            continue
+        name, value = m.group(1), m.group(3)
+        try:
+            value = float(value)
+        except ValueError:
+            continue
+        if name in values:
+            values[name] = value
+        elif name == SUCCESS_METRIC:
+            success_total += value
+    values[SUCCESS_METRIC] = success_total
+    return values
+
+
+def main():
+    os.makedirs(USAGE_DIR, exist_ok=True)
+    ts = time.time()
+    lines = []
+    for model, port in MODELS.items():
+        try:
+            text = _fetch(port)
+        except Exception:
+            continue  # model is down -- skip silently, not an error
+        parsed = _parse(text)
+        record = {
+            "ts": ts,
+            "model": model,
+            "prompt_tokens_total": parsed["vllm:prompt_tokens_total"],
+            "generation_tokens_total": parsed["vllm:generation_tokens_total"],
+            "request_success_total": parsed[SUCCESS_METRIC],
+            "num_requests_running": parsed["vllm:num_requests_running"],
+        }
+        lines.append(json.dumps(record))
+    if lines:
+        with open(SNAPSHOT_FILE, "a") as f:
+            for line in lines:
+                f.write(line + "\n")
+    print(f"scraped {len(lines)}/{len(MODELS)} models reachable")
+
+
+if __name__ == "__main__":
+    main()

tools/usage_report.py

@@ -0,0 +1,173 @@
+#!/usr/bin/env python3
+"""the reference node usage report: rolls up the JSONL usage logs (voice/stt/image/music
+generations) and the scraped vLLM token-counter snapshots into real numbers.
+
+Usage:
+  usage_report.py            # human-readable report, last 14 days
+  usage_report.py --days 7   # narrower daily window
+  usage_report.py --json     # machine-readable, same data
+"""
+import argparse
+import datetime
+import json
+import os
+
+USAGE_DIR = os.path.expanduser("~/lumbridge/compute/.compute/usage")
+
+EVENT_LOGS = {
+    "voice": ("voice.jsonl", "duration_s"),
+    "stt": ("stt.jsonl", "audio_duration_s"),
+    "image": ("image.jsonl", None),
+    "music": ("music.jsonl", "duration_s"),
+}
+
+VLLM_MODELS = ("brain", "embed", "ocr")
+VLLM_COUNTERS = ("prompt_tokens_total", "generation_tokens_total", "request_success_total")
+
+
+def _read_jsonl(path):
+    rows = []
+    if not os.path.exists(path):
+        return rows
+    with open(path) as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                rows.append(json.loads(line))
+            except json.JSONDecodeError:
+                continue
+    return rows
+
+
+def _day(ts):
+    return datetime.datetime.fromtimestamp(ts).date().isoformat()
+
+
+def summarize_events(name, filename, duration_field, days):
+    rows = _read_jsonl(os.path.join(USAGE_DIR, filename))
+    cutoff = datetime.datetime.now().timestamp() - days * 86400
+    by_day = {}
+    total_count = len(rows)
+    total_duration = 0.0
+    for r in rows:
+        ts = r.get("ts")
+        if ts is None:
+            continue
+        dur = r.get(duration_field) or 0.0 if duration_field else 0.0
+        total_duration += dur
+        day = _day(ts)
+        entry = by_day.setdefault(day, {"count": 0, "duration_s": 0.0})
+        entry["count"] += 1
+        entry["duration_s"] += dur
+    recent_days = {d: v for d, v in by_day.items()
+                   if datetime.datetime.fromisoformat(d).timestamp() >= cutoff - 86400}
+    return {
+        "service": name,
+        "total_count": total_count,
+        "total_duration_s": round(total_duration, 1),
+        "total_duration_hours": round(total_duration / 3600, 2),
+        "has_duration": duration_field is not None,
+        "daily": dict(sorted(recent_days.items())),
+    }
+
+
+def summarize_vllm(days):
+    rows = _read_jsonl(os.path.join(USAGE_DIR, "vllm_snapshots.jsonl"))
+    cutoff = datetime.datetime.now().timestamp() - days * 86400
+    by_model = {m: [] for m in VLLM_MODELS}
+    for r in rows:
+        model = r.get("model")
+        if model in by_model:
+            by_model[model].append(r)
+
+    result = {}
+    for model, snaps in by_model.items():
+        snaps.sort(key=lambda r: r["ts"])
+        current = snaps[-1] if snaps else None
+        tracked = {c: 0.0 for c in VLLM_COUNTERS}
+        prev = None
+        for s in snaps:
+            if s["ts"] < cutoff:
+                prev = s
+                continue
+            if prev is not None:
+                for c in VLLM_COUNTERS:
+                    delta = s.get(c, 0) - prev.get(c, 0)
+                    if delta > 0:
+                        tracked[c] += delta
+            prev = s
+        result[model] = {
+            "reachable_now": current is not None and current["ts"] >= cutoff,
+            "snapshots_recorded": len(snaps),
+            "current": {
+                "prompt_tokens_total": current["prompt_tokens_total"],
+                "generation_tokens_total": current["generation_tokens_total"],
+                "request_success_total": current["request_success_total"],
+            } if current else None,
+            "tracked_since_monitoring_started": tracked,
+        }
+    return result
+
+
+def render_text(events, vllm, days):
+    lines = []
+    lines.append(f"=== the reference node usage report (last {days} days) ===\n")
+
+    for e in events:
+        lines.append(f"-- {e['service']} --")
+        if e["has_duration"]:
+            lines.append(f"  total: {e['total_count']} generations, "
+                         f"{e['total_duration_hours']}h ({e['total_duration_s']}s)")
+        else:
+            lines.append(f"  total: {e['total_count']} generations")
+        if not e["daily"]:
+            lines.append("  (no activity in this window)")
+        else:
+            for day, v in e["daily"].items():
+                if e["has_duration"]:
+                    lines.append(f"  {day}: {v['count']:>5} gens, {round(v['duration_s']/60, 1):>7} min")
+                else:
+                    lines.append(f"  {day}: {v['count']:>5} gens")
+        lines.append("")
+
+    lines.append("-- local LLM / vLLM token throughput --")
+    for model, v in vllm.items():
+        if not v["current"]:
+            lines.append(f"  {model}: not reachable (no snapshot ever recorded)")
+            continue
+        status = "up" if v["reachable_now"] else f"down (last seen in a prior snapshot)"
+        c = v["current"]
+        t = v["tracked_since_monitoring_started"]
+        lines.append(f"  {model}: {status}")
+        lines.append(f"    current counters (since last process restart): "
+                     f"{int(c['prompt_tokens_total']):,} prompt tok, "
+                     f"{int(c['generation_tokens_total']):,} gen tok, "
+                     f"{int(c['request_success_total']):,} requests")
+        lines.append(f"    tracked in window ({v['snapshots_recorded']} scrapes): "
+                     f"{int(t['prompt_tokens_total']):,} prompt tok, "
+                     f"{int(t['generation_tokens_total']):,} gen tok, "
+                     f"{int(t['request_success_total']):,} requests")
+    lines.append("")
+    return "\n".join(lines)
+
+
+def main():
+    parser = argparse.ArgumentParser()
+    parser.add_argument("--days", type=int, default=14)
+    parser.add_argument("--json", action="store_true")
+    args = parser.parse_args()
+
+    events = [summarize_events(name, filename, duration_field, args.days)
+              for name, (filename, duration_field) in EVENT_LOGS.items()]
+    vllm = summarize_vllm(args.days)
+
+    if args.json:
+        print(json.dumps({"events": events, "vllm": vllm, "days": args.days}, indent=2))
+    else:
+        print(render_text(events, vllm, args.days))
+
+
+if __name__ == "__main__":
+    main()