diff --git a/server/src/satellites/celestrak.ts b/server/src/satellites/celestrak.ts index 3510c86..f44f1f9 100644 --- a/server/src/satellites/celestrak.ts +++ b/server/src/satellites/celestrak.ts @@ -75,15 +75,19 @@ export interface SatelliteSnapshot { * fields and the cubesat catalogue would add eleven thousand objects nobody can * see and nobody asked about, at the cost of a slower boot and a fatter cache. * - * Order matters only for de-duplication: a NORAD id seen twice keeps the **first** - * set, so the supplemental Starlink feed must come before anything that might - * also carry a Starlink. See `merge`. + * Order does not decide capacity — `merge` does, and it reserves the ceiling for + * the small groups before the `fill` one takes the remainder. Without that, the + * supplemental Starlink feed is large enough to consume the entire cap on its + * own and every other group here is fetched and then thrown away. */ -const GROUPS: { url: string; group: SatelliteGroup }[] = [ - // Supplemental first, and first for a reason — see the note above and `merge`. +const GROUPS: { url: string; group: SatelliteGroup; fill?: true }[] = [ + // `fill` — see `merge`. Starlink is both the largest group by an order of + // magnitude and the one everybody came for, so it takes whatever capacity the + // others leave rather than competing with them for it. { url: "https://celestrak.org/NORAD/elements/supplemental/sup-gp.php?FILE=starlink&FORMAT=tle", group: "starlink", + fill: true, }, { url: group("stations"), group: "station" }, { url: group("gps-ops"), group: "navigation" }, @@ -110,7 +114,8 @@ function group(name: string): string { * under what a laptop can propagate at 4 Hz, and going over it truncates rather * than failing, because a partial sky beats no sky. * - * The truncation is not silent: `createSatellitesService` logs the drop. + * This ceiling is reached on every real fetch, because Starlink alone exceeds it. + * `merge` says so in the log rather than trimming quietly. */ export const MAX_SATELLITES = 6000; @@ -167,7 +172,7 @@ export async function fetchCatalogue( }), ); - const satellites = merge(fetched); + const satellites = merge(fetched, GROUPS.findIndex((spec) => spec.fill === true), log); if (satellites.length === 0) return null; return { fetchedAt: Date.now(), satellites }; @@ -240,25 +245,88 @@ export function parseTle(text: string, group: SatelliteGroup): WireSatellite[] { } /** - * One list, first set per NORAD id wins, capped at `MAX_SATELLITES`. + * One list, one entry per NORAD id, capped at `MAX_SATELLITES`. * - * First-wins is what makes the group ordering in `GROUPS` load-bearing: a - * Starlink appearing in both the supplemental feed and, say, `last-30-days` - * should keep the operator's ephemeris and the `starlink` display bucket, and it - * does so purely because the supplemental feed is fetched first. Last-wins would - * silently re-bucket half the constellation as `other` and nobody would notice - * until the colours looked wrong. + * Two rules, and they pull in opposite directions, which is why this is not a + * loop with a counter in it. + * + * **Precedence goes to the `fill` group.** A Starlink appearing both in the + * supplemental feed and in some future catch-all group should keep the + * operator's ephemeris and the `starlink` display bucket. The alternative + * silently re-buckets half the constellation as `other` and nobody notices until + * the colours look wrong. + * + * **Capacity goes to everybody else first.** This is the half that was missing, + * and the bug it caused was total rather than subtle: the supplemental Starlink + * feed is past 6,000 objects on its own, so a straight first-wins loop spent the + * entire ceiling on it and served a sky containing *nothing else* — no ISS, no + * GPS, no weather satellites, none of the naked-eye catalogue. Every other group + * in `GROUPS` was fetched, parsed, and thrown away. The truncation warning fired + * exactly as designed, which is the only reason it was caught. + * + * So the small groups are counted first and the fill group takes the remainder. + * Together those two rules are worth about a thousand objects of everything else + * plus five thousand Starlinks, which is the sky that was wanted. */ -function merge(groups: WireSatellite[][]): WireSatellite[] { +function merge( + groups: WireSatellite[][], + fillIndex: number, + log: CelestrakLog, +): WireSatellite[] { + const fill = groups[fillIndex] ?? []; + const fillIds = new Set(fill.map((sat) => sat.noradId)); + const seen = new Set(); const out: WireSatellite[] = []; - for (const list of groups) { + let othersDropped = 0; + groups.forEach((list, i) => { + if (i === fillIndex) return; for (const sat of list) { - if (seen.has(sat.noradId)) continue; + // Dropped here rather than later: the fill group's copy of this object is + // the better one, and it is guaranteed a place below. + if (fillIds.has(sat.noradId) || seen.has(sat.noradId)) continue; seen.add(sat.noradId); - out.push(sat); - if (out.length >= MAX_SATELLITES) return out; + if (out.length < MAX_SATELLITES) out.push(sat); + else othersDropped += 1; } + }); + + const reserved = out.length; + for (const sat of fill) { + if (out.length >= MAX_SATELLITES) break; + if (seen.has(sat.noradId)) continue; + seen.add(sat.noradId); + out.push(sat); } + + /** + * What was left out, said out loud, once per fetch — so about four times a + * day. A cap nobody reports reads as "we served everything" when it did not. + * + * The fill group being trimmed is the **normal** state and not a fault: there + * are more Starlinks than this build will draw and there is no configuration + * in which that stops being true. It gets a line because the number is worth + * knowing, not because anything is wrong. + * + * The other branch is a fault. If the small groups alone reach the ceiling, + * somebody has added a large group to `GROUPS` without raising + * `MAX_SATELLITES`, and the symptom is a sky with no Starlinks in it at all. + */ + const fillServed = out.length - reserved; + if (othersDropped > 0) { + log.warn( + `satellites:celestrak: the groups other than the fill group filled the ` + + `${MAX_SATELLITES}-object ceiling on their own and ${othersDropped} were still ` + + `dropped — the fill group got no room at all. Raise MAX_SATELLITES or shorten GROUPS.`, + ); + } else if (fillServed < fill.length) { + log.warn( + `satellites:celestrak: serving ${fillServed} of ${fill.length} objects from the ` + + `fill group, ${reserved} from the rest; the ${MAX_SATELLITES}-object ceiling is ` + + `the limit. This is the expected state — there are more Starlinks than a browser ` + + `will propagate.`, + ); + } + return out; } diff --git a/server/src/satellites/index.ts b/server/src/satellites/index.ts index 73f3977..f844347 100644 --- a/server/src/satellites/index.ts +++ b/server/src/satellites/index.ts @@ -32,7 +32,7 @@ * trap `TERA_FLIGHTS_TTL=0` used to be. */ -import { fetchCatalogue, MAX_SATELLITES, type SatelliteSnapshot } from "./celestrak.ts"; +import { fetchCatalogue, type SatelliteSnapshot } from "./celestrak.ts"; import { createUpstream } from "../upstream.ts"; import type { Config } from "../config.ts"; import type { SatellitesBody } from "../../../src/server/wire.ts"; @@ -81,11 +81,6 @@ export function createSatellitesService(config: Config, log: SatellitesLog): Sat log, }); - // Once, not per response: a truncated catalogue is a property of the - // deployment and the feed, and logging it on every request would turn one - // useful sentence into a wall of the same sentence. - let warnedTruncated = false; - return { async current(): Promise { if (source === "none") return emptyBody(ttl); @@ -93,15 +88,6 @@ export function createSatellitesService(config: Config, log: SatellitesLog): Sat const snapshot = await upstream.get(CATALOGUE_KEY, () => fetchCatalogue(contact, log)); if (snapshot === null) return emptyBody(ttl); - if (snapshot.satellites.length >= MAX_SATELLITES && !warnedTruncated) { - warnedTruncated = true; - log.warn( - `satellites:celestrak: the merged catalogue hit the ${MAX_SATELLITES}-object ` + - `ceiling and was truncated; later groups in celestrak.ts are missing from ` + - `the served sky.`, - ); - } - return { source, fetchedAt: new Date(snapshot.fetchedAt).toISOString(), diff --git a/server/src/test/satellites.test.ts b/server/src/test/satellites.test.ts index 87d7a57..78c2264 100644 --- a/server/src/test/satellites.test.ts +++ b/server/src/test/satellites.test.ts @@ -10,18 +10,17 @@ * like a clear night — so a group that yields no element sets is treated as a * group that did not answer, and that is worth a test of its own. * - * The last group of tests is about the shape of the answer, and specifically - * about `merge` keeping the *first* element set per NORAD id. That ordering is - * what puts the operator's supplemental Starlink ephemeris ahead of a radar one - * and keeps the display bucket right; reverse it and half the constellation - * quietly re-colours itself. + * The other half is about `merge`, and about the one thing in this module that + * only real data ever revealed: the supplemental Starlink feed is larger than + * the whole object ceiling, so a merge that spends capacity in list order serves + * a catalogue containing nothing but Starlink. See the suite that says so. */ import assert from "node:assert/strict"; import { after, beforeEach, describe, it } from "node:test"; import { buildApp } from "../app.ts"; import { loadConfig } from "../config.ts"; -import { parseTle } from "../satellites/celestrak.ts"; +import { MAX_SATELLITES, parseTle } from "../satellites/celestrak.ts"; import type { SatellitesBody } from "../../../src/server/wire.ts"; const realFetch = globalThis.fetch; @@ -31,6 +30,8 @@ let agents: string[] = []; let feedIsUp = true; /** Served instead of a real TLE body, for the soft-failure tests. */ let feedOverride: string | undefined = undefined; +/** Served for the Starlink feed alone, for the ceiling test. */ +let starlinkOverride: string | undefined = undefined; /** * A plausible three-line element set. The numbers are ISS's published TLE @@ -52,7 +53,7 @@ const STARLINK = [ function feed(url: string): string | undefined { if (!feedIsUp) return undefined; if (feedOverride !== undefined) return feedOverride; - if (url.includes("FILE=starlink")) return STARLINK; + if (url.includes("FILE=starlink")) return starlinkOverride ?? STARLINK; if (url.includes("GROUP=stations")) return ISS; // Every other configured group answers with nothing usable, which is a // partial-merge case and must still produce a sky. @@ -77,6 +78,7 @@ beforeEach(() => { agents = []; feedIsUp = true; feedOverride = undefined; + starlinkOverride = undefined; }); function appWith(env: Record) { @@ -240,6 +242,65 @@ describe("a rate-limited fetch", () => { }); }); +/** + * The bug that only real data found. + * + * The supplemental Starlink feed is past six thousand objects on its own, which + * is the whole `MAX_SATELLITES` ceiling. A straight first-wins merge spent all of + * it there and served a catalogue that was **100% Starlink** — no ISS, no GPS, + * no weather satellites — having fetched and parsed every one of those groups + * first. Nothing failed; the sky just quietly had one constellation in it. + * + * The fixture below is the shape of that in miniature: a fill group far larger + * than the ceiling, and one small group that must survive it. + */ +describe("the ceiling reserves room for the small groups", () => { + /** + * A fill group larger than the whole ceiling, which is what the real + * supplemental Starlink feed is. Built rather than pasted: the point is the + * count, and 6,100 hand-written element sets is not a fixture anybody reads. + */ + function oversizedStarlinkFeed(): string { + const lines: string[] = []; + for (let i = 0; i < MAX_SATELLITES + 100; i += 1) { + // NORAD numbers are five columns, so this stays inside the real format. + const norad = String(10000 + i).padStart(5, "0"); + lines.push( + `STARLINK-${norad}`, + `1 ${norad}U 19074A 26037.54166667 .00002182 00000-0 16538-3 0 9995`, + `2 ${norad} 53.0538 156.7285 0001367 86.7419 273.3728 15.06391223 12345`, + ); + } + return lines.join("\n"); + } + + it("does not let the fill group crowd out the stations", async () => { + starlinkOverride = oversizedStarlinkFeed(); + const app = appWith(celestrakEnv); + after(() => app.close()); + + const body = await get(app); + assert.equal(body.satellites.length, MAX_SATELLITES, "the ceiling should be full"); + + const groups = new Set(body.satellites.map((s) => s.group)); + assert.ok(groups.has("station"), "the ISS was crowded out of the catalogue by the fill group"); + assert.ok(groups.has("starlink"), "the fill group got no room at all"); + }); + + it("spends what is left on the fill group rather than capping it early", async () => { + starlinkOverride = oversizedStarlinkFeed(); + const app = appWith(celestrakEnv); + after(() => app.close()); + + const body = await get(app); + const starlink = body.satellites.filter((s) => s.group === "starlink").length; + const others = body.satellites.length - starlink; + // One ISS in the fixture, so the fill group should take everything but it. + assert.equal(others, 1); + assert.equal(starlink, MAX_SATELLITES - others); + }); +}); + describe("parsing element sets", () => { it("reads the three-line form CelesTrak serves", () => { const parsed = parseTle(ISS, "station"); diff --git a/src/engine/satellites.ts b/src/engine/satellites.ts index 66a1981..7e8aaf4 100644 --- a/src/engine/satellites.ts +++ b/src/engine/satellites.ts @@ -311,23 +311,37 @@ export interface SatelliteLayer { /** * The dome's radius, as a fraction of the board's longest side. * - * It has to clear the city — a dot inside the buildings would be occluded by - * them, which is the one thing that is definitely wrong — and it has to stay - * inside the camera's far plane, which `scene.ts` sets at three board spans. At - * 1.2 the dome is outside every building and comfortably clear of the far plane - * even with the camera pulled all the way back. + * Three constraints, and the middle one is the one that is easy to miss: + * + * 1. Outside the city, or the buildings occlude the sky. + * 2. **Outside the camera's maximum orbit**, which `scene.ts` sets at + * `boardSpan * 1.5`. This was 1.2 first, which put the far end of the zoom + * *outside the dome* — pull back far enough and you were looking at the sky + * from space, with half the constellation behind the camera. A sky you can + * leave is not a sky. + * 3. Inside the far plane, which `scene.ts` sets at `boardSpan * 3`. + * + * 2.0 sits in the middle of the only window that satisfies all three. */ -const DOME_RADIUS_FACTOR = 1.2; +const DOME_RADIUS_FACTOR = 2.0; /** - * How large a dot is drawn, in scene units, before attenuation. + * How large a dot is drawn, in **pixels**, at any camera distance. * - * Everything on the dome is the same distance away, so `sizeAttenuation` cannot - * separate near from far here the way it does for a starfield — it only makes - * the whole layer shrink as the camera retreats, which is what keeps the sky - * looking like a sky rather than like a fixed-size overlay pasted on the frame. + * `sizeAttenuation` is off, and that is the physically honest choice rather than + * a convenience. Everything on this dome is the same distance away and the dome + * is a stand-in for something 550 km up: a satellite does not get bigger because + * you zoomed the map in, and a point of light at effectively infinite distance + * has an apparent size set by the eye rather than by the range. Attenuation was + * on first and did the wrong thing twice over — it shrank the whole sky as the + * camera pulled back, and at the far end of the zoom it took a 324-object + * constellation down to specks a pixel across that read as noise in the sky + * texture. + * + * Three and a half pixels is about what an actual naked-eye Starlink looks like + * against a dark sky, which is the reference this is aiming at. */ -const DOT_SIZE_FACTOR = 0.012; +const DOT_PIXELS = 3.5; /** Ceiling on dots, so the buffers are allocated once and never grow. */ const MAX_DOTS = 4096; @@ -392,8 +406,8 @@ export function createSatelliteLayer(boardSpan: number): SatelliteLayer { geo.setDrawRange(0, 0); const material = new THREE.PointsMaterial({ - size: boardSpan * DOT_SIZE_FACTOR, - sizeAttenuation: true, + size: DOT_PIXELS, + sizeAttenuation: false, vertexColors: true, transparent: true, // Dots are drawn over the sky and over each other; letting them write depth