From d34683bc681479c3ca16cc68f90cb0396e0a90f7 Mon Sep 17 00:00:00 2001 From: Oliver Rieger Date: Tue, 11 Aug 2026 20:29:34 +0200 Subject: [PATCH] A flaky gate is worse than no gate: seed the suite MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The pipeline caught two flaky tests of mine that sixty local runs had not. Both were bad tests rather than bad code, and both were mine from this session: - the eater test counted SURVIVORS, not arrivals. An eater crosses the whole court in ~24s and despawns, so a snapshot at 40s can legitimately show an empty court — that read as "the eaters never came". It tracks arrivals over the window now, and asserts the cap as a running peak. - the auto-aim test never actually aimed at the body it claimed to: fy was pinned at -0.95 and fx was a scaled x offset, so the ray leaned up-court and the called body could genuinely be the best-aligned thing in the cone. That is the magnet working. It now normalises a vector straight at the wrong body, picks the one furthest in x from the called one so the two are separable, and asserts the magnet takes what you pointed at. Measured 8 failures in 40 runs before, 0 in 60 after. And the durable fix: Math.random is seeded at the top of test.js. This suite is a Dockerfile build stage, so a red run blocks the deploy, which makes a 1-in-5 flake worse than no gate — it trains you to re-run it. The game leans on randomness everywhere, so the stream is made reproducible rather than the randomness stripped out. SEED=1234 node test.js re-rolls it; 25 seeds pass. The cost is noted in PROJECT.md: under one seed a guarded test can silently take its early-out forever, so I checked all four guards still run their bodies. Co-Authored-By: Claude Opus 5 --- PROJECT.md | 10 +++++++++ test.js | 63 +++++++++++++++++++++++++++++++++++++++++++++--------- 2 files changed, 63 insertions(+), 10 deletions(-) diff --git a/PROJECT.md b/PROJECT.md index d6b6c28..e863483 100644 --- a/PROJECT.md +++ b/PROJECT.md @@ -6,6 +6,16 @@ everything they specified is shipped, and what still matters from them is record long history, `~/.claude/projects/.../memory/splatter-tennis-history.md` is auto-loaded; for the detail of any individual change, read the git log. +**THE SUITE IS DETERMINISTIC ON PURPOSE (`Math.random` is seeded at the top of `test.js`).** It is a +Dockerfile build stage, so a red run blocks the deploy — which makes a test that fails one run in +five *worse than no gate*, because it trains you to re-run it. Two drill tests did exactly that and +the pipeline caught what sixty local runs had not: one asserted on a random batch composition, the +other on a SNAPSHOT of a population that legitimately empties (an eater crosses the whole court in +~24 s and despawns, so "no eaters on court at 40 s" is normal, not a failure). **Fuzz with +`SEED=1234 node test.js` whenever you touch anything probabilistic** — and beware the cost of one +fixed seed: a guarded test can silently take its early-out every run and quietly stop testing +anything, so if you add a `return` for an awkward deal, check the body still runs. + **`node test.js` must pass before any commit** — it drives the real game modules headless and pins the state-machine contracts that have historically produced the show-stoppers. The Dockerfile runs it as a build stage, so a red suite cannot deploy — but finding out from Coolify is the slow way. diff --git a/test.js b/test.js index 50190b6..c64d284 100644 --- a/test.js +++ b/test.js @@ -7,6 +7,27 @@ import assert from 'node:assert/strict'; +// ---- DETERMINISM. This suite is a Dockerfile build stage: a red run blocks the deploy, so a test +// that fails one run in five is worse than no gate at all — it trains you to re-run it. Two of the +// drill tests did exactly that (they asserted on a random batch composition and on a snapshot of a +// population that legitimately empties), and the pipeline caught what 60 local runs had not. +// +// The game leans on Math.random() everywhere — jitter, variants, lanes, spawn gaps — so the fix is +// to make the STREAM reproducible rather than to strip randomness out of the game. Fixed seed by +// default, and `SEED=1234 node test.js` re-rolls it: run the suite over a spread of seeds when you +// touch anything probabilistic, and any failure you see is one you can reproduce exactly. +// +// The cost to know about: under one seed a guarded test can silently take its early-out every time +// and quietly stop testing anything. If you add a `return` for an awkward deal, fuzz a few seeds and +// make sure the body actually runs. +let _seed = (Number(process.env.SEED) || 20260811) >>> 0; +Math.random = () => { // xorshift32 — small, fast, good enough for jitter + _seed ^= _seed << 13; _seed >>>= 0; + _seed ^= _seed >> 17; + _seed ^= _seed << 5; _seed >>>= 0; + return _seed / 4294967296; +}; + // ---- platform stubs, installed before the game modules load ---------------- const absorb = new Proxy(function () {}, { get: (t, p) => (p === Symbol.toPrimitive ? () => 0 : absorb), @@ -341,12 +362,25 @@ test('the eaters are the danger: plain shamblers, homing, and killing one is nev const eaten = TUNE.eatenR; TUNE.eatenR = 0; // the soak trick: don't get caught mid-measurement try { startDrill(); - for (let i = 0; i < 40 * 60; i++) update(DT); - const eaters = zombies.filter(z => !z.drill); - assert.ok(eaters.length > 0, 'plain shamblers arrive on their own clock'); - assert.ok(eaters.length <= TUNE.drillEaterMax, 'and are capped at ' + TUNE.drillEaterMax); - assert.ok(eaters.every(z => 'ab'.includes(z.variant)), - 'an eater is ALWAYS a plain shambler — that is what keeps a called type unambiguous'); + // COUNT ARRIVALS, NOT SURVIVORS. An eater crosses the whole court in about 24 s and then + // despawns, so a snapshot taken at 40 s can legitimately show an empty court — that read as + // "the eaters never came" and made this test fail about one run in five. + const seen = new Set(); + let peak = 0; + for (let i = 0; i < 40 * 60; i++) { + update(DT); + const live = zombies.filter(z => !z.drill); + peak = Math.max(peak, live.length); + for (const z of live) { + seen.add(z); + assert.ok('ab'.includes(z.variant), + 'an eater is ALWAYS a plain shambler — that is what keeps a called type unambiguous'); + } + } + assert.ok(seen.size > 0, 'plain shamblers arrive on their own clock'); + assert.ok(peak <= TUNE.drillEaterMax, 'and never more than ' + TUNE.drillEaterMax + ' at once, saw ' + peak); + const eaters = [...seen].filter(z => zombies.includes(z)); + if (!eaters.length) return; // all of them have walked off again: nothing left to shoot at // killing one must not touch the combination const seq = G.drill.seq.slice(), faults = G.drill.faults; const e = eaters[0]; @@ -421,10 +455,19 @@ test('THE AUTO-AIM IS GONE: the shot goes where you point it', () => { // and the fault unreachable — Oliver: "no skills involved at all... there is no penalty". drillPlaying(); const called = G.drill.seq[G.drill.step]; - const wrong = zombies.find(z => z.drill && z.variant !== called); - assert.ok(wrong, 'setup: something else is on court'); - const ft = flickTarget(P, { fx: Math.max(-1, Math.min(1, (wrong.x - P.x) / 9)), fy: -0.95, mag: 140 }); - assert.notEqual(ft.zt && ft.zt.variant, called, 'pointing away from the called body must not hit it'); + const target = zombies.find(z => z.drill && z.variant === called); + assert.ok(target, 'setup: the called body is on court'); + // the wrong body FURTHEST from the called one in x, so the two are genuinely separable — aiming + // at a body that happens to sit in the same lane proves nothing either way + const wrong = zombies.filter(z => z.drill && z.variant !== called) + .sort((a, b) => Math.abs(b.x - target.x) - Math.abs(a.x - target.x))[0]; + if (!wrong || Math.abs(wrong.x - target.x) < 1.5) return; // nothing separable on court this deal + // ...and point the ray AT it, properly normalised. The old version pinned fy at -0.95 and scaled + // fx, so it never actually aimed at the body it claimed to. + const dx = wrong.x - P.x, dz = wrong.z - P.z, L = Math.hypot(dx, dz); + const ft = flickTarget(P, { fx: dx / L, fy: dz / L, mag: 140 }); + assert.equal(ft.zt && ft.zt.variant, wrong.variant, 'the magnet takes the body you pointed at'); + assert.notEqual(ft.zt && ft.zt.variant, called, 'and never the called one just because it is called'); }); test('a wrong kill never leaves an un-winnable round', () => {