/** * tests/analytics/benchmark-harness.test.ts * ========================================== * BI-P104 — the harness's job is to stop a benchmark from lying. * * The tests are therefore attacks on the reporting, not on the timing: * * - two overlapping distributions whose MEDIANS differ 2x must come back * `indistinguishable`, because that is the claim a noisy host manufactures; * - the real 150x/130x pair from yesterday's budget-enforcement lane is fed * in as data, and the harness must describe a RANGE rather than pick one; * - a three-sample run must refuse to be compared at all; * - every sentence `describeComparison` produces must carry the range, the * sample size and the stability, so it cannot be quoted as a bare figure; * - an engine that was not run must appear in the evidence with a reason. */ import { describe, expect, it } from 'vitest'; import { BENCH_ENGINES, buildEvidence, compareDistributions, describeComparison, measure, MIN_SAMPLES, summarizeSamples, } from '@/src/analytics/bench/benchmark-harness'; describe('BI-P104 distribution summary', () => { it('reports the shape, and never a single number', () => { const d = summarizeSamples([10, 11, 12, 13, 14, 15, 16, 40]); expect(d.n).toBe(8); expect(d.min).toBe(10); expect(d.max).toBe(40); expect(d.p50).toBe(13); expect(d.spread_ratio).toBe(4); expect(d.cv).toBeGreaterThan(0); // Every field a reader needs to judge the p50 is present next to it. for (const key of [ 'min', 'p50', 'p95', 'p99', 'max', 'stdev', 'cv', 'iqr', 'spread_ratio', 'stability', ]) { expect(Object.keys(d)).toContain(key); } }); it('calls a tight run stable and a spread run unstable', () => { expect(summarizeSamples([100, 101, 99, 100, 102, 98, 100]).stability).toBe('stable'); expect(summarizeSamples([10, 200, 15, 400, 12, 350, 11]).stability).toBe('unstable'); }); it('refuses to characterise fewer than MIN_SAMPLES', () => { const d = summarizeSamples([10, 11, 12]); expect(d.n).toBe(3); expect(d.stability).toBe('insufficient_samples'); expect(MIN_SAMPLES).toBeGreaterThan(3); }); }); describe('BI-P104 comparison — variance beats a flattering median', () => { it('overlapping ranges are INDISTINGUISHABLE even when medians differ 2x', () => { const baseline = summarizeSamples([50, 60, 200, 70, 300, 55, 65, 280]); const candidate = summarizeSamples([25, 30, 190, 35, 260, 28, 33, 240]); const c = compareDistributions(baseline, candidate); expect(c.p50_ratio).toBeGreaterThan(1.5); // the tempting headline expect(c.ranges_overlap).toBe(true); expect(c.verdict).toBe('indistinguishable'); // the honest verdict expect(describeComparison(c)).toMatch(/INDISTINGUISHABLE/); }); it('clean separation is reported as faster, WITH the range', () => { const baseline = summarizeSamples([500, 520, 510, 530, 505, 515, 525]); const candidate = summarizeSamples([50, 55, 52, 58, 51, 53, 54]); const c = compareDistributions(baseline, candidate); expect(c.verdict).toBe('faster'); const sentence = describeComparison(c, 'pre-aggregate'); expect(sentence).toMatch(/FASTER/); expect(sentence).toMatch(/x to /); // a range, not a number expect(sentence).toMatch(/n=7\/7/); // sample sizes expect(sentence).toMatch(/stability/); }); it('the real 150x / 130x overshoot pair is described as a range, not a figure', () => { // Two runs of the same measurement on the same machine, from the BI-P021 // budget-enforcement lane. A harness that picks one of these is lying. const overshoot = summarizeSamples([153, 131, 149, 128, 144, 137, 152, 126, 141]); // Worth being precise about: by the CV test this run is STABLE (0.07). The // problem was never instability — it is that a stable run still spans // 126x-153x, so any single figure quoted from it is ~20% wrong. The // harness's answer is to keep the ends attached to the middle. expect(overshoot.stability).toBe('stable'); expect(overshoot.min).toBe(126); expect(overshoot.p50).toBe(141); expect(overshoot.max).toBe(153); expect(overshoot.spread_ratio).toBeCloseTo(1.21, 2); const sentence = describeComparison( compareDistributions(summarizeSamples([1, 1, 1, 1, 1, 1, 1, 1, 1]), overshoot), 'budget overshoot factor', ); expect(sentence).toMatch(/x to /); // a range survives into the prose expect(sentence).toMatch(/n=9\/9/); }); it('a three-sample run cannot be compared at all', () => { const c = compareDistributions(summarizeSamples([10, 11, 12]), summarizeSamples([1, 2, 3])); expect(c.verdict).toBe('not_comparable'); expect(describeComparison(c)).toMatch(/anecdote/); }); it('slower is reported as slower — the harness has no favourite', () => { const baseline = summarizeSamples([50, 51, 52, 53, 54, 55, 56]); const candidate = summarizeSamples([500, 510, 520, 530, 540, 550, 560]); expect(compareDistributions(baseline, candidate).verdict).toBe('slower'); }); }); describe('BI-P104 measurement', () => { it('records warmups but excludes them from the statistics', async () => { let call = 0; let clock = 0; const run = await measure({ engine: 'duckdb', dataset: 'small_saas', query_label: 'group by day', warmups: 2, repetitions: 6, // A cold first pair (1000ms) then a warm steady state (10ms). now: () => { const t = clock; clock += call <= 2 ? 1000 : 10; call += 1; return t; }, run: async () => ({ rows: 42 }), }); expect(run.warmup_ms).toHaveLength(2); expect(run.samples_ms).toHaveLength(6); expect(run.distribution.n).toBe(6); expect(run.distribution.max).toBeLessThan(1000); // the cold pair is not in it expect(run.rows_returned).toBe(42); }); }); describe('BI-P104 evidence', () => { it('lists every engine BI-P104 names, and says why the unrun ones were not run', () => { const evidence = buildEvidence({ host_note: 'one idle host, warm cache', engines: [ { engine: 'clickhouse', available: true, version: '26.3' }, { engine: 'duckdb', available: true, version: '1.5.5' }, { engine: 'trino', available: false, reason: 'the running Trino belongs to another lane' }, ], runs: [ { engine: 'clickhouse', dataset: 'small_saas', query_label: 'q', warmup_ms: [1], samples_ms: [1, 2, 3, 4, 5], distribution: summarizeSamples([1, 2, 3, 4, 5]), }, ], }); expect(evidence.engines.map((e) => e.engine)).toEqual([...BENCH_ENGINES]); const notRun = evidence.not_exercised.map((e) => e.engine); expect(notRun).toContain('superset'); expect(notRun).toContain('perspective'); expect(notRun).toContain('duckdb'); // declared available but produced no run for (const e of evidence.not_exercised) { expect(e.reason, e.engine).toBeTruthy(); } expect(evidence.datasets.map((d) => d.id)).toEqual([ 'small_saas', 'medium_business', 'vehiq_market', 'large_event', 'wide_table', 'geospatial', ]); }); });