0pt/tests/analytics/benchmark-harness.test.ts

/**
 * tests/analytics/benchmark-harness.test.ts
 * ==========================================
 * BI-P104 — the harness's job is to stop a benchmark from lying.
 *
 * The tests are therefore attacks on the reporting, not on the timing:
 *
 *   - two overlapping distributions whose MEDIANS differ 2x must come back
 *     `indistinguishable`, because that is the claim a noisy host manufactures;
 *   - the real 150x/130x pair from yesterday's budget-enforcement lane is fed
 *     in as data, and the harness must describe a RANGE rather than pick one;
 *   - a three-sample run must refuse to be compared at all;
 *   - every sentence `describeComparison` produces must carry the range, the
 *     sample size and the stability, so it cannot be quoted as a bare figure;
 *   - an engine that was not run must appear in the evidence with a reason.
 */
import { describe, expect, it } from 'vitest';
import {
  BENCH_ENGINES,
  buildEvidence,
  compareDistributions,
  describeComparison,
  measure,
  MIN_SAMPLES,
  summarizeSamples,
} from '@/src/analytics/bench/benchmark-harness';

describe('BI-P104 distribution summary', () => {
  it('reports the shape, and never a single number', () => {
    const d = summarizeSamples([10, 11, 12, 13, 14, 15, 16, 40]);
    expect(d.n).toBe(8);
    expect(d.min).toBe(10);
    expect(d.max).toBe(40);
    expect(d.p50).toBe(13);
    expect(d.spread_ratio).toBe(4);
    expect(d.cv).toBeGreaterThan(0);
    // Every field a reader needs to judge the p50 is present next to it.
    for (const key of [
      'min',
      'p50',
      'p95',
      'p99',
      'max',
      'stdev',
      'cv',
      'iqr',
      'spread_ratio',
      'stability',
    ]) {
      expect(Object.keys(d)).toContain(key);
    }
  });

  it('calls a tight run stable and a spread run unstable', () => {
    expect(summarizeSamples([100, 101, 99, 100, 102, 98, 100]).stability).toBe('stable');
    expect(summarizeSamples([10, 200, 15, 400, 12, 350, 11]).stability).toBe('unstable');
  });

  it('refuses to characterise fewer than MIN_SAMPLES', () => {
    const d = summarizeSamples([10, 11, 12]);
    expect(d.n).toBe(3);
    expect(d.stability).toBe('insufficient_samples');
    expect(MIN_SAMPLES).toBeGreaterThan(3);
  });
});

describe('BI-P104 comparison — variance beats a flattering median', () => {
  it('overlapping ranges are INDISTINGUISHABLE even when medians differ 2x', () => {
    const baseline = summarizeSamples([50, 60, 200, 70, 300, 55, 65, 280]);
    const candidate = summarizeSamples([25, 30, 190, 35, 260, 28, 33, 240]);
    const c = compareDistributions(baseline, candidate);

    expect(c.p50_ratio).toBeGreaterThan(1.5); // the tempting headline
    expect(c.ranges_overlap).toBe(true);
    expect(c.verdict).toBe('indistinguishable'); // the honest verdict
    expect(describeComparison(c)).toMatch(/INDISTINGUISHABLE/);
  });

  it('clean separation is reported as faster, WITH the range', () => {
    const baseline = summarizeSamples([500, 520, 510, 530, 505, 515, 525]);
    const candidate = summarizeSamples([50, 55, 52, 58, 51, 53, 54]);
    const c = compareDistributions(baseline, candidate);

    expect(c.verdict).toBe('faster');
    const sentence = describeComparison(c, 'pre-aggregate');
    expect(sentence).toMatch(/FASTER/);
    expect(sentence).toMatch(/x to /); // a range, not a number
    expect(sentence).toMatch(/n=7\/7/); // sample sizes
    expect(sentence).toMatch(/stability/);
  });

  it('the real 150x / 130x overshoot pair is described as a range, not a figure', () => {
    // Two runs of the same measurement on the same machine, from the BI-P021
    // budget-enforcement lane. A harness that picks one of these is lying.
    const overshoot = summarizeSamples([153, 131, 149, 128, 144, 137, 152, 126, 141]);

    // Worth being precise about: by the CV test this run is STABLE (0.07). The
    // problem was never instability — it is that a stable run still spans
    // 126x-153x, so any single figure quoted from it is ~20% wrong. The
    // harness's answer is to keep the ends attached to the middle.
    expect(overshoot.stability).toBe('stable');
    expect(overshoot.min).toBe(126);
    expect(overshoot.p50).toBe(141);
    expect(overshoot.max).toBe(153);
    expect(overshoot.spread_ratio).toBeCloseTo(1.21, 2);

    const sentence = describeComparison(
      compareDistributions(summarizeSamples([1, 1, 1, 1, 1, 1, 1, 1, 1]), overshoot),
      'budget overshoot factor',
    );
    expect(sentence).toMatch(/x to /); // a range survives into the prose
    expect(sentence).toMatch(/n=9\/9/);
  });

  it('a three-sample run cannot be compared at all', () => {
    const c = compareDistributions(summarizeSamples([10, 11, 12]), summarizeSamples([1, 2, 3]));
    expect(c.verdict).toBe('not_comparable');
    expect(describeComparison(c)).toMatch(/anecdote/);
  });

  it('slower is reported as slower — the harness has no favourite', () => {
    const baseline = summarizeSamples([50, 51, 52, 53, 54, 55, 56]);
    const candidate = summarizeSamples([500, 510, 520, 530, 540, 550, 560]);
    expect(compareDistributions(baseline, candidate).verdict).toBe('slower');
  });
});

describe('BI-P104 measurement', () => {
  it('records warmups but excludes them from the statistics', async () => {
    let call = 0;
    let clock = 0;
    const run = await measure({
      engine: 'duckdb',
      dataset: 'small_saas',
      query_label: 'group by day',
      warmups: 2,
      repetitions: 6,
      // A cold first pair (1000ms) then a warm steady state (10ms).
      now: () => {
        const t = clock;
        clock += call <= 2 ? 1000 : 10;
        call += 1;
        return t;
      },
      run: async () => ({ rows: 42 }),
    });

    expect(run.warmup_ms).toHaveLength(2);
    expect(run.samples_ms).toHaveLength(6);
    expect(run.distribution.n).toBe(6);
    expect(run.distribution.max).toBeLessThan(1000); // the cold pair is not in it
    expect(run.rows_returned).toBe(42);
  });
});

describe('BI-P104 evidence', () => {
  it('lists every engine BI-P104 names, and says why the unrun ones were not run', () => {
    const evidence = buildEvidence({
      host_note: 'one idle host, warm cache',
      engines: [
        { engine: 'clickhouse', available: true, version: '26.3' },
        { engine: 'duckdb', available: true, version: '1.5.5' },
        { engine: 'trino', available: false, reason: 'the running Trino belongs to another lane' },
      ],
      runs: [
        {
          engine: 'clickhouse',
          dataset: 'small_saas',
          query_label: 'q',
          warmup_ms: [1],
          samples_ms: [1, 2, 3, 4, 5],
          distribution: summarizeSamples([1, 2, 3, 4, 5]),
        },
      ],
    });

    expect(evidence.engines.map((e) => e.engine)).toEqual([...BENCH_ENGINES]);
    const notRun = evidence.not_exercised.map((e) => e.engine);
    expect(notRun).toContain('superset');
    expect(notRun).toContain('perspective');
    expect(notRun).toContain('duckdb'); // declared available but produced no run
    for (const e of evidence.not_exercised) {
      expect(e.reason, e.engine).toBeTruthy();
    }
    expect(evidence.datasets.map((d) => d.id)).toEqual([
      'small_saas',
      'medium_business',
      'vehiq_market',
      'large_event',
      'wide_table',
      'geospatial',
    ]);
  });
});