/** * tests/integration/bi-p104-benchmark.integration.test.ts * ======================================================== * BI-P104 — the benchmark itself, against real engines. Skips honestly when * CLICKHOUSE_URL is unset; DuckDB additionally needs DUCKDB_PYTHON (an * interpreter with the `duckdb` package), matching * tests/duckdb/duckdb-provider.live.test.ts. * * Six dataset profiles, one representative question each, the same logical * question on both engines. Deliberately NO engine-specific functions in the * geospatial case (bounding box + grid arithmetic rather than each engine's * GIS library) so the comparison is about the storage and execution engine * rather than about who ships the nicer geo function. * * Statistics come from src/analytics/bench/benchmark-harness.ts — the shipped * code, not a copy — so the evidence file is produced by the thing under test. * * Trino, Superset, native charts and Perspective are NOT exercised, and each * appears in the evidence with its reason. An engine omitted from a benchmark * table reads as a choice, so nothing is omitted. * * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ * DUCKDB_PYTHON=/path/to/venv/bin/python \ * pnpm vitest run tests/integration/bi-p104-benchmark.integration.test.ts * * It CREATEs and DROPs its own ClickHouse database and uses an in-memory * DuckDB. Never point it at the b0ts ClickHouse or any quarantined data. */ import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { afterAll, beforeAll, describe, expect, it } from 'vitest'; import { buildEvidence, compareDistributions, describeComparison, summarizeSamples, type BenchDatasetId, type BenchmarkRun, type EngineAvailability, } from '@/src/analytics/bench/benchmark-harness'; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const DUCKDB_PYTHON = process.env.DUCKDB_PYTHON; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_p104_${Date.now().toString(36)}`; const WARMUPS = 2; const REPETITIONS = 9; const WIDE_COLUMNS = 50; const headers: Record = CLICKHOUSE_AUTH ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } : {}; async function ch(query: string): Promise { const res = await fetch(CLICKHOUSE_URL ?? 'http://skipped.invalid', { method: 'POST', headers, body: query, }); const text = await res.text(); if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`); return text; } /** The six questions, one per profile, phrased for each engine. */ const QUERIES: Record = { small_saas: { label: 'revenue by day', clickhouse: `SELECT d, sum(amount) AS revenue, count() AS n FROM ${DB}.small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`, duckdb: `SELECT d, sum(amount) AS revenue, count(*) AS n FROM small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`, }, medium_business: { label: 'revenue by month and region for one tenant', clickhouse: `SELECT toStartOfMonth(d) AS m, region, sum(amount) AS revenue FROM ${DB}.medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`, duckdb: `SELECT date_trunc('month', d) AS m, region, sum(amount) AS revenue FROM medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`, }, vehiq_market: { label: 'average asking price by make and model year', clickhouse: `SELECT make, model_year, avg(price) AS avg_price, count() AS listings FROM ${DB}.vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`, duckdb: `SELECT make, model_year, avg(price) AS avg_price, count(*) AS listings FROM vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`, }, large_event: { label: 'hourly event volume for one event type', clickhouse: `SELECT toStartOfHour(ts) AS h, count() AS n, uniqExact(user_id) AS users FROM ${DB}.large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`, duckdb: `SELECT date_trunc('hour', ts) AS h, count(*) AS n, count(DISTINCT user_id) AS users FROM large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`, }, wide_table: { label: 'three columns of fifty, grouped', clickhouse: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM ${DB}.wide_table GROUP BY grp ORDER BY grp`, duckdb: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM wide_table GROUP BY grp ORDER BY grp`, }, geospatial: { label: 'bounding box, aggregated to a grid cell', clickhouse: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count() AS n FROM ${DB}.geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`, duckdb: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count(*) AS n FROM geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`, }, }; const CH_DDL: Array<[BenchDatasetId, string]> = [ [ 'small_saas', `CREATE TABLE ${DB}.small_saas ENGINE = MergeTree ORDER BY (tenant_id, d) AS SELECT concat('t_', toString(number % 3)) AS tenant_id, toDate('2026-01-01') + toIntervalDay(number % 365) AS d, ['free','pro','enterprise'][(number % 3) + 1] AS plan, toDecimal64((number % 499) + 9, 2) AS amount FROM numbers(100000)`, ], [ 'medium_business', `CREATE TABLE ${DB}.medium_business ENGINE = MergeTree ORDER BY (tenant_id, d) AS SELECT concat('t_', toString(number % 8)) AS tenant_id, toDate('2025-01-01') + toIntervalDay(number % 730) AS d, ['SE','NO','DK','FI','DE'][(number % 5) + 1] AS region, toDecimal64((number % 977) + 13, 2) AS amount FROM numbers(2000000)`, ], [ 'vehiq_market', `CREATE TABLE ${DB}.vehiq_market ENGINE = MergeTree ORDER BY (make, model_year) AS SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(number % 6) + 1] AS make, toUInt16(2005 + (number % 20)) AS model_year, toUInt32(30000 + (number % 900000)) AS price, toUInt32((number * 7) % 400000) AS mileage, ['diesel','petrol','hybrid','electric'][(number % 4) + 1] AS fuel FROM numbers(1000000)`, ], [ 'large_event', `CREATE TABLE ${DB}.large_event ENGINE = MergeTree ORDER BY (event_type, ts) AS SELECT toDateTime('2026-08-01 00:00:00') + toIntervalSecond(number % 2592000) AS ts, toUInt32(number % 250000) AS user_id, ['view','click','checkout','signup'][(number % 4) + 1] AS event_type, toFloat64(number % 1000) AS value FROM numbers(5000000)`, ], [ 'wide_table', `CREATE TABLE ${DB}.wide_table ENGINE = MergeTree ORDER BY grp AS SELECT toUInt16(number % 500) AS grp, ${Array.from({ length: WIDE_COLUMNS }, (_, i) => `toFloat64((number + ${i}) % 1000) AS c${i}`).join(',\n ')} FROM numbers(200000)`, ], [ 'geospatial', `CREATE TABLE ${DB}.geospatial ENGINE = MergeTree ORDER BY (lat, lon) AS SELECT 55.0 + (number % 10000) / 1000.0 AS lat, 10.0 + ((number * 13) % 10000) / 1000.0 AS lon, toUInt32(number) AS point_id FROM numbers(500000)`, ], ]; /** Runs all six DuckDB measurements inside ONE python process. */ const DUCKDB_PROGRAM = ` import duckdb, json, time, sys con = duckdb.connect() # The progress bar writes to STDOUT and corrupts the JSON transcript. It cost # one run to find, so it is turned off explicitly rather than filtered out. con.execute("SET enable_progress_bar=false") WIDE = ${WIDE_COLUMNS} wide_cols = ", ".join(f"(i + {j}) % 1000.0 AS c{j}" for j in range(WIDE)) ddl = { "small_saas": "CREATE TABLE small_saas AS SELECT 't_' || (i % 3) AS tenant_id, DATE '2026-01-01' + INTERVAL (i % 365) DAY AS d, ['free','pro','enterprise'][(i % 3) + 1] AS plan, ((i % 499) + 9)::DECIMAL(18,2) AS amount FROM range(100000) t(i)", "medium_business": "CREATE TABLE medium_business AS SELECT 't_' || (i % 8) AS tenant_id, DATE '2025-01-01' + INTERVAL (i % 730) DAY AS d, ['SE','NO','DK','FI','DE'][(i % 5) + 1] AS region, ((i % 977) + 13)::DECIMAL(18,2) AS amount FROM range(2000000) t(i)", "vehiq_market": "CREATE TABLE vehiq_market AS SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(i % 6) + 1] AS make, (2005 + (i % 20))::USMALLINT AS model_year, (30000 + (i % 900000))::UINTEGER AS price, ((i * 7) % 400000)::UINTEGER AS mileage, ['diesel','petrol','hybrid','electric'][(i % 4) + 1] AS fuel FROM range(1000000) t(i)", "large_event": "CREATE TABLE large_event AS SELECT TIMESTAMP '2026-08-01 00:00:00' + INTERVAL (i % 2592000) SECOND AS ts, (i % 250000)::UINTEGER AS user_id, ['view','click','checkout','signup'][(i % 4) + 1] AS event_type, (i % 1000)::DOUBLE AS value FROM range(5000000) t(i)", "wide_table": f"CREATE TABLE wide_table AS SELECT (i % 500)::USMALLINT AS grp, {wide_cols} FROM range(200000) t(i)", "geospatial": "CREATE TABLE geospatial AS SELECT 55.0 + (i % 10000) / 1000.0 AS lat, 10.0 + ((i * 13) % 10000) / 1000.0 AS lon, i::UINTEGER AS point_id FROM range(500000) t(i)", } queries = json.loads(sys.stdin.read()) out = {"version": duckdb.__version__, "runs": {}} for name, sql in ddl.items(): con.execute(sql) for name, q in queries.items(): warm = [] for _ in range(${WARMUPS}): t0 = time.perf_counter(); rows = con.execute(q).fetchall(); warm.append((time.perf_counter() - t0) * 1000) samples = [] for _ in range(${REPETITIONS}): t0 = time.perf_counter(); rows = con.execute(q).fetchall(); samples.append((time.perf_counter() - t0) * 1000) out["runs"][name] = {"warmup_ms": warm, "samples_ms": samples, "rows": len(rows)} sys.stdout.write("<<>>" + json.dumps(out)) `; const runs: BenchmarkRun[] = []; const engines: EngineAvailability[] = []; let duckdbResult: | { version: string; runs: Record } | undefined; run('BI-P104 benchmark harness against real engines', () => { beforeAll(async () => { const version = (await ch('SELECT version()')).trim(); engines.push({ engine: 'clickhouse', available: true, version }); await ch(`CREATE DATABASE IF NOT EXISTS ${DB}`); for (const [, ddl] of CH_DDL) await ch(ddl); }, 900_000); afterAll(async () => { if (!CLICKHOUSE_URL) return; try { await ch(`DROP DATABASE IF EXISTS ${DB}`); } finally { const comparisons = duckdbResult ? (Object.keys(QUERIES) as BenchDatasetId[]).flatMap((dataset) => { const chRun = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset); const duckRun = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset); if (!chRun || !duckRun) return []; const c = compareDistributions(chRun.distribution, duckRun.distribution); return [ { dataset, query_label: QUERIES[dataset].label, description: describeComparison(c, `duckdb vs clickhouse (${dataset})`), ...c, }, ]; }) : []; const evidence = buildEvidence({ host_note: 'One idle host, warm caches, both engines local, single-threaded client. These numbers ' + 'describe these query shapes on this data on this machine. They are not engine rankings.', engines, runs, comparisons, }); const dir = path.join(process.cwd(), 'evidence/bi-p104'); fs.mkdirSync(dir, { recursive: true }); fs.writeFileSync(path.join(dir, 'benchmark.json'), `${JSON.stringify(evidence, null, 2)}\n`); } }, 120_000); it('measures all six dataset profiles on ClickHouse', async () => { for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) { const q = QUERIES[dataset]; const warmup_ms: number[] = []; const samples_ms: number[] = []; let rows = 0; for (let i = 0; i < WARMUPS; i += 1) { const t0 = performance.now(); await ch(`${q.clickhouse} FORMAT JSONCompact`); warmup_ms.push(performance.now() - t0); } for (let i = 0; i < REPETITIONS; i += 1) { const t0 = performance.now(); const text = await ch(`${q.clickhouse} FORMAT JSONCompact`); samples_ms.push(performance.now() - t0); rows = (JSON.parse(text) as { rows: number }).rows; } runs.push({ engine: 'clickhouse', dataset, query_label: q.label, warmup_ms, samples_ms, distribution: summarizeSamples(samples_ms), rows_returned: rows, }); } expect(runs.filter((r) => r.engine === 'clickhouse')).toHaveLength(6); for (const r of runs) { expect(r.distribution.n).toBe(REPETITIONS); expect(r.distribution.stability).not.toBe('insufficient_samples'); } }, 900_000); it('measures the same six on DuckDB, or records why it did not', async () => { if (!DUCKDB_PYTHON) { engines.push({ engine: 'duckdb', available: false, reason: 'DUCKDB_PYTHON not set on this runner', }); expect(engines.find((e) => e.engine === 'duckdb')?.reason).toBeTruthy(); return; } const payload = Object.fromEntries( (Object.keys(QUERIES) as BenchDatasetId[]).map((d) => [d, QUERIES[d].duckdb]), ); const stdout = execFileSync(DUCKDB_PYTHON, ['-c', DUCKDB_PROGRAM], { input: JSON.stringify(payload), encoding: 'utf8', maxBuffer: 32 * 1024 * 1024, timeout: 900_000, }); // Parse from the sentinel: anything DuckDB writes to stdout before it // (a progress bar, a warning) must not be able to corrupt the transcript. const marker = stdout.lastIndexOf('<<>>'); expect(marker, 'duckdb transcript marker missing').toBeGreaterThanOrEqual(0); duckdbResult = JSON.parse(stdout.slice(marker + '<<>>'.length)); engines.push({ engine: 'duckdb', available: true, version: duckdbResult!.version }); for (const [dataset, r] of Object.entries(duckdbResult!.runs)) { runs.push({ engine: 'duckdb', dataset: dataset as BenchDatasetId, query_label: QUERIES[dataset as BenchDatasetId].label, warmup_ms: r.warmup_ms, samples_ms: r.samples_ms, distribution: summarizeSamples(r.samples_ms), rows_returned: r.rows, }); } expect(runs.filter((r) => r.engine === 'duckdb')).toHaveLength(6); }, 900_000); it('records the engines BI-P104 names that nothing measured, with reasons', () => { engines.push( { engine: 'trino', available: false, reason: 'a Trino is running on this host but belongs to another lane (st0re-verify-trino); ' + 'benchmarking through someone else’s container would measure their configuration', }, { engine: 'semantic_query', available: false, reason: 'MetricFlow has never been executed in 0pt (BI-P029 honest status)', }, { engine: 'superset', available: false, reason: 'no Superset deployment in the estate yet' }, { engine: 'native_charts', available: false, reason: 'render-side; DES1GN owns it (BI-P051..P057)' }, { engine: 'perspective', available: false, reason: 'not deployed' }, ); const evidence = buildEvidence({ host_note: 'x', engines, runs }); for (const e of evidence.not_exercised) expect(e.reason, e.engine).toBeTruthy(); expect(evidence.engines).toHaveLength(7); }); it('reports every comparison as a range with its stability, never a bare figure', () => { if (!duckdbResult) return; for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) { const a = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset)!; const b = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset)!; const sentence = describeComparison(compareDistributions(a.distribution, b.distribution), dataset); expect(sentence).toMatch(/n=9\/9/); expect(sentence).toMatch(/INDISTINGUISHABLE|x to /); } }); });