admin/0pt
5
0
Fork 0
Code Issues 10 Pull Requests Actions 2 Packages Projects Releases Wiki Activity

feat(analytics): BI-P104 — a benchmark harness that refuses to quote one number
CI (SIAX Cloud) / security (push) Successful in 30s
Details
CI (SIAX Cloud) / integration (push) Successful in 44s
Details
Security Scan (SIAX Cloud) / security (push) Successful in 36s
Details
CI (SIAX Cloud) / quality (push) Successful in 1m14s
Details
CI (SIAX Cloud) / security (pull_request) Successful in 34s
Details
CI (SIAX Cloud) / integration (pull_request) Successful in 50s
Details
Security Scan (SIAX Cloud) / security (pull_request) Successful in 27s
Details
CI (SIAX Cloud) / quality (pull_request) Successful in 1m7s
Details

Browse Source 
Six dataset profiles (small SaaS, medium business, VehIQ market, large event,
wide table, geospatial), one representative question each, run on ClickHouse
26.3.10.60 and DuckDB 1.5.5 with 2 warmups discarded and 9 timed repetitions.
evidence/bi-p104/benchmark.json.

The harness's job is to stop the benchmark from lying:

  - overlapping ranges are INDISTINGUISHABLE whatever the medians say. Three
    of the six real comparisons came out that way, one of them with a 1.6x
    median gap — which is exactly the headline a noisy host manufactures.
  - every sentence carries the range, the sample size and the stability, so it
    cannot be trimmed to "4.4x faster" without visibly deleting the caveats.
  - under 5 samples nothing is comparable at all. A three-run benchmark is an
    anecdote.
  - every engine BI-P104 names appears in the evidence, and the five nothing
    measured carry a reason each. The running Trino belongs to another lane,
    so benchmarking through it would have measured their configuration.

Measured shape, warm cache, one idle host, these query shapes only: DuckDB
faster on small_saas (1.6-8.2x) and vehiq_market (2.4-13.3x), SLOWER on
large_event (0.2-0.9x, count DISTINCT over 5M rows), indistinguishable on
medium_business, wide_table and geospatial.

One finding from the harness's own tests: an engine declared available that
produced no run came out of buildEvidence() with no reason — the dangerous
case, since it reads as "we could have and chose not to say". It now gets a
generated reason. A second from the runs: DuckDB's progress bar writes to
STDOUT and corrupted the first transcript; it is disabled explicitly and the
JSON is read from a sentinel.

10 unit tests + 4 against real engines.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
2026-09-07 04:06:43 +02:00
parent ed6e62809d
commit f33cc8d32f
4 changed files with 1517 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
evidence/bi-p104/benchmark.json
+638
View File
@@ -0,0 +1,638 @@
{
"recorded_at": "2026-09-07T02:06:11.079Z",
"host_note": "One idle host, warm caches, both engines local, single-threaded client. These numbers describe these query shapes on this data on this machine. They are not engine rankings.",
"datasets": [
{
"id": "small_saas",
"label": "small SaaS",
"approx_rows": 100000,
"shape": "narrow fact table, one tenant, high selectivity"
},
{
"id": "medium_business",
"label": "medium business",
"approx_rows": 2000000,
"shape": "multi-tenant fact table with a date and a few dimensions"
},
{
"id": "vehiq_market",
"label": "VehIQ market",
"approx_rows": 1000000,
"shape": "vehicle listings: make/model/year/price/mileage, skewed cardinalities"
},
{
"id": "large_event",
"label": "large event",
"approx_rows": 5000000,
"shape": "append-only event stream, timestamp-ordered, sparse dimensions"
},
{
"id": "wide_table",
"label": "wide table",
"approx_rows": 200000,
"shape": "many columns, few read per query — the columnar-store best case"
},
{
"id": "geospatial",
"label": "geospatial",
"approx_rows": 500000,
"shape": "lat/lon points with bounding-box and distance predicates"
}
],
"engines": [
{
"engine": "clickhouse",
"available": true,
"version": "26.3.10.60"
},
{
"engine": "duckdb",
"available": true,
"version": "1.5.5"
},
{
"engine": "trino",
"available": false,
"reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration"
},
{
"engine": "semantic_query",
"available": false,
"reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)"
},
{
"engine": "superset",
"available": false,
"reason": "no Superset deployment in the estate yet"
},
{
"engine": "native_charts",
"available": false,
"reason": "render-side; DES1GN owns it (BI-P051..P057)"
},
{
"engine": "perspective",
"available": false,
"reason": "not deployed"
}
],
"runs": [
{
"engine": "clickhouse",
"dataset": "small_saas",
"query_label": "revenue by day",
"warmup_ms": [58.856714000000466, 58.77232499999991],
"samples_ms": [
51.25258499999927, 23.924909000000298, 39.89413500000046, 29.40832799999953, 43.058938000000126,
39.98396799999955, 41.99595799999952, 32.193271000000095, 25.963267999999516
],
"distribution": {
"n": 9,
"min": 23.924909000000298,
"p50": 39.89413500000046,
"p95": 51.25258499999927,
"p99": 51.25258499999927,
"max": 51.25258499999927,
"mean": 36.40837333333315,
"stdev": 8.514314573768598,
"cv": 0.23385594560395923,
"iqr": 12.58762999999999,
"spread_ratio": 2.1422269568506462,
"stability": "noisy"
},
"rows_returned": 365
},
{
"engine": "clickhouse",
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"warmup_ms": [44.60071400000015, 70.7402890000003],
"samples_ms": [
70.63034199999947, 69.65554000000066, 58.036339000000225, 46.82895400000052, 44.67711700000018,
87.97018200000002, 49.249023999999736, 59.75650299999961, 41.953865000000405
],
"distribution": {
"n": 9,
"min": 41.953865000000405,
"p50": 58.036339000000225,
"p95": 87.97018200000002,
"p99": 87.97018200000002,
"max": 87.97018200000002,
"mean": 58.750874000000096,
"stdev": 14.251513712332908,
"cv": 0.24257534811027467,
"iqr": 22.826586000000134,
"spread_ratio": 2.096831412314436,
"stability": "noisy"
},
"rows_returned": 120
},
{
"engine": "clickhouse",
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"warmup_ms": [83.03972700000031, 59.77780099999927],
"samples_ms": [
74.38707999999951, 60.41443299999992, 59.13954199999989, 98.55565299999944, 98.45707200000015,
74.05828699999984, 194.5833190000003, 159.76290199999949, 87.88670300000013
],
"distribution": {
"n": 9,
"min": 59.13954199999989,
"p50": 87.88670300000013,
"p95": 194.5833190000003,
"p99": 194.5833190000003,
"max": 194.5833190000003,
"mean": 100.80499899999985,
"stdev": 43.75254087132061,
"cv": 0.4340314598021143,
"iqr": 24.4973659999996,
"spread_ratio": 3.290240546671813,
"stability": "noisy"
},
"rows_returned": 60
},
{
"engine": "clickhouse",
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"warmup_ms": [122.44560499999989, 136.30929500000002],
"samples_ms": [
142.78732200000013, 136.18330800000058, 82.08186000000023, 95.91282499999943, 74.52815499999997,
93.95668399999977, 112.81592300000011, 76.3508299999994, 93.78873100000055
],
"distribution": {
"n": 9,
"min": 74.52815499999997,
"p50": 93.95668399999977,
"p95": 142.78732200000013,
"p99": 142.78732200000013,
"max": 142.78732200000013,
"mean": 100.9339597777778,
"stdev": 23.374602687288522,
"cv": 0.23158313355337923,
"iqr": 30.73406299999988,
"spread_ratio": 1.9158842990276652,
"stability": "noisy"
},
"rows_returned": 720
},
{
"engine": "clickhouse",
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"warmup_ms": [28.096958000000086, 14.89937800000007],
"samples_ms": [
19.301219999999375, 21.628866000000926, 32.125191999999515, 25.563218999999663, 24.388967999999295,
16.8507410000002, 19.57230399999935, 20.52187000000049, 18.078355999999985
],
"distribution": {
"n": 9,
"min": 16.8507410000002,
"p50": 20.52187000000049,
"p95": 32.125191999999515,
"p99": 32.125191999999515,
"max": 32.125191999999515,
"mean": 22.00341511111098,
"stdev": 4.451179566867264,
"cv": 0.2022949412348072,
"iqr": 5.08774799999992,
"spread_ratio": 1.9064557457739775,
"stability": "noisy"
},
"rows_returned": 500
},
{
"engine": "clickhouse",
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"warmup_ms": [38.77805500000068, 17.665412000000288],
"samples_ms": [
29.736737000001085, 27.89697900000101, 25.586840999998458, 21.336174000000028, 22.492867999999362,
30.176470000000336, 22.63948900000105, 29.021173000001, 36.31744599999911
],
"distribution": {
"n": 9,
"min": 21.336174000000028,
"p50": 27.89697900000101,
"p95": 36.31744599999911,
"p99": 36.31744599999911,
"max": 36.31744599999911,
"mean": 27.244908555555714,
"stdev": 4.498748211227398,
"cv": 0.16512252929951657,
"iqr": 7.097248000000036,
"spread_ratio": 1.7021536288558137,
"stability": "noisy"
},
"rows_returned": 201
},
{
"engine": "duckdb",
"dataset": "small_saas",
"query_label": "revenue by day",
"warmup_ms": [12.830814997869311, 8.479544001602335],
"samples_ms": [
7.375242999842158, 9.8867060005432, 9.773318997758906, 9.169243003270822, 7.295589999557706,
10.63253999745939, 6.214401000761427, 15.184157000476262, 7.2293630000785924
],
"distribution": {
"n": 9,
"min": 6.214401000761427,
"p50": 9.169243003270822,
"p95": 15.184157000476262,
"p99": 15.184157000476262,
"max": 15.184157000476262,
"mean": 9.19561799997205,
"stdev": 2.5477019265669423,
"cv": 0.27705608547187216,
"iqr": 2.591116000985494,
"spread_ratio": 2.443382234042477,
"stability": "noisy"
},
"rows_returned": 365
},
{
"engine": "duckdb",
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"warmup_ms": [60.29315100022359, 47.50260100263404],
"samples_ms": [
47.711714996694354, 36.605457000405295, 30.622688002040377, 44.33469899959164, 49.469069999759085,
33.921502999874065, 33.03168200000073, 40.499283000826836, 33.27470799922594
],
"distribution": {
"n": 9,
"min": 30.622688002040377,
"p50": 36.605457000405295,
"p95": 49.469069999759085,
"p99": 49.469069999759085,
"max": 49.469069999759085,
"mean": 38.8300894442687,
"stdev": 6.549355811539825,
"cv": 0.1686670287211122,
"iqr": 11.059991000365699,
"spread_ratio": 1.615438527031428,
"stability": "noisy"
},
"rows_returned": 120
},
{
"engine": "duckdb",
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"warmup_ms": [17.676940999081125, 24.05106900187093],
"samples_ms": [
14.901875001669396, 14.926752002793364, 17.798165001295274, 15.059515000757528, 19.75441800095723,
14.601105001929682, 14.954296002542833, 20.32349000000977, 24.610873999336036
],
"distribution": {
"n": 9,
"min": 14.601105001929682,
"p50": 15.059515000757528,
"p95": 24.610873999336036,
"p99": 24.610873999336036,
"max": 24.610873999336036,
"mean": 17.436721112365678,
"stdev": 3.2978767006322265,
"cv": 0.18913399367805778,
"iqr": 4.827665998163866,
"spread_ratio": 1.6855487304613908,
"stability": "noisy"
},
"rows_returned": 60
},
{
"engine": "duckdb",
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"warmup_ms": [184.43432099957136, 163.9354379985889],
"samples_ms": [
160.28642499804846, 161.1321640011738, 177.48030700022355, 213.59406000192394, 337.0364080001309,
334.6067929996934, 232.15600899857236, 204.8815359994478, 196.73219499964034
],
"distribution": {
"n": 9,
"min": 160.28642499804846,
"p50": 204.8815359994478,
"p95": 337.0364080001309,
"p99": 337.0364080001309,
"max": 337.0364080001309,
"mean": 224.21176633320607,
"stdev": 63.65742323059742,
"cv": 0.28391651460429923,
"iqr": 54.67570199834881,
"spread_ratio": 2.1027133645549485,
"stability": "noisy"
},
"rows_returned": 720
},
{
"engine": "duckdb",
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"warmup_ms": [18.729354997049086, 21.23752199986484],
"samples_ms": [
14.388244999281596, 15.164796001045033, 18.07164799902239, 22.90704599727178, 19.191682997188764,
16.838232000736753, 16.024758999265032, 24.479674000758678, 22.105899999587564
],
"distribution": {
"n": 9,
"min": 14.388244999281596,
"p50": 18.07164799902239,
"p95": 24.479674000758678,
"p99": 24.479674000758678,
"max": 24.479674000758678,
"mean": 18.796886999350843,
"stdev": 3.4151315245862714,
"cv": 0.1816860166635153,
"iqr": 6.081141000322532,
"spread_ratio": 1.7013662195758374,
"stability": "noisy"
},
"rows_returned": 500
},
{
"engine": "duckdb",
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"warmup_ms": [27.98272599829943, 22.078424000937957],
"samples_ms": [
30.961295000452083, 28.811156000301708, 19.97572500113165, 22.212601001228904, 33.500631998322206,
34.46119200089015, 24.307676001626533, 20.643488001951482, 24.275364998175064
],
"distribution": {
"n": 9,
"min": 19.97572500113165,
"p50": 24.307676001626533,
"p95": 34.46119200089015,
"p99": 34.46119200089015,
"max": 34.46119200089015,
"mean": 26.572125556008864,
"stdev": 5.191758162396437,
"cv": 0.19538362301703047,
"iqr": 8.748693999223178,
"spread_ratio": 1.7251535050136042,
"stability": "noisy"
},
"rows_returned": 201
}
],
"comparisons": [
{
"dataset": "small_saas",
"query_label": "revenue by day",
"description": "duckdb vs clickhouse (small_saas): FASTER — 1.6x to 8.2x (median 4.4x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 4.350864622714171,
"conservative_ratio": 1.5756494745971,
"optimistic_ratio": 8.247389409489264,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 23.924909000000298,
"p50": 39.89413500000046,
"p95": 51.25258499999927,
"p99": 51.25258499999927,
"max": 51.25258499999927,
"mean": 36.40837333333315,
"stdev": 8.514314573768598,
"cv": 0.23385594560395923,
"iqr": 12.58762999999999,
"spread_ratio": 2.1422269568506462,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 6.214401000761427,
"p50": 9.169243003270822,
"p95": 15.184157000476262,
"p99": 15.184157000476262,
"max": 15.184157000476262,
"mean": 9.19561799997205,
"stdev": 2.5477019265669423,
"cv": 0.27705608547187216,
"iqr": 2.591116000985494,
"spread_ratio": 2.443382234042477,
"stability": "noisy"
},
"verdict": "faster"
},
{
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"description": "duckdb vs clickhouse (medium_business): INDISTINGUISHABLE at this sample size. Medians differ by 1.6x but the observed ranges overlap (baseline 42.0-88.0ms, candidate 30.6-49.5ms; n=9/9, CV 0.24/0.17).",
"p50_ratio": 1.5854559335062433,
"conservative_ratio": 0.8480827515092707,
"optimistic_ratio": 2.872712610798197,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 41.953865000000405,
"p50": 58.036339000000225,
"p95": 87.97018200000002,
"p99": 87.97018200000002,
"max": 87.97018200000002,
"mean": 58.750874000000096,
"stdev": 14.251513712332908,
"cv": 0.24257534811027467,
"iqr": 22.826586000000134,
"spread_ratio": 2.096831412314436,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 30.622688002040377,
"p50": 36.605457000405295,
"p95": 49.469069999759085,
"p99": 49.469069999759085,
"max": 49.469069999759085,
"mean": 38.8300894442687,
"stdev": 6.549355811539825,
"cv": 0.1686670287211122,
"iqr": 11.059991000365699,
"spread_ratio": 1.615438527031428,
"stability": "noisy"
},
"verdict": "indistinguishable"
},
{
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"description": "duckdb vs clickhouse (vehiq_market): FASTER — 2.4x to 13.3x (median 5.8x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 5.835958395444954,
"conservative_ratio": 2.4029842256555125,
"optimistic_ratio": 13.32661596326334,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 59.13954199999989,
"p50": 87.88670300000013,
"p95": 194.5833190000003,
"p99": 194.5833190000003,
"max": 194.5833190000003,
"mean": 100.80499899999985,
"stdev": 43.75254087132061,
"cv": 0.4340314598021143,
"iqr": 24.4973659999996,
"spread_ratio": 3.290240546671813,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 14.601105001929682,
"p50": 15.059515000757528,
"p95": 24.610873999336036,
"p99": 24.610873999336036,
"max": 24.610873999336036,
"mean": 17.436721112365678,
"stdev": 3.2978767006322265,
"cv": 0.18913399367805778,
"iqr": 4.827665998163866,
"spread_ratio": 1.6855487304613908,
"stability": "noisy"
},
"verdict": "faster"
},
{
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"description": "duckdb vs clickhouse (large_event): SLOWER — 0.2x to 0.9x (median 0.5x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 0.4585902948338546,
"conservative_ratio": 0.2211279055643479,
"optimistic_ratio": 0.8908260446992852,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 74.52815499999997,
"p50": 93.95668399999977,
"p95": 142.78732200000013,
"p99": 142.78732200000013,
"max": 142.78732200000013,
"mean": 100.9339597777778,
"stdev": 23.374602687288522,
"cv": 0.23158313355337923,
"iqr": 30.73406299999988,
"spread_ratio": 1.9158842990276652,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 160.28642499804846,
"p50": 204.8815359994478,
"p95": 337.0364080001309,
"p99": 337.0364080001309,
"max": 337.0364080001309,
"mean": 224.21176633320607,
"stdev": 63.65742323059742,
"cv": 0.28391651460429923,
"iqr": 54.67570199834881,
"spread_ratio": 2.1027133645549485,
"stability": "noisy"
},
"verdict": "slower"
},
{
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"description": "duckdb vs clickhouse (wide_table): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 16.9-32.1ms, candidate 14.4-24.5ms; n=9/9, CV 0.20/0.18).",
"p50_ratio": 1.1355837608784018,
"conservative_ratio": 0.688356429888648,
"optimistic_ratio": 2.232738739269697,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 16.8507410000002,
"p50": 20.52187000000049,
"p95": 32.125191999999515,
"p99": 32.125191999999515,
"max": 32.125191999999515,
"mean": 22.00341511111098,
"stdev": 4.451179566867264,
"cv": 0.2022949412348072,
"iqr": 5.08774799999992,
"spread_ratio": 1.9064557457739775,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 14.388244999281596,
"p50": 18.07164799902239,
"p95": 24.479674000758678,
"p99": 24.479674000758678,
"max": 24.479674000758678,
"mean": 18.796886999350843,
"stdev": 3.4151315245862714,
"cv": 0.1816860166635153,
"iqr": 6.081141000322532,
"spread_ratio": 1.7013662195758374,
"stability": "noisy"
},
"verdict": "indistinguishable"
},
{
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"description": "duckdb vs clickhouse (geospatial): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 21.3-36.3ms, candidate 20.0-34.5ms; n=9/9, CV 0.17/0.20).",
"p50_ratio": 1.1476612983542442,
"conservative_ratio": 0.619136273621902,
"optimistic_ratio": 1.8180789932751718,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 21.336174000000028,
"p50": 27.89697900000101,
"p95": 36.31744599999911,
"p99": 36.31744599999911,
"max": 36.31744599999911,
"mean": 27.244908555555714,
"stdev": 4.498748211227398,
"cv": 0.16512252929951657,
"iqr": 7.097248000000036,
"spread_ratio": 1.7021536288558137,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 19.97572500113165,
"p50": 24.307676001626533,
"p95": 34.46119200089015,
"p99": 34.46119200089015,
"max": 34.46119200089015,
"mean": 26.572125556008864,
"stdev": 5.191758162396437,
"cv": 0.19538362301703047,
"iqr": 8.748693999223178,
"spread_ratio": 1.7251535050136042,
"stability": "noisy"
},
"verdict": "indistinguishable"
}
],
"not_exercised": [
{
"engine": "trino",
"available": false,
"reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration"
},
{
"engine": "semantic_query",
"available": false,
"reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)"
},
{
"engine": "superset",
"available": false,
"reason": "no Superset deployment in the estate yet"
},
{
"engine": "native_charts",
"available": false,
"reason": "render-side; DES1GN owns it (BI-P051..P057)"
},
{
"engine": "perspective",
"available": false,
"reason": "not deployed"
}
]
}
src/analytics/bench/benchmark-harness.ts
+341
View File
@@ -0,0 +1,341 @@
/**
* src/analytics/bench/benchmark-harness.ts
* =========================================
* BI-P104 — BI PERFORMANCE BENCHMARK HARNESS. DoD: "results committed as
* machine-readable evidence."
*
* The harness exists because of a specific way benchmarks lie. Yesterday's
* lane measured the same budget-enforcement overshoot at ~150x on one run and
* ~130x on the next, on the same machine, and correctly reported the SHAPE
* rather than a number. That is the doctrine encoded here:
*
* 1. A SINGLE FIGURE IS NOT A SYSTEM PROPERTY. `summarizeSamples` returns a
* distribution — n, min, p50, p95, p99, max, mean, stdev, CV, IQR — and
* there is no function anywhere in this module that returns one number
* for a measurement. `describeComparison` renders a RANGE and the sample
* size into every sentence it produces.
*
* 2. OVERLAPPING DISTRIBUTIONS ARE INDISTINGUISHABLE, whatever the medians
* say. If the candidate's p95 is not below the baseline's minimum, the
* verdict is `indistinguishable` even when p50 differs by 2x — because a
* 2x median gap inside a 10x spread is noise with a story attached.
*
* 3. TOO FEW SAMPLES IS A REFUSAL. Under MIN_SAMPLES the summary carries
* `stability: 'insufficient_samples'` and comparisons refuse to judge.
* A three-run benchmark is an anecdote.
*
* 4. AN ENGINE THAT WAS NOT RUN IS RECORDED AS NOT RUN. `EngineAvailability`
* carries a reason. The evidence file lists every engine BI-P104 names,
* including the ones nothing measured — an omitted engine reads as "we
* chose not to report it".
*/
/** The six dataset profiles BI-P104 names, with what makes each one distinct. */
export const BENCH_DATASETS = [
{
id: 'small_saas',
label: 'small SaaS',
approx_rows: 100_000,
shape: 'narrow fact table, one tenant, high selectivity',
},
{
id: 'medium_business',
label: 'medium business',
approx_rows: 2_000_000,
shape: 'multi-tenant fact table with a date and a few dimensions',
},
{
id: 'vehiq_market',
label: 'VehIQ market',
approx_rows: 1_000_000,
shape: 'vehicle listings: make/model/year/price/mileage, skewed cardinalities',
},
{
id: 'large_event',
label: 'large event',
approx_rows: 5_000_000,
shape: 'append-only event stream, timestamp-ordered, sparse dimensions',
},
{
id: 'wide_table',
label: 'wide table',
approx_rows: 200_000,
shape: 'many columns, few read per query — the columnar-store best case',
},
{
id: 'geospatial',
label: 'geospatial',
approx_rows: 500_000,
shape: 'lat/lon points with bounding-box and distance predicates',
},
] as const;
export type BenchDatasetId = (typeof BENCH_DATASETS)[number]['id'];
/** Every engine BI-P104 lists. All of them appear in the evidence, run or not. */
export const BENCH_ENGINES = [
'clickhouse',
'duckdb',
'trino',
'semantic_query',
'superset',
'native_charts',
'perspective',
] as const;
export type BenchEngine = (typeof BENCH_ENGINES)[number];
export interface EngineAvailability {
engine: BenchEngine;
available: boolean;
/** Required when unavailable. "Unknown" is not a reason. */
reason?: string;
version?: string;
}
export const STABILITY_VERDICTS = ['stable', 'noisy', 'unstable', 'insufficient_samples'] as const;
export type StabilityVerdict = (typeof STABILITY_VERDICTS)[number];
/** Below this a run is an anecdote, not a measurement. */
export const MIN_SAMPLES = 5;
/** Coefficient of variation thresholds. Wide on purpose: real hosts are noisy. */
const CV_STABLE = 0.15;
const CV_NOISY = 0.5;
export interface Distribution {
n: number;
min: number;
p50: number;
p95: number;
p99: number;
max: number;
mean: number;
stdev: number;
/** stdev / mean. The number that decides whether a comparison is meaningful. */
cv: number;
iqr: number;
/** max / min. Reported because it is what a reader feels. */
spread_ratio: number;
stability: StabilityVerdict;
}
/** Nearest-rank; no interpolation, so every reported value really happened. */
function pct(sorted: number[], q: number): number {
const rank = Math.ceil(q * sorted.length);
return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
}
export function summarizeSamples(samples: readonly number[]): Distribution {
if (samples.length === 0) {
return {
n: 0,
min: 0,
p50: 0,
p95: 0,
p99: 0,
max: 0,
mean: 0,
stdev: 0,
cv: 0,
iqr: 0,
spread_ratio: 0,
stability: 'insufficient_samples',
};
}
const s = [...samples].sort((a, b) => a - b);
const mean = s.reduce((a, b) => a + b, 0) / s.length;
const variance = s.reduce((acc, v) => acc + (v - mean) ** 2, 0) / s.length;
const stdev = Math.sqrt(variance);
const cv = mean === 0 ? 0 : stdev / mean;
const stability: StabilityVerdict =
s.length < MIN_SAMPLES
? 'insufficient_samples'
: cv <= CV_STABLE
? 'stable'
: cv <= CV_NOISY
? 'noisy'
: 'unstable';
return {
n: s.length,
min: s[0],
p50: pct(s, 0.5),
p95: pct(s, 0.95),
p99: pct(s, 0.99),
max: s[s.length - 1],
mean,
stdev,
cv,
iqr: pct(s, 0.75) - pct(s, 0.25),
spread_ratio: s[0] === 0 ? Number.POSITIVE_INFINITY : s[s.length - 1] / s[0],
stability,
};
}
export const COMPARISON_VERDICTS = ['faster', 'slower', 'indistinguishable', 'not_comparable'] as const;
export type ComparisonVerdict = (typeof COMPARISON_VERDICTS)[number];
export interface Comparison {
verdict: ComparisonVerdict;
/** Ratio of medians. Present for information; never the claim on its own. */
p50_ratio: number;
/** Worst case for the candidate: baseline.min / candidate.max. */
conservative_ratio: number;
/** Best case for the candidate: baseline.max / candidate.min. */
optimistic_ratio: number;
/** True when the observed ranges overlap at all. */
ranges_overlap: boolean;
baseline: Distribution;
candidate: Distribution;
}
/**
* Compare two measured distributions.
*
* `faster` requires clean separation — the candidate's p95 below the
* baseline's minimum. Anything less is `indistinguishable`, however flattering
* the medians look, because that is exactly the claim a noisy host will
* manufacture for you.
*/
export function compareDistributions(baseline: Distribution, candidate: Distribution): Comparison {
const ratio = (a: number, b: number) => (b === 0 ? Number.POSITIVE_INFINITY : a / b);
const base: Omit<Comparison, 'verdict'> = {
p50_ratio: ratio(baseline.p50, candidate.p50),
conservative_ratio: ratio(baseline.min, candidate.max),
optimistic_ratio: ratio(baseline.max, candidate.min),
ranges_overlap: !(candidate.max < baseline.min || baseline.max < candidate.min),
baseline,
candidate,
};
if (baseline.stability === 'insufficient_samples' || candidate.stability === 'insufficient_samples') {
return { ...base, verdict: 'not_comparable' };
}
if (candidate.p95 < baseline.min) return { ...base, verdict: 'faster' };
if (baseline.p95 < candidate.min) return { ...base, verdict: 'slower' };
return { ...base, verdict: 'indistinguishable' };
}
/**
* The sentence a human is allowed to quote. It ALWAYS carries the range, the
* sample size and the stability verdict, so it cannot be trimmed down to "3.4x
* faster" without visibly removing the caveats.
*/
export function describeComparison(c: Comparison, label = 'candidate'): string {
const r = (v: number) => (Number.isFinite(v) ? v.toFixed(1) : '∞');
if (c.verdict === 'not_comparable') {
return (
`${label}: NOT COMPARABLE — ${c.baseline.n} baseline and ${c.candidate.n} candidate samples, ` +
`fewer than the ${MIN_SAMPLES} required. A run this short is an anecdote.`
);
}
if (c.verdict === 'indistinguishable') {
return (
`${label}: INDISTINGUISHABLE at this sample size. Medians differ by ${r(c.p50_ratio)}x but the ` +
`observed ranges overlap (baseline ${c.baseline.min.toFixed(1)}-${c.baseline.max.toFixed(1)}ms, ` +
`candidate ${c.candidate.min.toFixed(1)}-${c.candidate.max.toFixed(1)}ms; n=${c.baseline.n}/${c.candidate.n}, ` +
`CV ${c.baseline.cv.toFixed(2)}/${c.candidate.cv.toFixed(2)}).`
);
}
return (
`${label}: ${c.verdict.toUpperCase()} — ${r(c.conservative_ratio)}x to ${r(c.optimistic_ratio)}x ` +
`(median ${r(c.p50_ratio)}x) over n=${c.baseline.n}/${c.candidate.n}, ` +
`stability ${c.baseline.stability}/${c.candidate.stability}. Warm cache, one host, this query shape.`
);
}
export interface BenchmarkRun {
engine: BenchEngine;
dataset: BenchDatasetId;
query_label: string;
/** Discarded from the statistics; recorded so the discard is visible. */
warmup_ms: number[];
samples_ms: number[];
distribution: Distribution;
rows_returned?: number;
}
/**
* Run one measurement. Warmups are executed and RECORDED but excluded from the
* distribution: a first run that pays for cold marks and page cache is a
* different measurement, and silently folding it in is how a p95 becomes a
* story about disk.
*/
export async function measure(args: {
engine: BenchEngine;
dataset: BenchDatasetId;
query_label: string;
run: () => Promise<{ rows?: number }>;
warmups?: number;
repetitions?: number;
now?: () => number;
}): Promise<BenchmarkRun> {
const now = args.now ?? (() => performance.now());
const warmups = args.warmups ?? 2;
const repetitions = args.repetitions ?? 9;
const warmup_ms: number[] = [];
const samples_ms: number[] = [];
let rows: number | undefined;
for (let i = 0; i < warmups; i += 1) {
const t0 = now();
await args.run();
warmup_ms.push(now() - t0);
}
for (let i = 0; i < repetitions; i += 1) {
const t0 = now();
const out = await args.run();
samples_ms.push(now() - t0);
rows = out.rows ?? rows;
}
return {
engine: args.engine,
dataset: args.dataset,
query_label: args.query_label,
warmup_ms,
samples_ms,
distribution: summarizeSamples(samples_ms),
...(rows !== undefined ? { rows_returned: rows } : {}),
};
}
export interface BenchmarkEvidence {
recorded_at: string;
host_note: string;
datasets: typeof BENCH_DATASETS;
engines: EngineAvailability[];
runs: BenchmarkRun[];
comparisons: Array<{ dataset: BenchDatasetId; query_label: string; description: string } & Comparison>;
/** Engines BI-P104 names that produced no measurement, each with a reason. */
not_exercised: EngineAvailability[];
}
/**
* Assemble the evidence document. Every engine BI-P104 names appears, and the
* ones with no runs are listed under `not_exercised` with their reason — an
* engine quietly missing from a benchmark table reads as a choice.
*/
export function buildEvidence(args: {
host_note: string;
engines: EngineAvailability[];
runs: BenchmarkRun[];
comparisons?: BenchmarkEvidence['comparisons'];
now?: Date;
}): BenchmarkEvidence {
const measured = new Set(args.runs.map((r) => r.engine));
const declared = new Map(args.engines.map((e) => [e.engine, e]));
return {
recorded_at: (args.now ?? new Date()).toISOString(),
host_note: args.host_note,
datasets: BENCH_DATASETS,
engines: BENCH_ENGINES.map(
(engine) => declared.get(engine) ?? { engine, available: false, reason: 'not declared by this run' },
),
runs: args.runs,
comparisons: args.comparisons ?? [],
not_exercised: BENCH_ENGINES.filter((e) => !measured.has(e)).map((engine) => {
const d = declared.get(engine);
if (!d) return { engine, available: false, reason: 'not present in this environment' };
// An engine declared AVAILABLE that produced no measurement is the
// dangerous case: it reads as "we could have, and chose not to say".
// It gets a reason too, generated here rather than left blank.
return d.reason ? d : { ...d, reason: 'declared available but produced no measurement in this run' };
}),
};
}
tests/analytics/benchmark-harness.test.ts
+194
View File
@@ -0,0 +1,194 @@
/**
* tests/analytics/benchmark-harness.test.ts
* ==========================================
* BI-P104 — the harness's job is to stop a benchmark from lying.
*
* The tests are therefore attacks on the reporting, not on the timing:
*
* - two overlapping distributions whose MEDIANS differ 2x must come back
* `indistinguishable`, because that is the claim a noisy host manufactures;
* - the real 150x/130x pair from yesterday's budget-enforcement lane is fed
* in as data, and the harness must describe a RANGE rather than pick one;
* - a three-sample run must refuse to be compared at all;
* - every sentence `describeComparison` produces must carry the range, the
* sample size and the stability, so it cannot be quoted as a bare figure;
* - an engine that was not run must appear in the evidence with a reason.
*/
import { describe, expect, it } from 'vitest';
import {
BENCH_ENGINES,
buildEvidence,
compareDistributions,
describeComparison,
measure,
MIN_SAMPLES,
summarizeSamples,
} from '@/src/analytics/bench/benchmark-harness';
describe('BI-P104 distribution summary', () => {
it('reports the shape, and never a single number', () => {
const d = summarizeSamples([10, 11, 12, 13, 14, 15, 16, 40]);
expect(d.n).toBe(8);
expect(d.min).toBe(10);
expect(d.max).toBe(40);
expect(d.p50).toBe(13);
expect(d.spread_ratio).toBe(4);
expect(d.cv).toBeGreaterThan(0);
// Every field a reader needs to judge the p50 is present next to it.
for (const key of [
'min',
'p50',
'p95',
'p99',
'max',
'stdev',
'cv',
'iqr',
'spread_ratio',
'stability',
]) {
expect(Object.keys(d)).toContain(key);
}
});
it('calls a tight run stable and a spread run unstable', () => {
expect(summarizeSamples([100, 101, 99, 100, 102, 98, 100]).stability).toBe('stable');
expect(summarizeSamples([10, 200, 15, 400, 12, 350, 11]).stability).toBe('unstable');
});
it('refuses to characterise fewer than MIN_SAMPLES', () => {
const d = summarizeSamples([10, 11, 12]);
expect(d.n).toBe(3);
expect(d.stability).toBe('insufficient_samples');
expect(MIN_SAMPLES).toBeGreaterThan(3);
});
});
describe('BI-P104 comparison — variance beats a flattering median', () => {
it('overlapping ranges are INDISTINGUISHABLE even when medians differ 2x', () => {
const baseline = summarizeSamples([50, 60, 200, 70, 300, 55, 65, 280]);
const candidate = summarizeSamples([25, 30, 190, 35, 260, 28, 33, 240]);
const c = compareDistributions(baseline, candidate);
expect(c.p50_ratio).toBeGreaterThan(1.5); // the tempting headline
expect(c.ranges_overlap).toBe(true);
expect(c.verdict).toBe('indistinguishable'); // the honest verdict
expect(describeComparison(c)).toMatch(/INDISTINGUISHABLE/);
});
it('clean separation is reported as faster, WITH the range', () => {
const baseline = summarizeSamples([500, 520, 510, 530, 505, 515, 525]);
const candidate = summarizeSamples([50, 55, 52, 58, 51, 53, 54]);
const c = compareDistributions(baseline, candidate);
expect(c.verdict).toBe('faster');
const sentence = describeComparison(c, 'pre-aggregate');
expect(sentence).toMatch(/FASTER/);
expect(sentence).toMatch(/x to /); // a range, not a number
expect(sentence).toMatch(/n=7\/7/); // sample sizes
expect(sentence).toMatch(/stability/);
});
it('the real 150x / 130x overshoot pair is described as a range, not a figure', () => {
// Two runs of the same measurement on the same machine, from the BI-P021
// budget-enforcement lane. A harness that picks one of these is lying.
const overshoot = summarizeSamples([153, 131, 149, 128, 144, 137, 152, 126, 141]);
// Worth being precise about: by the CV test this run is STABLE (0.07). The
// problem was never instability — it is that a stable run still spans
// 126x-153x, so any single figure quoted from it is ~20% wrong. The
// harness's answer is to keep the ends attached to the middle.
expect(overshoot.stability).toBe('stable');
expect(overshoot.min).toBe(126);
expect(overshoot.p50).toBe(141);
expect(overshoot.max).toBe(153);
expect(overshoot.spread_ratio).toBeCloseTo(1.21, 2);
const sentence = describeComparison(
compareDistributions(summarizeSamples([1, 1, 1, 1, 1, 1, 1, 1, 1]), overshoot),
'budget overshoot factor',
);
expect(sentence).toMatch(/x to /); // a range survives into the prose
expect(sentence).toMatch(/n=9\/9/);
});
it('a three-sample run cannot be compared at all', () => {
const c = compareDistributions(summarizeSamples([10, 11, 12]), summarizeSamples([1, 2, 3]));
expect(c.verdict).toBe('not_comparable');
expect(describeComparison(c)).toMatch(/anecdote/);
});
it('slower is reported as slower — the harness has no favourite', () => {
const baseline = summarizeSamples([50, 51, 52, 53, 54, 55, 56]);
const candidate = summarizeSamples([500, 510, 520, 530, 540, 550, 560]);
expect(compareDistributions(baseline, candidate).verdict).toBe('slower');
});
});
describe('BI-P104 measurement', () => {
it('records warmups but excludes them from the statistics', async () => {
let call = 0;
let clock = 0;
const run = await measure({
engine: 'duckdb',
dataset: 'small_saas',
query_label: 'group by day',
warmups: 2,
repetitions: 6,
// A cold first pair (1000ms) then a warm steady state (10ms).
now: () => {
const t = clock;
clock += call <= 2 ? 1000 : 10;
call += 1;
return t;
},
run: async () => ({ rows: 42 }),
});
expect(run.warmup_ms).toHaveLength(2);
expect(run.samples_ms).toHaveLength(6);
expect(run.distribution.n).toBe(6);
expect(run.distribution.max).toBeLessThan(1000); // the cold pair is not in it
expect(run.rows_returned).toBe(42);
});
});
describe('BI-P104 evidence', () => {
it('lists every engine BI-P104 names, and says why the unrun ones were not run', () => {
const evidence = buildEvidence({
host_note: 'one idle host, warm cache',
engines: [
{ engine: 'clickhouse', available: true, version: '26.3' },
{ engine: 'duckdb', available: true, version: '1.5.5' },
{ engine: 'trino', available: false, reason: 'the running Trino belongs to another lane' },
],
runs: [
{
engine: 'clickhouse',
dataset: 'small_saas',
query_label: 'q',
warmup_ms: [1],
samples_ms: [1, 2, 3, 4, 5],
distribution: summarizeSamples([1, 2, 3, 4, 5]),
},
],
});
expect(evidence.engines.map((e) => e.engine)).toEqual([...BENCH_ENGINES]);
const notRun = evidence.not_exercised.map((e) => e.engine);
expect(notRun).toContain('superset');
expect(notRun).toContain('perspective');
expect(notRun).toContain('duckdb'); // declared available but produced no run
for (const e of evidence.not_exercised) {
expect(e.reason, e.engine).toBeTruthy();
}
expect(evidence.datasets.map((d) => d.id)).toEqual([
'small_saas',
'medium_business',
'vehiq_market',
'large_event',
'wide_table',
'geospatial',
]);
});
});
tests/integration/bi-p104-benchmark.integration.test.ts
+344
View File
@@ -0,0 +1,344 @@
/**
* tests/integration/bi-p104-benchmark.integration.test.ts
* ========================================================
* BI-P104 — the benchmark itself, against real engines. Skips honestly when
* CLICKHOUSE_URL is unset; DuckDB additionally needs DUCKDB_PYTHON (an
* interpreter with the `duckdb` package), matching
* tests/duckdb/duckdb-provider.live.test.ts.
*
* Six dataset profiles, one representative question each, the same logical
* question on both engines. Deliberately NO engine-specific functions in the
* geospatial case (bounding box + grid arithmetic rather than each engine's
* GIS library) so the comparison is about the storage and execution engine
* rather than about who ships the nicer geo function.
*
* Statistics come from src/analytics/bench/benchmark-harness.ts — the shipped
* code, not a copy — so the evidence file is produced by the thing under test.
*
* Trino, Superset, native charts and Perspective are NOT exercised, and each
* appears in the evidence with its reason. An engine omitted from a benchmark
* table reads as a choice, so nothing is omitted.
*
* CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \
* DUCKDB_PYTHON=/path/to/venv/bin/python \
* pnpm vitest run tests/integration/bi-p104-benchmark.integration.test.ts
*
* It CREATEs and DROPs its own ClickHouse database and uses an in-memory
* DuckDB. Never point it at the b0ts ClickHouse or any quarantined data.
*/
import { execFileSync } from 'node:child_process';
import fs from 'node:fs';
import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import {
buildEvidence,
compareDistributions,
describeComparison,
summarizeSamples,
type BenchDatasetId,
type BenchmarkRun,
type EngineAvailability,
} from '@/src/analytics/bench/benchmark-harness';
const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const DUCKDB_PYTHON = process.env.DUCKDB_PYTHON;
const run = CLICKHOUSE_URL ? describe : describe.skip;
const DB = `opt_p104_${Date.now().toString(36)}`;
const WARMUPS = 2;
const REPETITIONS = 9;
const WIDE_COLUMNS = 50;
const headers: Record<string, string> = CLICKHOUSE_AUTH
? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` }
: {};
async function ch(query: string): Promise<string> {
const res = await fetch(CLICKHOUSE_URL ?? 'http://skipped.invalid', {
method: 'POST',
headers,
body: query,
});
const text = await res.text();
if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`);
return text;
}
/** The six questions, one per profile, phrased for each engine. */
const QUERIES: Record<BenchDatasetId, { label: string; clickhouse: string; duckdb: string }> = {
small_saas: {
label: 'revenue by day',
clickhouse: `SELECT d, sum(amount) AS revenue, count() AS n FROM ${DB}.small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`,
duckdb: `SELECT d, sum(amount) AS revenue, count(*) AS n FROM small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`,
},
medium_business: {
label: 'revenue by month and region for one tenant',
clickhouse: `SELECT toStartOfMonth(d) AS m, region, sum(amount) AS revenue FROM ${DB}.medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`,
duckdb: `SELECT date_trunc('month', d) AS m, region, sum(amount) AS revenue FROM medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`,
},
vehiq_market: {
label: 'average asking price by make and model year',
clickhouse: `SELECT make, model_year, avg(price) AS avg_price, count() AS listings FROM ${DB}.vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`,
duckdb: `SELECT make, model_year, avg(price) AS avg_price, count(*) AS listings FROM vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`,
},
large_event: {
label: 'hourly event volume for one event type',
clickhouse: `SELECT toStartOfHour(ts) AS h, count() AS n, uniqExact(user_id) AS users FROM ${DB}.large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`,
duckdb: `SELECT date_trunc('hour', ts) AS h, count(*) AS n, count(DISTINCT user_id) AS users FROM large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`,
},
wide_table: {
label: 'three columns of fifty, grouped',
clickhouse: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM ${DB}.wide_table GROUP BY grp ORDER BY grp`,
duckdb: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM wide_table GROUP BY grp ORDER BY grp`,
},
geospatial: {
label: 'bounding box, aggregated to a grid cell',
clickhouse: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count() AS n FROM ${DB}.geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`,
duckdb: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count(*) AS n FROM geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`,
},
};
const CH_DDL: Array<[BenchDatasetId, string]> = [
[
'small_saas',
`CREATE TABLE ${DB}.small_saas ENGINE = MergeTree ORDER BY (tenant_id, d) AS
SELECT concat('t_', toString(number % 3)) AS tenant_id,
toDate('2026-01-01') + toIntervalDay(number % 365) AS d,
['free','pro','enterprise'][(number % 3) + 1] AS plan,
toDecimal64((number % 499) + 9, 2) AS amount
FROM numbers(100000)`,
],
[
'medium_business',
`CREATE TABLE ${DB}.medium_business ENGINE = MergeTree ORDER BY (tenant_id, d) AS
SELECT concat('t_', toString(number % 8)) AS tenant_id,
toDate('2025-01-01') + toIntervalDay(number % 730) AS d,
['SE','NO','DK','FI','DE'][(number % 5) + 1] AS region,
toDecimal64((number % 977) + 13, 2) AS amount
FROM numbers(2000000)`,
],
[
'vehiq_market',
`CREATE TABLE ${DB}.vehiq_market ENGINE = MergeTree ORDER BY (make, model_year) AS
SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(number % 6) + 1] AS make,
toUInt16(2005 + (number % 20)) AS model_year,
toUInt32(30000 + (number % 900000)) AS price,
toUInt32((number * 7) % 400000) AS mileage,
['diesel','petrol','hybrid','electric'][(number % 4) + 1] AS fuel
FROM numbers(1000000)`,
],
[
'large_event',
`CREATE TABLE ${DB}.large_event ENGINE = MergeTree ORDER BY (event_type, ts) AS
SELECT toDateTime('2026-08-01 00:00:00') + toIntervalSecond(number % 2592000) AS ts,
toUInt32(number % 250000) AS user_id,
['view','click','checkout','signup'][(number % 4) + 1] AS event_type,
toFloat64(number % 1000) AS value
FROM numbers(5000000)`,
],
[
'wide_table',
`CREATE TABLE ${DB}.wide_table ENGINE = MergeTree ORDER BY grp AS
SELECT toUInt16(number % 500) AS grp,
${Array.from({ length: WIDE_COLUMNS }, (_, i) => `toFloat64((number + ${i}) % 1000) AS c${i}`).join(',\n ')}
FROM numbers(200000)`,
],
[
'geospatial',
`CREATE TABLE ${DB}.geospatial ENGINE = MergeTree ORDER BY (lat, lon) AS
SELECT 55.0 + (number % 10000) / 1000.0 AS lat,
10.0 + ((number * 13) % 10000) / 1000.0 AS lon,
toUInt32(number) AS point_id
FROM numbers(500000)`,
],
];
/** Runs all six DuckDB measurements inside ONE python process. */
const DUCKDB_PROGRAM = `
import duckdb, json, time, sys
con = duckdb.connect()
# The progress bar writes to STDOUT and corrupts the JSON transcript. It cost
# one run to find, so it is turned off explicitly rather than filtered out.
con.execute("SET enable_progress_bar=false")
WIDE = ${WIDE_COLUMNS}
wide_cols = ", ".join(f"(i + {j}) % 1000.0 AS c{j}" for j in range(WIDE))
ddl = {
"small_saas": "CREATE TABLE small_saas AS SELECT 't_' || (i % 3) AS tenant_id, DATE '2026-01-01' + INTERVAL (i % 365) DAY AS d, ['free','pro','enterprise'][(i % 3) + 1] AS plan, ((i % 499) + 9)::DECIMAL(18,2) AS amount FROM range(100000) t(i)",
"medium_business": "CREATE TABLE medium_business AS SELECT 't_' || (i % 8) AS tenant_id, DATE '2025-01-01' + INTERVAL (i % 730) DAY AS d, ['SE','NO','DK','FI','DE'][(i % 5) + 1] AS region, ((i % 977) + 13)::DECIMAL(18,2) AS amount FROM range(2000000) t(i)",
"vehiq_market": "CREATE TABLE vehiq_market AS SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(i % 6) + 1] AS make, (2005 + (i % 20))::USMALLINT AS model_year, (30000 + (i % 900000))::UINTEGER AS price, ((i * 7) % 400000)::UINTEGER AS mileage, ['diesel','petrol','hybrid','electric'][(i % 4) + 1] AS fuel FROM range(1000000) t(i)",
"large_event": "CREATE TABLE large_event AS SELECT TIMESTAMP '2026-08-01 00:00:00' + INTERVAL (i % 2592000) SECOND AS ts, (i % 250000)::UINTEGER AS user_id, ['view','click','checkout','signup'][(i % 4) + 1] AS event_type, (i % 1000)::DOUBLE AS value FROM range(5000000) t(i)",
"wide_table": f"CREATE TABLE wide_table AS SELECT (i % 500)::USMALLINT AS grp, {wide_cols} FROM range(200000) t(i)",
"geospatial": "CREATE TABLE geospatial AS SELECT 55.0 + (i % 10000) / 1000.0 AS lat, 10.0 + ((i * 13) % 10000) / 1000.0 AS lon, i::UINTEGER AS point_id FROM range(500000) t(i)",
}
queries = json.loads(sys.stdin.read())
out = {"version": duckdb.__version__, "runs": {}}
for name, sql in ddl.items():
con.execute(sql)
for name, q in queries.items():
warm = []
for _ in range(${WARMUPS}):
t0 = time.perf_counter(); rows = con.execute(q).fetchall(); warm.append((time.perf_counter() - t0) * 1000)
samples = []
for _ in range(${REPETITIONS}):
t0 = time.perf_counter(); rows = con.execute(q).fetchall(); samples.append((time.perf_counter() - t0) * 1000)
out["runs"][name] = {"warmup_ms": warm, "samples_ms": samples, "rows": len(rows)}
sys.stdout.write("<<<BENCH>>>" + json.dumps(out))
`;
const runs: BenchmarkRun[] = [];
const engines: EngineAvailability[] = [];
let duckdbResult:
| { version: string; runs: Record<string, { warmup_ms: number[]; samples_ms: number[]; rows: number }> }
| undefined;
run('BI-P104 benchmark harness against real engines', () => {
beforeAll(async () => {
const version = (await ch('SELECT version()')).trim();
engines.push({ engine: 'clickhouse', available: true, version });
await ch(`CREATE DATABASE IF NOT EXISTS ${DB}`);
for (const [, ddl] of CH_DDL) await ch(ddl);
}, 900_000);
afterAll(async () => {
if (!CLICKHOUSE_URL) return;
try {
await ch(`DROP DATABASE IF EXISTS ${DB}`);
} finally {
const comparisons = duckdbResult
? (Object.keys(QUERIES) as BenchDatasetId[]).flatMap((dataset) => {
const chRun = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset);
const duckRun = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset);
if (!chRun || !duckRun) return [];
const c = compareDistributions(chRun.distribution, duckRun.distribution);
return [
{
dataset,
query_label: QUERIES[dataset].label,
description: describeComparison(c, `duckdb vs clickhouse (${dataset})`),
...c,
},
];
})
: [];
const evidence = buildEvidence({
host_note:
'One idle host, warm caches, both engines local, single-threaded client. These numbers ' +
'describe these query shapes on this data on this machine. They are not engine rankings.',
engines,
runs,
comparisons,
});
const dir = path.join(process.cwd(), 'evidence/bi-p104');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(path.join(dir, 'benchmark.json'), `${JSON.stringify(evidence, null, 2)}\n`);
}
}, 120_000);
it('measures all six dataset profiles on ClickHouse', async () => {
for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) {
const q = QUERIES[dataset];
const warmup_ms: number[] = [];
const samples_ms: number[] = [];
let rows = 0;
for (let i = 0; i < WARMUPS; i += 1) {
const t0 = performance.now();
await ch(`${q.clickhouse} FORMAT JSONCompact`);
warmup_ms.push(performance.now() - t0);
}
for (let i = 0; i < REPETITIONS; i += 1) {
const t0 = performance.now();
const text = await ch(`${q.clickhouse} FORMAT JSONCompact`);
samples_ms.push(performance.now() - t0);
rows = (JSON.parse(text) as { rows: number }).rows;
}
runs.push({
engine: 'clickhouse',
dataset,
query_label: q.label,
warmup_ms,
samples_ms,
distribution: summarizeSamples(samples_ms),
rows_returned: rows,
});
}
expect(runs.filter((r) => r.engine === 'clickhouse')).toHaveLength(6);
for (const r of runs) {
expect(r.distribution.n).toBe(REPETITIONS);
expect(r.distribution.stability).not.toBe('insufficient_samples');
}
}, 900_000);
it('measures the same six on DuckDB, or records why it did not', async () => {
if (!DUCKDB_PYTHON) {
engines.push({
engine: 'duckdb',
available: false,
reason: 'DUCKDB_PYTHON not set on this runner',
});
expect(engines.find((e) => e.engine === 'duckdb')?.reason).toBeTruthy();
return;
}
const payload = Object.fromEntries(
(Object.keys(QUERIES) as BenchDatasetId[]).map((d) => [d, QUERIES[d].duckdb]),
);
const stdout = execFileSync(DUCKDB_PYTHON, ['-c', DUCKDB_PROGRAM], {
input: JSON.stringify(payload),
encoding: 'utf8',
maxBuffer: 32 * 1024 * 1024,
timeout: 900_000,
});
// Parse from the sentinel: anything DuckDB writes to stdout before it
// (a progress bar, a warning) must not be able to corrupt the transcript.
const marker = stdout.lastIndexOf('<<<BENCH>>>');
expect(marker, 'duckdb transcript marker missing').toBeGreaterThanOrEqual(0);
duckdbResult = JSON.parse(stdout.slice(marker + '<<<BENCH>>>'.length));
engines.push({ engine: 'duckdb', available: true, version: duckdbResult!.version });
for (const [dataset, r] of Object.entries(duckdbResult!.runs)) {
runs.push({
engine: 'duckdb',
dataset: dataset as BenchDatasetId,
query_label: QUERIES[dataset as BenchDatasetId].label,
warmup_ms: r.warmup_ms,
samples_ms: r.samples_ms,
distribution: summarizeSamples(r.samples_ms),
rows_returned: r.rows,
});
}
expect(runs.filter((r) => r.engine === 'duckdb')).toHaveLength(6);
}, 900_000);
it('records the engines BI-P104 names that nothing measured, with reasons', () => {
engines.push(
{
engine: 'trino',
available: false,
reason:
'a Trino is running on this host but belongs to another lane (st0re-verify-trino); ' +
'benchmarking through someone else’s container would measure their configuration',
},
{
engine: 'semantic_query',
available: false,
reason: 'MetricFlow has never been executed in 0pt (BI-P029 honest status)',
},
{ engine: 'superset', available: false, reason: 'no Superset deployment in the estate yet' },
{ engine: 'native_charts', available: false, reason: 'render-side; DES1GN owns it (BI-P051..P057)' },
{ engine: 'perspective', available: false, reason: 'not deployed' },
);
const evidence = buildEvidence({ host_note: 'x', engines, runs });
for (const e of evidence.not_exercised) expect(e.reason, e.engine).toBeTruthy();
expect(evidence.engines).toHaveLength(7);
});
it('reports every comparison as a range with its stability, never a bare figure', () => {
if (!duckdbResult) return;
for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) {
const a = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset)!;
const b = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset)!;
const sentence = describeComparison(compareDistributions(a.distribution, b.distribution), dataset);
expect(sentence).toMatch(/n=9\/9/);
expect(sentence).toMatch(/INDISTINGUISHABLE|x to /);
}
});
});