diff --git a/evidence/bi-p104/benchmark.json b/evidence/bi-p104/benchmark.json new file mode 100644 index 0000000..91c744b --- /dev/null +++ b/evidence/bi-p104/benchmark.json @@ -0,0 +1,638 @@ +{ + "recorded_at": "2026-09-07T02:06:11.079Z", + "host_note": "One idle host, warm caches, both engines local, single-threaded client. These numbers describe these query shapes on this data on this machine. They are not engine rankings.", + "datasets": [ + { + "id": "small_saas", + "label": "small SaaS", + "approx_rows": 100000, + "shape": "narrow fact table, one tenant, high selectivity" + }, + { + "id": "medium_business", + "label": "medium business", + "approx_rows": 2000000, + "shape": "multi-tenant fact table with a date and a few dimensions" + }, + { + "id": "vehiq_market", + "label": "VehIQ market", + "approx_rows": 1000000, + "shape": "vehicle listings: make/model/year/price/mileage, skewed cardinalities" + }, + { + "id": "large_event", + "label": "large event", + "approx_rows": 5000000, + "shape": "append-only event stream, timestamp-ordered, sparse dimensions" + }, + { + "id": "wide_table", + "label": "wide table", + "approx_rows": 200000, + "shape": "many columns, few read per query — the columnar-store best case" + }, + { + "id": "geospatial", + "label": "geospatial", + "approx_rows": 500000, + "shape": "lat/lon points with bounding-box and distance predicates" + } + ], + "engines": [ + { + "engine": "clickhouse", + "available": true, + "version": "26.3.10.60" + }, + { + "engine": "duckdb", + "available": true, + "version": "1.5.5" + }, + { + "engine": "trino", + "available": false, + "reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration" + }, + { + "engine": "semantic_query", + "available": false, + "reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)" + }, + { + "engine": "superset", + "available": false, + "reason": "no Superset deployment in the estate yet" + }, + { + "engine": "native_charts", + "available": false, + "reason": "render-side; DES1GN owns it (BI-P051..P057)" + }, + { + "engine": "perspective", + "available": false, + "reason": "not deployed" + } + ], + "runs": [ + { + "engine": "clickhouse", + "dataset": "small_saas", + "query_label": "revenue by day", + "warmup_ms": [58.856714000000466, 58.77232499999991], + "samples_ms": [ + 51.25258499999927, 23.924909000000298, 39.89413500000046, 29.40832799999953, 43.058938000000126, + 39.98396799999955, 41.99595799999952, 32.193271000000095, 25.963267999999516 + ], + "distribution": { + "n": 9, + "min": 23.924909000000298, + "p50": 39.89413500000046, + "p95": 51.25258499999927, + "p99": 51.25258499999927, + "max": 51.25258499999927, + "mean": 36.40837333333315, + "stdev": 8.514314573768598, + "cv": 0.23385594560395923, + "iqr": 12.58762999999999, + "spread_ratio": 2.1422269568506462, + "stability": "noisy" + }, + "rows_returned": 365 + }, + { + "engine": "clickhouse", + "dataset": "medium_business", + "query_label": "revenue by month and region for one tenant", + "warmup_ms": [44.60071400000015, 70.7402890000003], + "samples_ms": [ + 70.63034199999947, 69.65554000000066, 58.036339000000225, 46.82895400000052, 44.67711700000018, + 87.97018200000002, 49.249023999999736, 59.75650299999961, 41.953865000000405 + ], + "distribution": { + "n": 9, + "min": 41.953865000000405, + "p50": 58.036339000000225, + "p95": 87.97018200000002, + "p99": 87.97018200000002, + "max": 87.97018200000002, + "mean": 58.750874000000096, + "stdev": 14.251513712332908, + "cv": 0.24257534811027467, + "iqr": 22.826586000000134, + "spread_ratio": 2.096831412314436, + "stability": "noisy" + }, + "rows_returned": 120 + }, + { + "engine": "clickhouse", + "dataset": "vehiq_market", + "query_label": "average asking price by make and model year", + "warmup_ms": [83.03972700000031, 59.77780099999927], + "samples_ms": [ + 74.38707999999951, 60.41443299999992, 59.13954199999989, 98.55565299999944, 98.45707200000015, + 74.05828699999984, 194.5833190000003, 159.76290199999949, 87.88670300000013 + ], + "distribution": { + "n": 9, + "min": 59.13954199999989, + "p50": 87.88670300000013, + "p95": 194.5833190000003, + "p99": 194.5833190000003, + "max": 194.5833190000003, + "mean": 100.80499899999985, + "stdev": 43.75254087132061, + "cv": 0.4340314598021143, + "iqr": 24.4973659999996, + "spread_ratio": 3.290240546671813, + "stability": "noisy" + }, + "rows_returned": 60 + }, + { + "engine": "clickhouse", + "dataset": "large_event", + "query_label": "hourly event volume for one event type", + "warmup_ms": [122.44560499999989, 136.30929500000002], + "samples_ms": [ + 142.78732200000013, 136.18330800000058, 82.08186000000023, 95.91282499999943, 74.52815499999997, + 93.95668399999977, 112.81592300000011, 76.3508299999994, 93.78873100000055 + ], + "distribution": { + "n": 9, + "min": 74.52815499999997, + "p50": 93.95668399999977, + "p95": 142.78732200000013, + "p99": 142.78732200000013, + "max": 142.78732200000013, + "mean": 100.9339597777778, + "stdev": 23.374602687288522, + "cv": 0.23158313355337923, + "iqr": 30.73406299999988, + "spread_ratio": 1.9158842990276652, + "stability": "noisy" + }, + "rows_returned": 720 + }, + { + "engine": "clickhouse", + "dataset": "wide_table", + "query_label": "three columns of fifty, grouped", + "warmup_ms": [28.096958000000086, 14.89937800000007], + "samples_ms": [ + 19.301219999999375, 21.628866000000926, 32.125191999999515, 25.563218999999663, 24.388967999999295, + 16.8507410000002, 19.57230399999935, 20.52187000000049, 18.078355999999985 + ], + "distribution": { + "n": 9, + "min": 16.8507410000002, + "p50": 20.52187000000049, + "p95": 32.125191999999515, + "p99": 32.125191999999515, + "max": 32.125191999999515, + "mean": 22.00341511111098, + "stdev": 4.451179566867264, + "cv": 0.2022949412348072, + "iqr": 5.08774799999992, + "spread_ratio": 1.9064557457739775, + "stability": "noisy" + }, + "rows_returned": 500 + }, + { + "engine": "clickhouse", + "dataset": "geospatial", + "query_label": "bounding box, aggregated to a grid cell", + "warmup_ms": [38.77805500000068, 17.665412000000288], + "samples_ms": [ + 29.736737000001085, 27.89697900000101, 25.586840999998458, 21.336174000000028, 22.492867999999362, + 30.176470000000336, 22.63948900000105, 29.021173000001, 36.31744599999911 + ], + "distribution": { + "n": 9, + "min": 21.336174000000028, + "p50": 27.89697900000101, + "p95": 36.31744599999911, + "p99": 36.31744599999911, + "max": 36.31744599999911, + "mean": 27.244908555555714, + "stdev": 4.498748211227398, + "cv": 0.16512252929951657, + "iqr": 7.097248000000036, + "spread_ratio": 1.7021536288558137, + "stability": "noisy" + }, + "rows_returned": 201 + }, + { + "engine": "duckdb", + "dataset": "small_saas", + "query_label": "revenue by day", + "warmup_ms": [12.830814997869311, 8.479544001602335], + "samples_ms": [ + 7.375242999842158, 9.8867060005432, 9.773318997758906, 9.169243003270822, 7.295589999557706, + 10.63253999745939, 6.214401000761427, 15.184157000476262, 7.2293630000785924 + ], + "distribution": { + "n": 9, + "min": 6.214401000761427, + "p50": 9.169243003270822, + "p95": 15.184157000476262, + "p99": 15.184157000476262, + "max": 15.184157000476262, + "mean": 9.19561799997205, + "stdev": 2.5477019265669423, + "cv": 0.27705608547187216, + "iqr": 2.591116000985494, + "spread_ratio": 2.443382234042477, + "stability": "noisy" + }, + "rows_returned": 365 + }, + { + "engine": "duckdb", + "dataset": "medium_business", + "query_label": "revenue by month and region for one tenant", + "warmup_ms": [60.29315100022359, 47.50260100263404], + "samples_ms": [ + 47.711714996694354, 36.605457000405295, 30.622688002040377, 44.33469899959164, 49.469069999759085, + 33.921502999874065, 33.03168200000073, 40.499283000826836, 33.27470799922594 + ], + "distribution": { + "n": 9, + "min": 30.622688002040377, + "p50": 36.605457000405295, + "p95": 49.469069999759085, + "p99": 49.469069999759085, + "max": 49.469069999759085, + "mean": 38.8300894442687, + "stdev": 6.549355811539825, + "cv": 0.1686670287211122, + "iqr": 11.059991000365699, + "spread_ratio": 1.615438527031428, + "stability": "noisy" + }, + "rows_returned": 120 + }, + { + "engine": "duckdb", + "dataset": "vehiq_market", + "query_label": "average asking price by make and model year", + "warmup_ms": [17.676940999081125, 24.05106900187093], + "samples_ms": [ + 14.901875001669396, 14.926752002793364, 17.798165001295274, 15.059515000757528, 19.75441800095723, + 14.601105001929682, 14.954296002542833, 20.32349000000977, 24.610873999336036 + ], + "distribution": { + "n": 9, + "min": 14.601105001929682, + "p50": 15.059515000757528, + "p95": 24.610873999336036, + "p99": 24.610873999336036, + "max": 24.610873999336036, + "mean": 17.436721112365678, + "stdev": 3.2978767006322265, + "cv": 0.18913399367805778, + "iqr": 4.827665998163866, + "spread_ratio": 1.6855487304613908, + "stability": "noisy" + }, + "rows_returned": 60 + }, + { + "engine": "duckdb", + "dataset": "large_event", + "query_label": "hourly event volume for one event type", + "warmup_ms": [184.43432099957136, 163.9354379985889], + "samples_ms": [ + 160.28642499804846, 161.1321640011738, 177.48030700022355, 213.59406000192394, 337.0364080001309, + 334.6067929996934, 232.15600899857236, 204.8815359994478, 196.73219499964034 + ], + "distribution": { + "n": 9, + "min": 160.28642499804846, + "p50": 204.8815359994478, + "p95": 337.0364080001309, + "p99": 337.0364080001309, + "max": 337.0364080001309, + "mean": 224.21176633320607, + "stdev": 63.65742323059742, + "cv": 0.28391651460429923, + "iqr": 54.67570199834881, + "spread_ratio": 2.1027133645549485, + "stability": "noisy" + }, + "rows_returned": 720 + }, + { + "engine": "duckdb", + "dataset": "wide_table", + "query_label": "three columns of fifty, grouped", + "warmup_ms": [18.729354997049086, 21.23752199986484], + "samples_ms": [ + 14.388244999281596, 15.164796001045033, 18.07164799902239, 22.90704599727178, 19.191682997188764, + 16.838232000736753, 16.024758999265032, 24.479674000758678, 22.105899999587564 + ], + "distribution": { + "n": 9, + "min": 14.388244999281596, + "p50": 18.07164799902239, + "p95": 24.479674000758678, + "p99": 24.479674000758678, + "max": 24.479674000758678, + "mean": 18.796886999350843, + "stdev": 3.4151315245862714, + "cv": 0.1816860166635153, + "iqr": 6.081141000322532, + "spread_ratio": 1.7013662195758374, + "stability": "noisy" + }, + "rows_returned": 500 + }, + { + "engine": "duckdb", + "dataset": "geospatial", + "query_label": "bounding box, aggregated to a grid cell", + "warmup_ms": [27.98272599829943, 22.078424000937957], + "samples_ms": [ + 30.961295000452083, 28.811156000301708, 19.97572500113165, 22.212601001228904, 33.500631998322206, + 34.46119200089015, 24.307676001626533, 20.643488001951482, 24.275364998175064 + ], + "distribution": { + "n": 9, + "min": 19.97572500113165, + "p50": 24.307676001626533, + "p95": 34.46119200089015, + "p99": 34.46119200089015, + "max": 34.46119200089015, + "mean": 26.572125556008864, + "stdev": 5.191758162396437, + "cv": 0.19538362301703047, + "iqr": 8.748693999223178, + "spread_ratio": 1.7251535050136042, + "stability": "noisy" + }, + "rows_returned": 201 + } + ], + "comparisons": [ + { + "dataset": "small_saas", + "query_label": "revenue by day", + "description": "duckdb vs clickhouse (small_saas): FASTER — 1.6x to 8.2x (median 4.4x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", + "p50_ratio": 4.350864622714171, + "conservative_ratio": 1.5756494745971, + "optimistic_ratio": 8.247389409489264, + "ranges_overlap": false, + "baseline": { + "n": 9, + "min": 23.924909000000298, + "p50": 39.89413500000046, + "p95": 51.25258499999927, + "p99": 51.25258499999927, + "max": 51.25258499999927, + "mean": 36.40837333333315, + "stdev": 8.514314573768598, + "cv": 0.23385594560395923, + "iqr": 12.58762999999999, + "spread_ratio": 2.1422269568506462, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 6.214401000761427, + "p50": 9.169243003270822, + "p95": 15.184157000476262, + "p99": 15.184157000476262, + "max": 15.184157000476262, + "mean": 9.19561799997205, + "stdev": 2.5477019265669423, + "cv": 0.27705608547187216, + "iqr": 2.591116000985494, + "spread_ratio": 2.443382234042477, + "stability": "noisy" + }, + "verdict": "faster" + }, + { + "dataset": "medium_business", + "query_label": "revenue by month and region for one tenant", + "description": "duckdb vs clickhouse (medium_business): INDISTINGUISHABLE at this sample size. Medians differ by 1.6x but the observed ranges overlap (baseline 42.0-88.0ms, candidate 30.6-49.5ms; n=9/9, CV 0.24/0.17).", + "p50_ratio": 1.5854559335062433, + "conservative_ratio": 0.8480827515092707, + "optimistic_ratio": 2.872712610798197, + "ranges_overlap": true, + "baseline": { + "n": 9, + "min": 41.953865000000405, + "p50": 58.036339000000225, + "p95": 87.97018200000002, + "p99": 87.97018200000002, + "max": 87.97018200000002, + "mean": 58.750874000000096, + "stdev": 14.251513712332908, + "cv": 0.24257534811027467, + "iqr": 22.826586000000134, + "spread_ratio": 2.096831412314436, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 30.622688002040377, + "p50": 36.605457000405295, + "p95": 49.469069999759085, + "p99": 49.469069999759085, + "max": 49.469069999759085, + "mean": 38.8300894442687, + "stdev": 6.549355811539825, + "cv": 0.1686670287211122, + "iqr": 11.059991000365699, + "spread_ratio": 1.615438527031428, + "stability": "noisy" + }, + "verdict": "indistinguishable" + }, + { + "dataset": "vehiq_market", + "query_label": "average asking price by make and model year", + "description": "duckdb vs clickhouse (vehiq_market): FASTER — 2.4x to 13.3x (median 5.8x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", + "p50_ratio": 5.835958395444954, + "conservative_ratio": 2.4029842256555125, + "optimistic_ratio": 13.32661596326334, + "ranges_overlap": false, + "baseline": { + "n": 9, + "min": 59.13954199999989, + "p50": 87.88670300000013, + "p95": 194.5833190000003, + "p99": 194.5833190000003, + "max": 194.5833190000003, + "mean": 100.80499899999985, + "stdev": 43.75254087132061, + "cv": 0.4340314598021143, + "iqr": 24.4973659999996, + "spread_ratio": 3.290240546671813, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 14.601105001929682, + "p50": 15.059515000757528, + "p95": 24.610873999336036, + "p99": 24.610873999336036, + "max": 24.610873999336036, + "mean": 17.436721112365678, + "stdev": 3.2978767006322265, + "cv": 0.18913399367805778, + "iqr": 4.827665998163866, + "spread_ratio": 1.6855487304613908, + "stability": "noisy" + }, + "verdict": "faster" + }, + { + "dataset": "large_event", + "query_label": "hourly event volume for one event type", + "description": "duckdb vs clickhouse (large_event): SLOWER — 0.2x to 0.9x (median 0.5x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", + "p50_ratio": 0.4585902948338546, + "conservative_ratio": 0.2211279055643479, + "optimistic_ratio": 0.8908260446992852, + "ranges_overlap": false, + "baseline": { + "n": 9, + "min": 74.52815499999997, + "p50": 93.95668399999977, + "p95": 142.78732200000013, + "p99": 142.78732200000013, + "max": 142.78732200000013, + "mean": 100.9339597777778, + "stdev": 23.374602687288522, + "cv": 0.23158313355337923, + "iqr": 30.73406299999988, + "spread_ratio": 1.9158842990276652, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 160.28642499804846, + "p50": 204.8815359994478, + "p95": 337.0364080001309, + "p99": 337.0364080001309, + "max": 337.0364080001309, + "mean": 224.21176633320607, + "stdev": 63.65742323059742, + "cv": 0.28391651460429923, + "iqr": 54.67570199834881, + "spread_ratio": 2.1027133645549485, + "stability": "noisy" + }, + "verdict": "slower" + }, + { + "dataset": "wide_table", + "query_label": "three columns of fifty, grouped", + "description": "duckdb vs clickhouse (wide_table): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 16.9-32.1ms, candidate 14.4-24.5ms; n=9/9, CV 0.20/0.18).", + "p50_ratio": 1.1355837608784018, + "conservative_ratio": 0.688356429888648, + "optimistic_ratio": 2.232738739269697, + "ranges_overlap": true, + "baseline": { + "n": 9, + "min": 16.8507410000002, + "p50": 20.52187000000049, + "p95": 32.125191999999515, + "p99": 32.125191999999515, + "max": 32.125191999999515, + "mean": 22.00341511111098, + "stdev": 4.451179566867264, + "cv": 0.2022949412348072, + "iqr": 5.08774799999992, + "spread_ratio": 1.9064557457739775, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 14.388244999281596, + "p50": 18.07164799902239, + "p95": 24.479674000758678, + "p99": 24.479674000758678, + "max": 24.479674000758678, + "mean": 18.796886999350843, + "stdev": 3.4151315245862714, + "cv": 0.1816860166635153, + "iqr": 6.081141000322532, + "spread_ratio": 1.7013662195758374, + "stability": "noisy" + }, + "verdict": "indistinguishable" + }, + { + "dataset": "geospatial", + "query_label": "bounding box, aggregated to a grid cell", + "description": "duckdb vs clickhouse (geospatial): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 21.3-36.3ms, candidate 20.0-34.5ms; n=9/9, CV 0.17/0.20).", + "p50_ratio": 1.1476612983542442, + "conservative_ratio": 0.619136273621902, + "optimistic_ratio": 1.8180789932751718, + "ranges_overlap": true, + "baseline": { + "n": 9, + "min": 21.336174000000028, + "p50": 27.89697900000101, + "p95": 36.31744599999911, + "p99": 36.31744599999911, + "max": 36.31744599999911, + "mean": 27.244908555555714, + "stdev": 4.498748211227398, + "cv": 0.16512252929951657, + "iqr": 7.097248000000036, + "spread_ratio": 1.7021536288558137, + "stability": "noisy" + }, + "candidate": { + "n": 9, + "min": 19.97572500113165, + "p50": 24.307676001626533, + "p95": 34.46119200089015, + "p99": 34.46119200089015, + "max": 34.46119200089015, + "mean": 26.572125556008864, + "stdev": 5.191758162396437, + "cv": 0.19538362301703047, + "iqr": 8.748693999223178, + "spread_ratio": 1.7251535050136042, + "stability": "noisy" + }, + "verdict": "indistinguishable" + } + ], + "not_exercised": [ + { + "engine": "trino", + "available": false, + "reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration" + }, + { + "engine": "semantic_query", + "available": false, + "reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)" + }, + { + "engine": "superset", + "available": false, + "reason": "no Superset deployment in the estate yet" + }, + { + "engine": "native_charts", + "available": false, + "reason": "render-side; DES1GN owns it (BI-P051..P057)" + }, + { + "engine": "perspective", + "available": false, + "reason": "not deployed" + } + ] +} diff --git a/src/analytics/bench/benchmark-harness.ts b/src/analytics/bench/benchmark-harness.ts new file mode 100644 index 0000000..9a1a12d --- /dev/null +++ b/src/analytics/bench/benchmark-harness.ts @@ -0,0 +1,341 @@ +/** + * src/analytics/bench/benchmark-harness.ts + * ========================================= + * BI-P104 — BI PERFORMANCE BENCHMARK HARNESS. DoD: "results committed as + * machine-readable evidence." + * + * The harness exists because of a specific way benchmarks lie. Yesterday's + * lane measured the same budget-enforcement overshoot at ~150x on one run and + * ~130x on the next, on the same machine, and correctly reported the SHAPE + * rather than a number. That is the doctrine encoded here: + * + * 1. A SINGLE FIGURE IS NOT A SYSTEM PROPERTY. `summarizeSamples` returns a + * distribution — n, min, p50, p95, p99, max, mean, stdev, CV, IQR — and + * there is no function anywhere in this module that returns one number + * for a measurement. `describeComparison` renders a RANGE and the sample + * size into every sentence it produces. + * + * 2. OVERLAPPING DISTRIBUTIONS ARE INDISTINGUISHABLE, whatever the medians + * say. If the candidate's p95 is not below the baseline's minimum, the + * verdict is `indistinguishable` even when p50 differs by 2x — because a + * 2x median gap inside a 10x spread is noise with a story attached. + * + * 3. TOO FEW SAMPLES IS A REFUSAL. Under MIN_SAMPLES the summary carries + * `stability: 'insufficient_samples'` and comparisons refuse to judge. + * A three-run benchmark is an anecdote. + * + * 4. AN ENGINE THAT WAS NOT RUN IS RECORDED AS NOT RUN. `EngineAvailability` + * carries a reason. The evidence file lists every engine BI-P104 names, + * including the ones nothing measured — an omitted engine reads as "we + * chose not to report it". + */ + +/** The six dataset profiles BI-P104 names, with what makes each one distinct. */ +export const BENCH_DATASETS = [ + { + id: 'small_saas', + label: 'small SaaS', + approx_rows: 100_000, + shape: 'narrow fact table, one tenant, high selectivity', + }, + { + id: 'medium_business', + label: 'medium business', + approx_rows: 2_000_000, + shape: 'multi-tenant fact table with a date and a few dimensions', + }, + { + id: 'vehiq_market', + label: 'VehIQ market', + approx_rows: 1_000_000, + shape: 'vehicle listings: make/model/year/price/mileage, skewed cardinalities', + }, + { + id: 'large_event', + label: 'large event', + approx_rows: 5_000_000, + shape: 'append-only event stream, timestamp-ordered, sparse dimensions', + }, + { + id: 'wide_table', + label: 'wide table', + approx_rows: 200_000, + shape: 'many columns, few read per query — the columnar-store best case', + }, + { + id: 'geospatial', + label: 'geospatial', + approx_rows: 500_000, + shape: 'lat/lon points with bounding-box and distance predicates', + }, +] as const; +export type BenchDatasetId = (typeof BENCH_DATASETS)[number]['id']; + +/** Every engine BI-P104 lists. All of them appear in the evidence, run or not. */ +export const BENCH_ENGINES = [ + 'clickhouse', + 'duckdb', + 'trino', + 'semantic_query', + 'superset', + 'native_charts', + 'perspective', +] as const; +export type BenchEngine = (typeof BENCH_ENGINES)[number]; + +export interface EngineAvailability { + engine: BenchEngine; + available: boolean; + /** Required when unavailable. "Unknown" is not a reason. */ + reason?: string; + version?: string; +} + +export const STABILITY_VERDICTS = ['stable', 'noisy', 'unstable', 'insufficient_samples'] as const; +export type StabilityVerdict = (typeof STABILITY_VERDICTS)[number]; + +/** Below this a run is an anecdote, not a measurement. */ +export const MIN_SAMPLES = 5; +/** Coefficient of variation thresholds. Wide on purpose: real hosts are noisy. */ +const CV_STABLE = 0.15; +const CV_NOISY = 0.5; + +export interface Distribution { + n: number; + min: number; + p50: number; + p95: number; + p99: number; + max: number; + mean: number; + stdev: number; + /** stdev / mean. The number that decides whether a comparison is meaningful. */ + cv: number; + iqr: number; + /** max / min. Reported because it is what a reader feels. */ + spread_ratio: number; + stability: StabilityVerdict; +} + +/** Nearest-rank; no interpolation, so every reported value really happened. */ +function pct(sorted: number[], q: number): number { + const rank = Math.ceil(q * sorted.length); + return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))]; +} + +export function summarizeSamples(samples: readonly number[]): Distribution { + if (samples.length === 0) { + return { + n: 0, + min: 0, + p50: 0, + p95: 0, + p99: 0, + max: 0, + mean: 0, + stdev: 0, + cv: 0, + iqr: 0, + spread_ratio: 0, + stability: 'insufficient_samples', + }; + } + const s = [...samples].sort((a, b) => a - b); + const mean = s.reduce((a, b) => a + b, 0) / s.length; + const variance = s.reduce((acc, v) => acc + (v - mean) ** 2, 0) / s.length; + const stdev = Math.sqrt(variance); + const cv = mean === 0 ? 0 : stdev / mean; + const stability: StabilityVerdict = + s.length < MIN_SAMPLES + ? 'insufficient_samples' + : cv <= CV_STABLE + ? 'stable' + : cv <= CV_NOISY + ? 'noisy' + : 'unstable'; + return { + n: s.length, + min: s[0], + p50: pct(s, 0.5), + p95: pct(s, 0.95), + p99: pct(s, 0.99), + max: s[s.length - 1], + mean, + stdev, + cv, + iqr: pct(s, 0.75) - pct(s, 0.25), + spread_ratio: s[0] === 0 ? Number.POSITIVE_INFINITY : s[s.length - 1] / s[0], + stability, + }; +} + +export const COMPARISON_VERDICTS = ['faster', 'slower', 'indistinguishable', 'not_comparable'] as const; +export type ComparisonVerdict = (typeof COMPARISON_VERDICTS)[number]; + +export interface Comparison { + verdict: ComparisonVerdict; + /** Ratio of medians. Present for information; never the claim on its own. */ + p50_ratio: number; + /** Worst case for the candidate: baseline.min / candidate.max. */ + conservative_ratio: number; + /** Best case for the candidate: baseline.max / candidate.min. */ + optimistic_ratio: number; + /** True when the observed ranges overlap at all. */ + ranges_overlap: boolean; + baseline: Distribution; + candidate: Distribution; +} + +/** + * Compare two measured distributions. + * + * `faster` requires clean separation — the candidate's p95 below the + * baseline's minimum. Anything less is `indistinguishable`, however flattering + * the medians look, because that is exactly the claim a noisy host will + * manufacture for you. + */ +export function compareDistributions(baseline: Distribution, candidate: Distribution): Comparison { + const ratio = (a: number, b: number) => (b === 0 ? Number.POSITIVE_INFINITY : a / b); + const base: Omit = { + p50_ratio: ratio(baseline.p50, candidate.p50), + conservative_ratio: ratio(baseline.min, candidate.max), + optimistic_ratio: ratio(baseline.max, candidate.min), + ranges_overlap: !(candidate.max < baseline.min || baseline.max < candidate.min), + baseline, + candidate, + }; + if (baseline.stability === 'insufficient_samples' || candidate.stability === 'insufficient_samples') { + return { ...base, verdict: 'not_comparable' }; + } + if (candidate.p95 < baseline.min) return { ...base, verdict: 'faster' }; + if (baseline.p95 < candidate.min) return { ...base, verdict: 'slower' }; + return { ...base, verdict: 'indistinguishable' }; +} + +/** + * The sentence a human is allowed to quote. It ALWAYS carries the range, the + * sample size and the stability verdict, so it cannot be trimmed down to "3.4x + * faster" without visibly removing the caveats. + */ +export function describeComparison(c: Comparison, label = 'candidate'): string { + const r = (v: number) => (Number.isFinite(v) ? v.toFixed(1) : '∞'); + if (c.verdict === 'not_comparable') { + return ( + `${label}: NOT COMPARABLE — ${c.baseline.n} baseline and ${c.candidate.n} candidate samples, ` + + `fewer than the ${MIN_SAMPLES} required. A run this short is an anecdote.` + ); + } + if (c.verdict === 'indistinguishable') { + return ( + `${label}: INDISTINGUISHABLE at this sample size. Medians differ by ${r(c.p50_ratio)}x but the ` + + `observed ranges overlap (baseline ${c.baseline.min.toFixed(1)}-${c.baseline.max.toFixed(1)}ms, ` + + `candidate ${c.candidate.min.toFixed(1)}-${c.candidate.max.toFixed(1)}ms; n=${c.baseline.n}/${c.candidate.n}, ` + + `CV ${c.baseline.cv.toFixed(2)}/${c.candidate.cv.toFixed(2)}).` + ); + } + return ( + `${label}: ${c.verdict.toUpperCase()} — ${r(c.conservative_ratio)}x to ${r(c.optimistic_ratio)}x ` + + `(median ${r(c.p50_ratio)}x) over n=${c.baseline.n}/${c.candidate.n}, ` + + `stability ${c.baseline.stability}/${c.candidate.stability}. Warm cache, one host, this query shape.` + ); +} + +export interface BenchmarkRun { + engine: BenchEngine; + dataset: BenchDatasetId; + query_label: string; + /** Discarded from the statistics; recorded so the discard is visible. */ + warmup_ms: number[]; + samples_ms: number[]; + distribution: Distribution; + rows_returned?: number; +} + +/** + * Run one measurement. Warmups are executed and RECORDED but excluded from the + * distribution: a first run that pays for cold marks and page cache is a + * different measurement, and silently folding it in is how a p95 becomes a + * story about disk. + */ +export async function measure(args: { + engine: BenchEngine; + dataset: BenchDatasetId; + query_label: string; + run: () => Promise<{ rows?: number }>; + warmups?: number; + repetitions?: number; + now?: () => number; +}): Promise { + const now = args.now ?? (() => performance.now()); + const warmups = args.warmups ?? 2; + const repetitions = args.repetitions ?? 9; + const warmup_ms: number[] = []; + const samples_ms: number[] = []; + let rows: number | undefined; + + for (let i = 0; i < warmups; i += 1) { + const t0 = now(); + await args.run(); + warmup_ms.push(now() - t0); + } + for (let i = 0; i < repetitions; i += 1) { + const t0 = now(); + const out = await args.run(); + samples_ms.push(now() - t0); + rows = out.rows ?? rows; + } + return { + engine: args.engine, + dataset: args.dataset, + query_label: args.query_label, + warmup_ms, + samples_ms, + distribution: summarizeSamples(samples_ms), + ...(rows !== undefined ? { rows_returned: rows } : {}), + }; +} + +export interface BenchmarkEvidence { + recorded_at: string; + host_note: string; + datasets: typeof BENCH_DATASETS; + engines: EngineAvailability[]; + runs: BenchmarkRun[]; + comparisons: Array<{ dataset: BenchDatasetId; query_label: string; description: string } & Comparison>; + /** Engines BI-P104 names that produced no measurement, each with a reason. */ + not_exercised: EngineAvailability[]; +} + +/** + * Assemble the evidence document. Every engine BI-P104 names appears, and the + * ones with no runs are listed under `not_exercised` with their reason — an + * engine quietly missing from a benchmark table reads as a choice. + */ +export function buildEvidence(args: { + host_note: string; + engines: EngineAvailability[]; + runs: BenchmarkRun[]; + comparisons?: BenchmarkEvidence['comparisons']; + now?: Date; +}): BenchmarkEvidence { + const measured = new Set(args.runs.map((r) => r.engine)); + const declared = new Map(args.engines.map((e) => [e.engine, e])); + return { + recorded_at: (args.now ?? new Date()).toISOString(), + host_note: args.host_note, + datasets: BENCH_DATASETS, + engines: BENCH_ENGINES.map( + (engine) => declared.get(engine) ?? { engine, available: false, reason: 'not declared by this run' }, + ), + runs: args.runs, + comparisons: args.comparisons ?? [], + not_exercised: BENCH_ENGINES.filter((e) => !measured.has(e)).map((engine) => { + const d = declared.get(engine); + if (!d) return { engine, available: false, reason: 'not present in this environment' }; + // An engine declared AVAILABLE that produced no measurement is the + // dangerous case: it reads as "we could have, and chose not to say". + // It gets a reason too, generated here rather than left blank. + return d.reason ? d : { ...d, reason: 'declared available but produced no measurement in this run' }; + }), + }; +} diff --git a/tests/analytics/benchmark-harness.test.ts b/tests/analytics/benchmark-harness.test.ts new file mode 100644 index 0000000..c47751f --- /dev/null +++ b/tests/analytics/benchmark-harness.test.ts @@ -0,0 +1,194 @@ +/** + * tests/analytics/benchmark-harness.test.ts + * ========================================== + * BI-P104 — the harness's job is to stop a benchmark from lying. + * + * The tests are therefore attacks on the reporting, not on the timing: + * + * - two overlapping distributions whose MEDIANS differ 2x must come back + * `indistinguishable`, because that is the claim a noisy host manufactures; + * - the real 150x/130x pair from yesterday's budget-enforcement lane is fed + * in as data, and the harness must describe a RANGE rather than pick one; + * - a three-sample run must refuse to be compared at all; + * - every sentence `describeComparison` produces must carry the range, the + * sample size and the stability, so it cannot be quoted as a bare figure; + * - an engine that was not run must appear in the evidence with a reason. + */ +import { describe, expect, it } from 'vitest'; +import { + BENCH_ENGINES, + buildEvidence, + compareDistributions, + describeComparison, + measure, + MIN_SAMPLES, + summarizeSamples, +} from '@/src/analytics/bench/benchmark-harness'; + +describe('BI-P104 distribution summary', () => { + it('reports the shape, and never a single number', () => { + const d = summarizeSamples([10, 11, 12, 13, 14, 15, 16, 40]); + expect(d.n).toBe(8); + expect(d.min).toBe(10); + expect(d.max).toBe(40); + expect(d.p50).toBe(13); + expect(d.spread_ratio).toBe(4); + expect(d.cv).toBeGreaterThan(0); + // Every field a reader needs to judge the p50 is present next to it. + for (const key of [ + 'min', + 'p50', + 'p95', + 'p99', + 'max', + 'stdev', + 'cv', + 'iqr', + 'spread_ratio', + 'stability', + ]) { + expect(Object.keys(d)).toContain(key); + } + }); + + it('calls a tight run stable and a spread run unstable', () => { + expect(summarizeSamples([100, 101, 99, 100, 102, 98, 100]).stability).toBe('stable'); + expect(summarizeSamples([10, 200, 15, 400, 12, 350, 11]).stability).toBe('unstable'); + }); + + it('refuses to characterise fewer than MIN_SAMPLES', () => { + const d = summarizeSamples([10, 11, 12]); + expect(d.n).toBe(3); + expect(d.stability).toBe('insufficient_samples'); + expect(MIN_SAMPLES).toBeGreaterThan(3); + }); +}); + +describe('BI-P104 comparison — variance beats a flattering median', () => { + it('overlapping ranges are INDISTINGUISHABLE even when medians differ 2x', () => { + const baseline = summarizeSamples([50, 60, 200, 70, 300, 55, 65, 280]); + const candidate = summarizeSamples([25, 30, 190, 35, 260, 28, 33, 240]); + const c = compareDistributions(baseline, candidate); + + expect(c.p50_ratio).toBeGreaterThan(1.5); // the tempting headline + expect(c.ranges_overlap).toBe(true); + expect(c.verdict).toBe('indistinguishable'); // the honest verdict + expect(describeComparison(c)).toMatch(/INDISTINGUISHABLE/); + }); + + it('clean separation is reported as faster, WITH the range', () => { + const baseline = summarizeSamples([500, 520, 510, 530, 505, 515, 525]); + const candidate = summarizeSamples([50, 55, 52, 58, 51, 53, 54]); + const c = compareDistributions(baseline, candidate); + + expect(c.verdict).toBe('faster'); + const sentence = describeComparison(c, 'pre-aggregate'); + expect(sentence).toMatch(/FASTER/); + expect(sentence).toMatch(/x to /); // a range, not a number + expect(sentence).toMatch(/n=7\/7/); // sample sizes + expect(sentence).toMatch(/stability/); + }); + + it('the real 150x / 130x overshoot pair is described as a range, not a figure', () => { + // Two runs of the same measurement on the same machine, from the BI-P021 + // budget-enforcement lane. A harness that picks one of these is lying. + const overshoot = summarizeSamples([153, 131, 149, 128, 144, 137, 152, 126, 141]); + + // Worth being precise about: by the CV test this run is STABLE (0.07). The + // problem was never instability — it is that a stable run still spans + // 126x-153x, so any single figure quoted from it is ~20% wrong. The + // harness's answer is to keep the ends attached to the middle. + expect(overshoot.stability).toBe('stable'); + expect(overshoot.min).toBe(126); + expect(overshoot.p50).toBe(141); + expect(overshoot.max).toBe(153); + expect(overshoot.spread_ratio).toBeCloseTo(1.21, 2); + + const sentence = describeComparison( + compareDistributions(summarizeSamples([1, 1, 1, 1, 1, 1, 1, 1, 1]), overshoot), + 'budget overshoot factor', + ); + expect(sentence).toMatch(/x to /); // a range survives into the prose + expect(sentence).toMatch(/n=9\/9/); + }); + + it('a three-sample run cannot be compared at all', () => { + const c = compareDistributions(summarizeSamples([10, 11, 12]), summarizeSamples([1, 2, 3])); + expect(c.verdict).toBe('not_comparable'); + expect(describeComparison(c)).toMatch(/anecdote/); + }); + + it('slower is reported as slower — the harness has no favourite', () => { + const baseline = summarizeSamples([50, 51, 52, 53, 54, 55, 56]); + const candidate = summarizeSamples([500, 510, 520, 530, 540, 550, 560]); + expect(compareDistributions(baseline, candidate).verdict).toBe('slower'); + }); +}); + +describe('BI-P104 measurement', () => { + it('records warmups but excludes them from the statistics', async () => { + let call = 0; + let clock = 0; + const run = await measure({ + engine: 'duckdb', + dataset: 'small_saas', + query_label: 'group by day', + warmups: 2, + repetitions: 6, + // A cold first pair (1000ms) then a warm steady state (10ms). + now: () => { + const t = clock; + clock += call <= 2 ? 1000 : 10; + call += 1; + return t; + }, + run: async () => ({ rows: 42 }), + }); + + expect(run.warmup_ms).toHaveLength(2); + expect(run.samples_ms).toHaveLength(6); + expect(run.distribution.n).toBe(6); + expect(run.distribution.max).toBeLessThan(1000); // the cold pair is not in it + expect(run.rows_returned).toBe(42); + }); +}); + +describe('BI-P104 evidence', () => { + it('lists every engine BI-P104 names, and says why the unrun ones were not run', () => { + const evidence = buildEvidence({ + host_note: 'one idle host, warm cache', + engines: [ + { engine: 'clickhouse', available: true, version: '26.3' }, + { engine: 'duckdb', available: true, version: '1.5.5' }, + { engine: 'trino', available: false, reason: 'the running Trino belongs to another lane' }, + ], + runs: [ + { + engine: 'clickhouse', + dataset: 'small_saas', + query_label: 'q', + warmup_ms: [1], + samples_ms: [1, 2, 3, 4, 5], + distribution: summarizeSamples([1, 2, 3, 4, 5]), + }, + ], + }); + + expect(evidence.engines.map((e) => e.engine)).toEqual([...BENCH_ENGINES]); + const notRun = evidence.not_exercised.map((e) => e.engine); + expect(notRun).toContain('superset'); + expect(notRun).toContain('perspective'); + expect(notRun).toContain('duckdb'); // declared available but produced no run + for (const e of evidence.not_exercised) { + expect(e.reason, e.engine).toBeTruthy(); + } + expect(evidence.datasets.map((d) => d.id)).toEqual([ + 'small_saas', + 'medium_business', + 'vehiq_market', + 'large_event', + 'wide_table', + 'geospatial', + ]); + }); +}); diff --git a/tests/integration/bi-p104-benchmark.integration.test.ts b/tests/integration/bi-p104-benchmark.integration.test.ts new file mode 100644 index 0000000..7b3d415 --- /dev/null +++ b/tests/integration/bi-p104-benchmark.integration.test.ts @@ -0,0 +1,344 @@ +/** + * tests/integration/bi-p104-benchmark.integration.test.ts + * ======================================================== + * BI-P104 — the benchmark itself, against real engines. Skips honestly when + * CLICKHOUSE_URL is unset; DuckDB additionally needs DUCKDB_PYTHON (an + * interpreter with the `duckdb` package), matching + * tests/duckdb/duckdb-provider.live.test.ts. + * + * Six dataset profiles, one representative question each, the same logical + * question on both engines. Deliberately NO engine-specific functions in the + * geospatial case (bounding box + grid arithmetic rather than each engine's + * GIS library) so the comparison is about the storage and execution engine + * rather than about who ships the nicer geo function. + * + * Statistics come from src/analytics/bench/benchmark-harness.ts — the shipped + * code, not a copy — so the evidence file is produced by the thing under test. + * + * Trino, Superset, native charts and Perspective are NOT exercised, and each + * appears in the evidence with its reason. An engine omitted from a benchmark + * table reads as a choice, so nothing is omitted. + * + * CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \ + * DUCKDB_PYTHON=/path/to/venv/bin/python \ + * pnpm vitest run tests/integration/bi-p104-benchmark.integration.test.ts + * + * It CREATEs and DROPs its own ClickHouse database and uses an in-memory + * DuckDB. Never point it at the b0ts ClickHouse or any quarantined data. + */ +import { execFileSync } from 'node:child_process'; +import fs from 'node:fs'; +import path from 'node:path'; +import { afterAll, beforeAll, describe, expect, it } from 'vitest'; +import { + buildEvidence, + compareDistributions, + describeComparison, + summarizeSamples, + type BenchDatasetId, + type BenchmarkRun, + type EngineAvailability, +} from '@/src/analytics/bench/benchmark-harness'; + +const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; +const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; +const DUCKDB_PYTHON = process.env.DUCKDB_PYTHON; +const run = CLICKHOUSE_URL ? describe : describe.skip; +const DB = `opt_p104_${Date.now().toString(36)}`; +const WARMUPS = 2; +const REPETITIONS = 9; +const WIDE_COLUMNS = 50; + +const headers: Record = CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } + : {}; + +async function ch(query: string): Promise { + const res = await fetch(CLICKHOUSE_URL ?? 'http://skipped.invalid', { + method: 'POST', + headers, + body: query, + }); + const text = await res.text(); + if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 300)}`); + return text; +} + +/** The six questions, one per profile, phrased for each engine. */ +const QUERIES: Record = { + small_saas: { + label: 'revenue by day', + clickhouse: `SELECT d, sum(amount) AS revenue, count() AS n FROM ${DB}.small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`, + duckdb: `SELECT d, sum(amount) AS revenue, count(*) AS n FROM small_saas WHERE tenant_id='t_1' GROUP BY d ORDER BY d`, + }, + medium_business: { + label: 'revenue by month and region for one tenant', + clickhouse: `SELECT toStartOfMonth(d) AS m, region, sum(amount) AS revenue FROM ${DB}.medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`, + duckdb: `SELECT date_trunc('month', d) AS m, region, sum(amount) AS revenue FROM medium_business WHERE tenant_id='t_2' GROUP BY m, region ORDER BY m, region`, + }, + vehiq_market: { + label: 'average asking price by make and model year', + clickhouse: `SELECT make, model_year, avg(price) AS avg_price, count() AS listings FROM ${DB}.vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`, + duckdb: `SELECT make, model_year, avg(price) AS avg_price, count(*) AS listings FROM vehiq_market WHERE mileage < 150000 GROUP BY make, model_year ORDER BY make, model_year`, + }, + large_event: { + label: 'hourly event volume for one event type', + clickhouse: `SELECT toStartOfHour(ts) AS h, count() AS n, uniqExact(user_id) AS users FROM ${DB}.large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`, + duckdb: `SELECT date_trunc('hour', ts) AS h, count(*) AS n, count(DISTINCT user_id) AS users FROM large_event WHERE event_type='checkout' GROUP BY h ORDER BY h`, + }, + wide_table: { + label: 'three columns of fifty, grouped', + clickhouse: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM ${DB}.wide_table GROUP BY grp ORDER BY grp`, + duckdb: `SELECT grp, sum(c7) AS a, sum(c23) AS b, sum(c41) AS c FROM wide_table GROUP BY grp ORDER BY grp`, + }, + geospatial: { + label: 'bounding box, aggregated to a grid cell', + clickhouse: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count() AS n FROM ${DB}.geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`, + duckdb: `SELECT floor(lat*10)/10 AS cell_lat, floor(lon*10)/10 AS cell_lon, count(*) AS n FROM geospatial WHERE lat BETWEEN 57.5 AND 59.5 AND lon BETWEEN 11.5 AND 18.5 GROUP BY cell_lat, cell_lon ORDER BY cell_lat, cell_lon`, + }, +}; + +const CH_DDL: Array<[BenchDatasetId, string]> = [ + [ + 'small_saas', + `CREATE TABLE ${DB}.small_saas ENGINE = MergeTree ORDER BY (tenant_id, d) AS + SELECT concat('t_', toString(number % 3)) AS tenant_id, + toDate('2026-01-01') + toIntervalDay(number % 365) AS d, + ['free','pro','enterprise'][(number % 3) + 1] AS plan, + toDecimal64((number % 499) + 9, 2) AS amount + FROM numbers(100000)`, + ], + [ + 'medium_business', + `CREATE TABLE ${DB}.medium_business ENGINE = MergeTree ORDER BY (tenant_id, d) AS + SELECT concat('t_', toString(number % 8)) AS tenant_id, + toDate('2025-01-01') + toIntervalDay(number % 730) AS d, + ['SE','NO','DK','FI','DE'][(number % 5) + 1] AS region, + toDecimal64((number % 977) + 13, 2) AS amount + FROM numbers(2000000)`, + ], + [ + 'vehiq_market', + `CREATE TABLE ${DB}.vehiq_market ENGINE = MergeTree ORDER BY (make, model_year) AS + SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(number % 6) + 1] AS make, + toUInt16(2005 + (number % 20)) AS model_year, + toUInt32(30000 + (number % 900000)) AS price, + toUInt32((number * 7) % 400000) AS mileage, + ['diesel','petrol','hybrid','electric'][(number % 4) + 1] AS fuel + FROM numbers(1000000)`, + ], + [ + 'large_event', + `CREATE TABLE ${DB}.large_event ENGINE = MergeTree ORDER BY (event_type, ts) AS + SELECT toDateTime('2026-08-01 00:00:00') + toIntervalSecond(number % 2592000) AS ts, + toUInt32(number % 250000) AS user_id, + ['view','click','checkout','signup'][(number % 4) + 1] AS event_type, + toFloat64(number % 1000) AS value + FROM numbers(5000000)`, + ], + [ + 'wide_table', + `CREATE TABLE ${DB}.wide_table ENGINE = MergeTree ORDER BY grp AS + SELECT toUInt16(number % 500) AS grp, + ${Array.from({ length: WIDE_COLUMNS }, (_, i) => `toFloat64((number + ${i}) % 1000) AS c${i}`).join(',\n ')} + FROM numbers(200000)`, + ], + [ + 'geospatial', + `CREATE TABLE ${DB}.geospatial ENGINE = MergeTree ORDER BY (lat, lon) AS + SELECT 55.0 + (number % 10000) / 1000.0 AS lat, + 10.0 + ((number * 13) % 10000) / 1000.0 AS lon, + toUInt32(number) AS point_id + FROM numbers(500000)`, + ], +]; + +/** Runs all six DuckDB measurements inside ONE python process. */ +const DUCKDB_PROGRAM = ` +import duckdb, json, time, sys +con = duckdb.connect() +# The progress bar writes to STDOUT and corrupts the JSON transcript. It cost +# one run to find, so it is turned off explicitly rather than filtered out. +con.execute("SET enable_progress_bar=false") +WIDE = ${WIDE_COLUMNS} +wide_cols = ", ".join(f"(i + {j}) % 1000.0 AS c{j}" for j in range(WIDE)) +ddl = { + "small_saas": "CREATE TABLE small_saas AS SELECT 't_' || (i % 3) AS tenant_id, DATE '2026-01-01' + INTERVAL (i % 365) DAY AS d, ['free','pro','enterprise'][(i % 3) + 1] AS plan, ((i % 499) + 9)::DECIMAL(18,2) AS amount FROM range(100000) t(i)", + "medium_business": "CREATE TABLE medium_business AS SELECT 't_' || (i % 8) AS tenant_id, DATE '2025-01-01' + INTERVAL (i % 730) DAY AS d, ['SE','NO','DK','FI','DE'][(i % 5) + 1] AS region, ((i % 977) + 13)::DECIMAL(18,2) AS amount FROM range(2000000) t(i)", + "vehiq_market": "CREATE TABLE vehiq_market AS SELECT ['Volvo','VW','Toyota','BMW','Kia','Tesla'][(i % 6) + 1] AS make, (2005 + (i % 20))::USMALLINT AS model_year, (30000 + (i % 900000))::UINTEGER AS price, ((i * 7) % 400000)::UINTEGER AS mileage, ['diesel','petrol','hybrid','electric'][(i % 4) + 1] AS fuel FROM range(1000000) t(i)", + "large_event": "CREATE TABLE large_event AS SELECT TIMESTAMP '2026-08-01 00:00:00' + INTERVAL (i % 2592000) SECOND AS ts, (i % 250000)::UINTEGER AS user_id, ['view','click','checkout','signup'][(i % 4) + 1] AS event_type, (i % 1000)::DOUBLE AS value FROM range(5000000) t(i)", + "wide_table": f"CREATE TABLE wide_table AS SELECT (i % 500)::USMALLINT AS grp, {wide_cols} FROM range(200000) t(i)", + "geospatial": "CREATE TABLE geospatial AS SELECT 55.0 + (i % 10000) / 1000.0 AS lat, 10.0 + ((i * 13) % 10000) / 1000.0 AS lon, i::UINTEGER AS point_id FROM range(500000) t(i)", +} +queries = json.loads(sys.stdin.read()) +out = {"version": duckdb.__version__, "runs": {}} +for name, sql in ddl.items(): + con.execute(sql) +for name, q in queries.items(): + warm = [] + for _ in range(${WARMUPS}): + t0 = time.perf_counter(); rows = con.execute(q).fetchall(); warm.append((time.perf_counter() - t0) * 1000) + samples = [] + for _ in range(${REPETITIONS}): + t0 = time.perf_counter(); rows = con.execute(q).fetchall(); samples.append((time.perf_counter() - t0) * 1000) + out["runs"][name] = {"warmup_ms": warm, "samples_ms": samples, "rows": len(rows)} +sys.stdout.write("<<>>" + json.dumps(out)) +`; + +const runs: BenchmarkRun[] = []; +const engines: EngineAvailability[] = []; +let duckdbResult: + | { version: string; runs: Record } + | undefined; + +run('BI-P104 benchmark harness against real engines', () => { + beforeAll(async () => { + const version = (await ch('SELECT version()')).trim(); + engines.push({ engine: 'clickhouse', available: true, version }); + await ch(`CREATE DATABASE IF NOT EXISTS ${DB}`); + for (const [, ddl] of CH_DDL) await ch(ddl); + }, 900_000); + + afterAll(async () => { + if (!CLICKHOUSE_URL) return; + try { + await ch(`DROP DATABASE IF EXISTS ${DB}`); + } finally { + const comparisons = duckdbResult + ? (Object.keys(QUERIES) as BenchDatasetId[]).flatMap((dataset) => { + const chRun = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset); + const duckRun = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset); + if (!chRun || !duckRun) return []; + const c = compareDistributions(chRun.distribution, duckRun.distribution); + return [ + { + dataset, + query_label: QUERIES[dataset].label, + description: describeComparison(c, `duckdb vs clickhouse (${dataset})`), + ...c, + }, + ]; + }) + : []; + const evidence = buildEvidence({ + host_note: + 'One idle host, warm caches, both engines local, single-threaded client. These numbers ' + + 'describe these query shapes on this data on this machine. They are not engine rankings.', + engines, + runs, + comparisons, + }); + const dir = path.join(process.cwd(), 'evidence/bi-p104'); + fs.mkdirSync(dir, { recursive: true }); + fs.writeFileSync(path.join(dir, 'benchmark.json'), `${JSON.stringify(evidence, null, 2)}\n`); + } + }, 120_000); + + it('measures all six dataset profiles on ClickHouse', async () => { + for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) { + const q = QUERIES[dataset]; + const warmup_ms: number[] = []; + const samples_ms: number[] = []; + let rows = 0; + for (let i = 0; i < WARMUPS; i += 1) { + const t0 = performance.now(); + await ch(`${q.clickhouse} FORMAT JSONCompact`); + warmup_ms.push(performance.now() - t0); + } + for (let i = 0; i < REPETITIONS; i += 1) { + const t0 = performance.now(); + const text = await ch(`${q.clickhouse} FORMAT JSONCompact`); + samples_ms.push(performance.now() - t0); + rows = (JSON.parse(text) as { rows: number }).rows; + } + runs.push({ + engine: 'clickhouse', + dataset, + query_label: q.label, + warmup_ms, + samples_ms, + distribution: summarizeSamples(samples_ms), + rows_returned: rows, + }); + } + expect(runs.filter((r) => r.engine === 'clickhouse')).toHaveLength(6); + for (const r of runs) { + expect(r.distribution.n).toBe(REPETITIONS); + expect(r.distribution.stability).not.toBe('insufficient_samples'); + } + }, 900_000); + + it('measures the same six on DuckDB, or records why it did not', async () => { + if (!DUCKDB_PYTHON) { + engines.push({ + engine: 'duckdb', + available: false, + reason: 'DUCKDB_PYTHON not set on this runner', + }); + expect(engines.find((e) => e.engine === 'duckdb')?.reason).toBeTruthy(); + return; + } + const payload = Object.fromEntries( + (Object.keys(QUERIES) as BenchDatasetId[]).map((d) => [d, QUERIES[d].duckdb]), + ); + const stdout = execFileSync(DUCKDB_PYTHON, ['-c', DUCKDB_PROGRAM], { + input: JSON.stringify(payload), + encoding: 'utf8', + maxBuffer: 32 * 1024 * 1024, + timeout: 900_000, + }); + // Parse from the sentinel: anything DuckDB writes to stdout before it + // (a progress bar, a warning) must not be able to corrupt the transcript. + const marker = stdout.lastIndexOf('<<>>'); + expect(marker, 'duckdb transcript marker missing').toBeGreaterThanOrEqual(0); + duckdbResult = JSON.parse(stdout.slice(marker + '<<>>'.length)); + engines.push({ engine: 'duckdb', available: true, version: duckdbResult!.version }); + + for (const [dataset, r] of Object.entries(duckdbResult!.runs)) { + runs.push({ + engine: 'duckdb', + dataset: dataset as BenchDatasetId, + query_label: QUERIES[dataset as BenchDatasetId].label, + warmup_ms: r.warmup_ms, + samples_ms: r.samples_ms, + distribution: summarizeSamples(r.samples_ms), + rows_returned: r.rows, + }); + } + expect(runs.filter((r) => r.engine === 'duckdb')).toHaveLength(6); + }, 900_000); + + it('records the engines BI-P104 names that nothing measured, with reasons', () => { + engines.push( + { + engine: 'trino', + available: false, + reason: + 'a Trino is running on this host but belongs to another lane (st0re-verify-trino); ' + + 'benchmarking through someone else’s container would measure their configuration', + }, + { + engine: 'semantic_query', + available: false, + reason: 'MetricFlow has never been executed in 0pt (BI-P029 honest status)', + }, + { engine: 'superset', available: false, reason: 'no Superset deployment in the estate yet' }, + { engine: 'native_charts', available: false, reason: 'render-side; DES1GN owns it (BI-P051..P057)' }, + { engine: 'perspective', available: false, reason: 'not deployed' }, + ); + const evidence = buildEvidence({ host_note: 'x', engines, runs }); + for (const e of evidence.not_exercised) expect(e.reason, e.engine).toBeTruthy(); + expect(evidence.engines).toHaveLength(7); + }); + + it('reports every comparison as a range with its stability, never a bare figure', () => { + if (!duckdbResult) return; + for (const dataset of Object.keys(QUERIES) as BenchDatasetId[]) { + const a = runs.find((r) => r.engine === 'clickhouse' && r.dataset === dataset)!; + const b = runs.find((r) => r.engine === 'duckdb' && r.dataset === dataset)!; + const sentence = describeComparison(compareDistributions(a.distribution, b.distribution), dataset); + expect(sentence).toMatch(/n=9\/9/); + expect(sentence).toMatch(/INDISTINGUISHABLE|x to /); + } + }); +});