{ "recorded_at": "2026-09-07T02:06:11.079Z", "host_note": "One idle host, warm caches, both engines local, single-threaded client. These numbers describe these query shapes on this data on this machine. They are not engine rankings.", "datasets": [ { "id": "small_saas", "label": "small SaaS", "approx_rows": 100000, "shape": "narrow fact table, one tenant, high selectivity" }, { "id": "medium_business", "label": "medium business", "approx_rows": 2000000, "shape": "multi-tenant fact table with a date and a few dimensions" }, { "id": "vehiq_market", "label": "VehIQ market", "approx_rows": 1000000, "shape": "vehicle listings: make/model/year/price/mileage, skewed cardinalities" }, { "id": "large_event", "label": "large event", "approx_rows": 5000000, "shape": "append-only event stream, timestamp-ordered, sparse dimensions" }, { "id": "wide_table", "label": "wide table", "approx_rows": 200000, "shape": "many columns, few read per query — the columnar-store best case" }, { "id": "geospatial", "label": "geospatial", "approx_rows": 500000, "shape": "lat/lon points with bounding-box and distance predicates" } ], "engines": [ { "engine": "clickhouse", "available": true, "version": "26.3.10.60" }, { "engine": "duckdb", "available": true, "version": "1.5.5" }, { "engine": "trino", "available": false, "reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration" }, { "engine": "semantic_query", "available": false, "reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)" }, { "engine": "superset", "available": false, "reason": "no Superset deployment in the estate yet" }, { "engine": "native_charts", "available": false, "reason": "render-side; DES1GN owns it (BI-P051..P057)" }, { "engine": "perspective", "available": false, "reason": "not deployed" } ], "runs": [ { "engine": "clickhouse", "dataset": "small_saas", "query_label": "revenue by day", "warmup_ms": [58.856714000000466, 58.77232499999991], "samples_ms": [ 51.25258499999927, 23.924909000000298, 39.89413500000046, 29.40832799999953, 43.058938000000126, 39.98396799999955, 41.99595799999952, 32.193271000000095, 25.963267999999516 ], "distribution": { "n": 9, "min": 23.924909000000298, "p50": 39.89413500000046, "p95": 51.25258499999927, "p99": 51.25258499999927, "max": 51.25258499999927, "mean": 36.40837333333315, "stdev": 8.514314573768598, "cv": 0.23385594560395923, "iqr": 12.58762999999999, "spread_ratio": 2.1422269568506462, "stability": "noisy" }, "rows_returned": 365 }, { "engine": "clickhouse", "dataset": "medium_business", "query_label": "revenue by month and region for one tenant", "warmup_ms": [44.60071400000015, 70.7402890000003], "samples_ms": [ 70.63034199999947, 69.65554000000066, 58.036339000000225, 46.82895400000052, 44.67711700000018, 87.97018200000002, 49.249023999999736, 59.75650299999961, 41.953865000000405 ], "distribution": { "n": 9, "min": 41.953865000000405, "p50": 58.036339000000225, "p95": 87.97018200000002, "p99": 87.97018200000002, "max": 87.97018200000002, "mean": 58.750874000000096, "stdev": 14.251513712332908, "cv": 0.24257534811027467, "iqr": 22.826586000000134, "spread_ratio": 2.096831412314436, "stability": "noisy" }, "rows_returned": 120 }, { "engine": "clickhouse", "dataset": "vehiq_market", "query_label": "average asking price by make and model year", "warmup_ms": [83.03972700000031, 59.77780099999927], "samples_ms": [ 74.38707999999951, 60.41443299999992, 59.13954199999989, 98.55565299999944, 98.45707200000015, 74.05828699999984, 194.5833190000003, 159.76290199999949, 87.88670300000013 ], "distribution": { "n": 9, "min": 59.13954199999989, "p50": 87.88670300000013, "p95": 194.5833190000003, "p99": 194.5833190000003, "max": 194.5833190000003, "mean": 100.80499899999985, "stdev": 43.75254087132061, "cv": 0.4340314598021143, "iqr": 24.4973659999996, "spread_ratio": 3.290240546671813, "stability": "noisy" }, "rows_returned": 60 }, { "engine": "clickhouse", "dataset": "large_event", "query_label": "hourly event volume for one event type", "warmup_ms": [122.44560499999989, 136.30929500000002], "samples_ms": [ 142.78732200000013, 136.18330800000058, 82.08186000000023, 95.91282499999943, 74.52815499999997, 93.95668399999977, 112.81592300000011, 76.3508299999994, 93.78873100000055 ], "distribution": { "n": 9, "min": 74.52815499999997, "p50": 93.95668399999977, "p95": 142.78732200000013, "p99": 142.78732200000013, "max": 142.78732200000013, "mean": 100.9339597777778, "stdev": 23.374602687288522, "cv": 0.23158313355337923, "iqr": 30.73406299999988, "spread_ratio": 1.9158842990276652, "stability": "noisy" }, "rows_returned": 720 }, { "engine": "clickhouse", "dataset": "wide_table", "query_label": "three columns of fifty, grouped", "warmup_ms": [28.096958000000086, 14.89937800000007], "samples_ms": [ 19.301219999999375, 21.628866000000926, 32.125191999999515, 25.563218999999663, 24.388967999999295, 16.8507410000002, 19.57230399999935, 20.52187000000049, 18.078355999999985 ], "distribution": { "n": 9, "min": 16.8507410000002, "p50": 20.52187000000049, "p95": 32.125191999999515, "p99": 32.125191999999515, "max": 32.125191999999515, "mean": 22.00341511111098, "stdev": 4.451179566867264, "cv": 0.2022949412348072, "iqr": 5.08774799999992, "spread_ratio": 1.9064557457739775, "stability": "noisy" }, "rows_returned": 500 }, { "engine": "clickhouse", "dataset": "geospatial", "query_label": "bounding box, aggregated to a grid cell", "warmup_ms": [38.77805500000068, 17.665412000000288], "samples_ms": [ 29.736737000001085, 27.89697900000101, 25.586840999998458, 21.336174000000028, 22.492867999999362, 30.176470000000336, 22.63948900000105, 29.021173000001, 36.31744599999911 ], "distribution": { "n": 9, "min": 21.336174000000028, "p50": 27.89697900000101, "p95": 36.31744599999911, "p99": 36.31744599999911, "max": 36.31744599999911, "mean": 27.244908555555714, "stdev": 4.498748211227398, "cv": 0.16512252929951657, "iqr": 7.097248000000036, "spread_ratio": 1.7021536288558137, "stability": "noisy" }, "rows_returned": 201 }, { "engine": "duckdb", "dataset": "small_saas", "query_label": "revenue by day", "warmup_ms": [12.830814997869311, 8.479544001602335], "samples_ms": [ 7.375242999842158, 9.8867060005432, 9.773318997758906, 9.169243003270822, 7.295589999557706, 10.63253999745939, 6.214401000761427, 15.184157000476262, 7.2293630000785924 ], "distribution": { "n": 9, "min": 6.214401000761427, "p50": 9.169243003270822, "p95": 15.184157000476262, "p99": 15.184157000476262, "max": 15.184157000476262, "mean": 9.19561799997205, "stdev": 2.5477019265669423, "cv": 0.27705608547187216, "iqr": 2.591116000985494, "spread_ratio": 2.443382234042477, "stability": "noisy" }, "rows_returned": 365 }, { "engine": "duckdb", "dataset": "medium_business", "query_label": "revenue by month and region for one tenant", "warmup_ms": [60.29315100022359, 47.50260100263404], "samples_ms": [ 47.711714996694354, 36.605457000405295, 30.622688002040377, 44.33469899959164, 49.469069999759085, 33.921502999874065, 33.03168200000073, 40.499283000826836, 33.27470799922594 ], "distribution": { "n": 9, "min": 30.622688002040377, "p50": 36.605457000405295, "p95": 49.469069999759085, "p99": 49.469069999759085, "max": 49.469069999759085, "mean": 38.8300894442687, "stdev": 6.549355811539825, "cv": 0.1686670287211122, "iqr": 11.059991000365699, "spread_ratio": 1.615438527031428, "stability": "noisy" }, "rows_returned": 120 }, { "engine": "duckdb", "dataset": "vehiq_market", "query_label": "average asking price by make and model year", "warmup_ms": [17.676940999081125, 24.05106900187093], "samples_ms": [ 14.901875001669396, 14.926752002793364, 17.798165001295274, 15.059515000757528, 19.75441800095723, 14.601105001929682, 14.954296002542833, 20.32349000000977, 24.610873999336036 ], "distribution": { "n": 9, "min": 14.601105001929682, "p50": 15.059515000757528, "p95": 24.610873999336036, "p99": 24.610873999336036, "max": 24.610873999336036, "mean": 17.436721112365678, "stdev": 3.2978767006322265, "cv": 0.18913399367805778, "iqr": 4.827665998163866, "spread_ratio": 1.6855487304613908, "stability": "noisy" }, "rows_returned": 60 }, { "engine": "duckdb", "dataset": "large_event", "query_label": "hourly event volume for one event type", "warmup_ms": [184.43432099957136, 163.9354379985889], "samples_ms": [ 160.28642499804846, 161.1321640011738, 177.48030700022355, 213.59406000192394, 337.0364080001309, 334.6067929996934, 232.15600899857236, 204.8815359994478, 196.73219499964034 ], "distribution": { "n": 9, "min": 160.28642499804846, "p50": 204.8815359994478, "p95": 337.0364080001309, "p99": 337.0364080001309, "max": 337.0364080001309, "mean": 224.21176633320607, "stdev": 63.65742323059742, "cv": 0.28391651460429923, "iqr": 54.67570199834881, "spread_ratio": 2.1027133645549485, "stability": "noisy" }, "rows_returned": 720 }, { "engine": "duckdb", "dataset": "wide_table", "query_label": "three columns of fifty, grouped", "warmup_ms": [18.729354997049086, 21.23752199986484], "samples_ms": [ 14.388244999281596, 15.164796001045033, 18.07164799902239, 22.90704599727178, 19.191682997188764, 16.838232000736753, 16.024758999265032, 24.479674000758678, 22.105899999587564 ], "distribution": { "n": 9, "min": 14.388244999281596, "p50": 18.07164799902239, "p95": 24.479674000758678, "p99": 24.479674000758678, "max": 24.479674000758678, "mean": 18.796886999350843, "stdev": 3.4151315245862714, "cv": 0.1816860166635153, "iqr": 6.081141000322532, "spread_ratio": 1.7013662195758374, "stability": "noisy" }, "rows_returned": 500 }, { "engine": "duckdb", "dataset": "geospatial", "query_label": "bounding box, aggregated to a grid cell", "warmup_ms": [27.98272599829943, 22.078424000937957], "samples_ms": [ 30.961295000452083, 28.811156000301708, 19.97572500113165, 22.212601001228904, 33.500631998322206, 34.46119200089015, 24.307676001626533, 20.643488001951482, 24.275364998175064 ], "distribution": { "n": 9, "min": 19.97572500113165, "p50": 24.307676001626533, "p95": 34.46119200089015, "p99": 34.46119200089015, "max": 34.46119200089015, "mean": 26.572125556008864, "stdev": 5.191758162396437, "cv": 0.19538362301703047, "iqr": 8.748693999223178, "spread_ratio": 1.7251535050136042, "stability": "noisy" }, "rows_returned": 201 } ], "comparisons": [ { "dataset": "small_saas", "query_label": "revenue by day", "description": "duckdb vs clickhouse (small_saas): FASTER — 1.6x to 8.2x (median 4.4x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", "p50_ratio": 4.350864622714171, "conservative_ratio": 1.5756494745971, "optimistic_ratio": 8.247389409489264, "ranges_overlap": false, "baseline": { "n": 9, "min": 23.924909000000298, "p50": 39.89413500000046, "p95": 51.25258499999927, "p99": 51.25258499999927, "max": 51.25258499999927, "mean": 36.40837333333315, "stdev": 8.514314573768598, "cv": 0.23385594560395923, "iqr": 12.58762999999999, "spread_ratio": 2.1422269568506462, "stability": "noisy" }, "candidate": { "n": 9, "min": 6.214401000761427, "p50": 9.169243003270822, "p95": 15.184157000476262, "p99": 15.184157000476262, "max": 15.184157000476262, "mean": 9.19561799997205, "stdev": 2.5477019265669423, "cv": 0.27705608547187216, "iqr": 2.591116000985494, "spread_ratio": 2.443382234042477, "stability": "noisy" }, "verdict": "faster" }, { "dataset": "medium_business", "query_label": "revenue by month and region for one tenant", "description": "duckdb vs clickhouse (medium_business): INDISTINGUISHABLE at this sample size. Medians differ by 1.6x but the observed ranges overlap (baseline 42.0-88.0ms, candidate 30.6-49.5ms; n=9/9, CV 0.24/0.17).", "p50_ratio": 1.5854559335062433, "conservative_ratio": 0.8480827515092707, "optimistic_ratio": 2.872712610798197, "ranges_overlap": true, "baseline": { "n": 9, "min": 41.953865000000405, "p50": 58.036339000000225, "p95": 87.97018200000002, "p99": 87.97018200000002, "max": 87.97018200000002, "mean": 58.750874000000096, "stdev": 14.251513712332908, "cv": 0.24257534811027467, "iqr": 22.826586000000134, "spread_ratio": 2.096831412314436, "stability": "noisy" }, "candidate": { "n": 9, "min": 30.622688002040377, "p50": 36.605457000405295, "p95": 49.469069999759085, "p99": 49.469069999759085, "max": 49.469069999759085, "mean": 38.8300894442687, "stdev": 6.549355811539825, "cv": 0.1686670287211122, "iqr": 11.059991000365699, "spread_ratio": 1.615438527031428, "stability": "noisy" }, "verdict": "indistinguishable" }, { "dataset": "vehiq_market", "query_label": "average asking price by make and model year", "description": "duckdb vs clickhouse (vehiq_market): FASTER — 2.4x to 13.3x (median 5.8x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", "p50_ratio": 5.835958395444954, "conservative_ratio": 2.4029842256555125, "optimistic_ratio": 13.32661596326334, "ranges_overlap": false, "baseline": { "n": 9, "min": 59.13954199999989, "p50": 87.88670300000013, "p95": 194.5833190000003, "p99": 194.5833190000003, "max": 194.5833190000003, "mean": 100.80499899999985, "stdev": 43.75254087132061, "cv": 0.4340314598021143, "iqr": 24.4973659999996, "spread_ratio": 3.290240546671813, "stability": "noisy" }, "candidate": { "n": 9, "min": 14.601105001929682, "p50": 15.059515000757528, "p95": 24.610873999336036, "p99": 24.610873999336036, "max": 24.610873999336036, "mean": 17.436721112365678, "stdev": 3.2978767006322265, "cv": 0.18913399367805778, "iqr": 4.827665998163866, "spread_ratio": 1.6855487304613908, "stability": "noisy" }, "verdict": "faster" }, { "dataset": "large_event", "query_label": "hourly event volume for one event type", "description": "duckdb vs clickhouse (large_event): SLOWER — 0.2x to 0.9x (median 0.5x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.", "p50_ratio": 0.4585902948338546, "conservative_ratio": 0.2211279055643479, "optimistic_ratio": 0.8908260446992852, "ranges_overlap": false, "baseline": { "n": 9, "min": 74.52815499999997, "p50": 93.95668399999977, "p95": 142.78732200000013, "p99": 142.78732200000013, "max": 142.78732200000013, "mean": 100.9339597777778, "stdev": 23.374602687288522, "cv": 0.23158313355337923, "iqr": 30.73406299999988, "spread_ratio": 1.9158842990276652, "stability": "noisy" }, "candidate": { "n": 9, "min": 160.28642499804846, "p50": 204.8815359994478, "p95": 337.0364080001309, "p99": 337.0364080001309, "max": 337.0364080001309, "mean": 224.21176633320607, "stdev": 63.65742323059742, "cv": 0.28391651460429923, "iqr": 54.67570199834881, "spread_ratio": 2.1027133645549485, "stability": "noisy" }, "verdict": "slower" }, { "dataset": "wide_table", "query_label": "three columns of fifty, grouped", "description": "duckdb vs clickhouse (wide_table): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 16.9-32.1ms, candidate 14.4-24.5ms; n=9/9, CV 0.20/0.18).", "p50_ratio": 1.1355837608784018, "conservative_ratio": 0.688356429888648, "optimistic_ratio": 2.232738739269697, "ranges_overlap": true, "baseline": { "n": 9, "min": 16.8507410000002, "p50": 20.52187000000049, "p95": 32.125191999999515, "p99": 32.125191999999515, "max": 32.125191999999515, "mean": 22.00341511111098, "stdev": 4.451179566867264, "cv": 0.2022949412348072, "iqr": 5.08774799999992, "spread_ratio": 1.9064557457739775, "stability": "noisy" }, "candidate": { "n": 9, "min": 14.388244999281596, "p50": 18.07164799902239, "p95": 24.479674000758678, "p99": 24.479674000758678, "max": 24.479674000758678, "mean": 18.796886999350843, "stdev": 3.4151315245862714, "cv": 0.1816860166635153, "iqr": 6.081141000322532, "spread_ratio": 1.7013662195758374, "stability": "noisy" }, "verdict": "indistinguishable" }, { "dataset": "geospatial", "query_label": "bounding box, aggregated to a grid cell", "description": "duckdb vs clickhouse (geospatial): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 21.3-36.3ms, candidate 20.0-34.5ms; n=9/9, CV 0.17/0.20).", "p50_ratio": 1.1476612983542442, "conservative_ratio": 0.619136273621902, "optimistic_ratio": 1.8180789932751718, "ranges_overlap": true, "baseline": { "n": 9, "min": 21.336174000000028, "p50": 27.89697900000101, "p95": 36.31744599999911, "p99": 36.31744599999911, "max": 36.31744599999911, "mean": 27.244908555555714, "stdev": 4.498748211227398, "cv": 0.16512252929951657, "iqr": 7.097248000000036, "spread_ratio": 1.7021536288558137, "stability": "noisy" }, "candidate": { "n": 9, "min": 19.97572500113165, "p50": 24.307676001626533, "p95": 34.46119200089015, "p99": 34.46119200089015, "max": 34.46119200089015, "mean": 26.572125556008864, "stdev": 5.191758162396437, "cv": 0.19538362301703047, "iqr": 8.748693999223178, "spread_ratio": 1.7251535050136042, "stability": "noisy" }, "verdict": "indistinguishable" } ], "not_exercised": [ { "engine": "trino", "available": false, "reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration" }, { "engine": "semantic_query", "available": false, "reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)" }, { "engine": "superset", "available": false, "reason": "no Superset deployment in the estate yet" }, { "engine": "native_charts", "available": false, "reason": "render-side; DES1GN owns it (BI-P051..P057)" }, { "engine": "perspective", "available": false, "reason": "not deployed" } ] }