Files

639 lines
21 KiB
JSON
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
{
"recorded_at": "2026-09-07T02:06:11.079Z",
"host_note": "One idle host, warm caches, both engines local, single-threaded client. These numbers describe these query shapes on this data on this machine. They are not engine rankings.",
"datasets": [
{
"id": "small_saas",
"label": "small SaaS",
"approx_rows": 100000,
"shape": "narrow fact table, one tenant, high selectivity"
},
{
"id": "medium_business",
"label": "medium business",
"approx_rows": 2000000,
"shape": "multi-tenant fact table with a date and a few dimensions"
},
{
"id": "vehiq_market",
"label": "VehIQ market",
"approx_rows": 1000000,
"shape": "vehicle listings: make/model/year/price/mileage, skewed cardinalities"
},
{
"id": "large_event",
"label": "large event",
"approx_rows": 5000000,
"shape": "append-only event stream, timestamp-ordered, sparse dimensions"
},
{
"id": "wide_table",
"label": "wide table",
"approx_rows": 200000,
"shape": "many columns, few read per query — the columnar-store best case"
},
{
"id": "geospatial",
"label": "geospatial",
"approx_rows": 500000,
"shape": "lat/lon points with bounding-box and distance predicates"
}
],
"engines": [
{
"engine": "clickhouse",
"available": true,
"version": "26.3.10.60"
},
{
"engine": "duckdb",
"available": true,
"version": "1.5.5"
},
{
"engine": "trino",
"available": false,
"reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration"
},
{
"engine": "semantic_query",
"available": false,
"reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)"
},
{
"engine": "superset",
"available": false,
"reason": "no Superset deployment in the estate yet"
},
{
"engine": "native_charts",
"available": false,
"reason": "render-side; DES1GN owns it (BI-P051..P057)"
},
{
"engine": "perspective",
"available": false,
"reason": "not deployed"
}
],
"runs": [
{
"engine": "clickhouse",
"dataset": "small_saas",
"query_label": "revenue by day",
"warmup_ms": [58.856714000000466, 58.77232499999991],
"samples_ms": [
51.25258499999927, 23.924909000000298, 39.89413500000046, 29.40832799999953, 43.058938000000126,
39.98396799999955, 41.99595799999952, 32.193271000000095, 25.963267999999516
],
"distribution": {
"n": 9,
"min": 23.924909000000298,
"p50": 39.89413500000046,
"p95": 51.25258499999927,
"p99": 51.25258499999927,
"max": 51.25258499999927,
"mean": 36.40837333333315,
"stdev": 8.514314573768598,
"cv": 0.23385594560395923,
"iqr": 12.58762999999999,
"spread_ratio": 2.1422269568506462,
"stability": "noisy"
},
"rows_returned": 365
},
{
"engine": "clickhouse",
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"warmup_ms": [44.60071400000015, 70.7402890000003],
"samples_ms": [
70.63034199999947, 69.65554000000066, 58.036339000000225, 46.82895400000052, 44.67711700000018,
87.97018200000002, 49.249023999999736, 59.75650299999961, 41.953865000000405
],
"distribution": {
"n": 9,
"min": 41.953865000000405,
"p50": 58.036339000000225,
"p95": 87.97018200000002,
"p99": 87.97018200000002,
"max": 87.97018200000002,
"mean": 58.750874000000096,
"stdev": 14.251513712332908,
"cv": 0.24257534811027467,
"iqr": 22.826586000000134,
"spread_ratio": 2.096831412314436,
"stability": "noisy"
},
"rows_returned": 120
},
{
"engine": "clickhouse",
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"warmup_ms": [83.03972700000031, 59.77780099999927],
"samples_ms": [
74.38707999999951, 60.41443299999992, 59.13954199999989, 98.55565299999944, 98.45707200000015,
74.05828699999984, 194.5833190000003, 159.76290199999949, 87.88670300000013
],
"distribution": {
"n": 9,
"min": 59.13954199999989,
"p50": 87.88670300000013,
"p95": 194.5833190000003,
"p99": 194.5833190000003,
"max": 194.5833190000003,
"mean": 100.80499899999985,
"stdev": 43.75254087132061,
"cv": 0.4340314598021143,
"iqr": 24.4973659999996,
"spread_ratio": 3.290240546671813,
"stability": "noisy"
},
"rows_returned": 60
},
{
"engine": "clickhouse",
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"warmup_ms": [122.44560499999989, 136.30929500000002],
"samples_ms": [
142.78732200000013, 136.18330800000058, 82.08186000000023, 95.91282499999943, 74.52815499999997,
93.95668399999977, 112.81592300000011, 76.3508299999994, 93.78873100000055
],
"distribution": {
"n": 9,
"min": 74.52815499999997,
"p50": 93.95668399999977,
"p95": 142.78732200000013,
"p99": 142.78732200000013,
"max": 142.78732200000013,
"mean": 100.9339597777778,
"stdev": 23.374602687288522,
"cv": 0.23158313355337923,
"iqr": 30.73406299999988,
"spread_ratio": 1.9158842990276652,
"stability": "noisy"
},
"rows_returned": 720
},
{
"engine": "clickhouse",
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"warmup_ms": [28.096958000000086, 14.89937800000007],
"samples_ms": [
19.301219999999375, 21.628866000000926, 32.125191999999515, 25.563218999999663, 24.388967999999295,
16.8507410000002, 19.57230399999935, 20.52187000000049, 18.078355999999985
],
"distribution": {
"n": 9,
"min": 16.8507410000002,
"p50": 20.52187000000049,
"p95": 32.125191999999515,
"p99": 32.125191999999515,
"max": 32.125191999999515,
"mean": 22.00341511111098,
"stdev": 4.451179566867264,
"cv": 0.2022949412348072,
"iqr": 5.08774799999992,
"spread_ratio": 1.9064557457739775,
"stability": "noisy"
},
"rows_returned": 500
},
{
"engine": "clickhouse",
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"warmup_ms": [38.77805500000068, 17.665412000000288],
"samples_ms": [
29.736737000001085, 27.89697900000101, 25.586840999998458, 21.336174000000028, 22.492867999999362,
30.176470000000336, 22.63948900000105, 29.021173000001, 36.31744599999911
],
"distribution": {
"n": 9,
"min": 21.336174000000028,
"p50": 27.89697900000101,
"p95": 36.31744599999911,
"p99": 36.31744599999911,
"max": 36.31744599999911,
"mean": 27.244908555555714,
"stdev": 4.498748211227398,
"cv": 0.16512252929951657,
"iqr": 7.097248000000036,
"spread_ratio": 1.7021536288558137,
"stability": "noisy"
},
"rows_returned": 201
},
{
"engine": "duckdb",
"dataset": "small_saas",
"query_label": "revenue by day",
"warmup_ms": [12.830814997869311, 8.479544001602335],
"samples_ms": [
7.375242999842158, 9.8867060005432, 9.773318997758906, 9.169243003270822, 7.295589999557706,
10.63253999745939, 6.214401000761427, 15.184157000476262, 7.2293630000785924
],
"distribution": {
"n": 9,
"min": 6.214401000761427,
"p50": 9.169243003270822,
"p95": 15.184157000476262,
"p99": 15.184157000476262,
"max": 15.184157000476262,
"mean": 9.19561799997205,
"stdev": 2.5477019265669423,
"cv": 0.27705608547187216,
"iqr": 2.591116000985494,
"spread_ratio": 2.443382234042477,
"stability": "noisy"
},
"rows_returned": 365
},
{
"engine": "duckdb",
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"warmup_ms": [60.29315100022359, 47.50260100263404],
"samples_ms": [
47.711714996694354, 36.605457000405295, 30.622688002040377, 44.33469899959164, 49.469069999759085,
33.921502999874065, 33.03168200000073, 40.499283000826836, 33.27470799922594
],
"distribution": {
"n": 9,
"min": 30.622688002040377,
"p50": 36.605457000405295,
"p95": 49.469069999759085,
"p99": 49.469069999759085,
"max": 49.469069999759085,
"mean": 38.8300894442687,
"stdev": 6.549355811539825,
"cv": 0.1686670287211122,
"iqr": 11.059991000365699,
"spread_ratio": 1.615438527031428,
"stability": "noisy"
},
"rows_returned": 120
},
{
"engine": "duckdb",
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"warmup_ms": [17.676940999081125, 24.05106900187093],
"samples_ms": [
14.901875001669396, 14.926752002793364, 17.798165001295274, 15.059515000757528, 19.75441800095723,
14.601105001929682, 14.954296002542833, 20.32349000000977, 24.610873999336036
],
"distribution": {
"n": 9,
"min": 14.601105001929682,
"p50": 15.059515000757528,
"p95": 24.610873999336036,
"p99": 24.610873999336036,
"max": 24.610873999336036,
"mean": 17.436721112365678,
"stdev": 3.2978767006322265,
"cv": 0.18913399367805778,
"iqr": 4.827665998163866,
"spread_ratio": 1.6855487304613908,
"stability": "noisy"
},
"rows_returned": 60
},
{
"engine": "duckdb",
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"warmup_ms": [184.43432099957136, 163.9354379985889],
"samples_ms": [
160.28642499804846, 161.1321640011738, 177.48030700022355, 213.59406000192394, 337.0364080001309,
334.6067929996934, 232.15600899857236, 204.8815359994478, 196.73219499964034
],
"distribution": {
"n": 9,
"min": 160.28642499804846,
"p50": 204.8815359994478,
"p95": 337.0364080001309,
"p99": 337.0364080001309,
"max": 337.0364080001309,
"mean": 224.21176633320607,
"stdev": 63.65742323059742,
"cv": 0.28391651460429923,
"iqr": 54.67570199834881,
"spread_ratio": 2.1027133645549485,
"stability": "noisy"
},
"rows_returned": 720
},
{
"engine": "duckdb",
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"warmup_ms": [18.729354997049086, 21.23752199986484],
"samples_ms": [
14.388244999281596, 15.164796001045033, 18.07164799902239, 22.90704599727178, 19.191682997188764,
16.838232000736753, 16.024758999265032, 24.479674000758678, 22.105899999587564
],
"distribution": {
"n": 9,
"min": 14.388244999281596,
"p50": 18.07164799902239,
"p95": 24.479674000758678,
"p99": 24.479674000758678,
"max": 24.479674000758678,
"mean": 18.796886999350843,
"stdev": 3.4151315245862714,
"cv": 0.1816860166635153,
"iqr": 6.081141000322532,
"spread_ratio": 1.7013662195758374,
"stability": "noisy"
},
"rows_returned": 500
},
{
"engine": "duckdb",
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"warmup_ms": [27.98272599829943, 22.078424000937957],
"samples_ms": [
30.961295000452083, 28.811156000301708, 19.97572500113165, 22.212601001228904, 33.500631998322206,
34.46119200089015, 24.307676001626533, 20.643488001951482, 24.275364998175064
],
"distribution": {
"n": 9,
"min": 19.97572500113165,
"p50": 24.307676001626533,
"p95": 34.46119200089015,
"p99": 34.46119200089015,
"max": 34.46119200089015,
"mean": 26.572125556008864,
"stdev": 5.191758162396437,
"cv": 0.19538362301703047,
"iqr": 8.748693999223178,
"spread_ratio": 1.7251535050136042,
"stability": "noisy"
},
"rows_returned": 201
}
],
"comparisons": [
{
"dataset": "small_saas",
"query_label": "revenue by day",
"description": "duckdb vs clickhouse (small_saas): FASTER — 1.6x to 8.2x (median 4.4x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 4.350864622714171,
"conservative_ratio": 1.5756494745971,
"optimistic_ratio": 8.247389409489264,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 23.924909000000298,
"p50": 39.89413500000046,
"p95": 51.25258499999927,
"p99": 51.25258499999927,
"max": 51.25258499999927,
"mean": 36.40837333333315,
"stdev": 8.514314573768598,
"cv": 0.23385594560395923,
"iqr": 12.58762999999999,
"spread_ratio": 2.1422269568506462,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 6.214401000761427,
"p50": 9.169243003270822,
"p95": 15.184157000476262,
"p99": 15.184157000476262,
"max": 15.184157000476262,
"mean": 9.19561799997205,
"stdev": 2.5477019265669423,
"cv": 0.27705608547187216,
"iqr": 2.591116000985494,
"spread_ratio": 2.443382234042477,
"stability": "noisy"
},
"verdict": "faster"
},
{
"dataset": "medium_business",
"query_label": "revenue by month and region for one tenant",
"description": "duckdb vs clickhouse (medium_business): INDISTINGUISHABLE at this sample size. Medians differ by 1.6x but the observed ranges overlap (baseline 42.0-88.0ms, candidate 30.6-49.5ms; n=9/9, CV 0.24/0.17).",
"p50_ratio": 1.5854559335062433,
"conservative_ratio": 0.8480827515092707,
"optimistic_ratio": 2.872712610798197,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 41.953865000000405,
"p50": 58.036339000000225,
"p95": 87.97018200000002,
"p99": 87.97018200000002,
"max": 87.97018200000002,
"mean": 58.750874000000096,
"stdev": 14.251513712332908,
"cv": 0.24257534811027467,
"iqr": 22.826586000000134,
"spread_ratio": 2.096831412314436,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 30.622688002040377,
"p50": 36.605457000405295,
"p95": 49.469069999759085,
"p99": 49.469069999759085,
"max": 49.469069999759085,
"mean": 38.8300894442687,
"stdev": 6.549355811539825,
"cv": 0.1686670287211122,
"iqr": 11.059991000365699,
"spread_ratio": 1.615438527031428,
"stability": "noisy"
},
"verdict": "indistinguishable"
},
{
"dataset": "vehiq_market",
"query_label": "average asking price by make and model year",
"description": "duckdb vs clickhouse (vehiq_market): FASTER — 2.4x to 13.3x (median 5.8x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 5.835958395444954,
"conservative_ratio": 2.4029842256555125,
"optimistic_ratio": 13.32661596326334,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 59.13954199999989,
"p50": 87.88670300000013,
"p95": 194.5833190000003,
"p99": 194.5833190000003,
"max": 194.5833190000003,
"mean": 100.80499899999985,
"stdev": 43.75254087132061,
"cv": 0.4340314598021143,
"iqr": 24.4973659999996,
"spread_ratio": 3.290240546671813,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 14.601105001929682,
"p50": 15.059515000757528,
"p95": 24.610873999336036,
"p99": 24.610873999336036,
"max": 24.610873999336036,
"mean": 17.436721112365678,
"stdev": 3.2978767006322265,
"cv": 0.18913399367805778,
"iqr": 4.827665998163866,
"spread_ratio": 1.6855487304613908,
"stability": "noisy"
},
"verdict": "faster"
},
{
"dataset": "large_event",
"query_label": "hourly event volume for one event type",
"description": "duckdb vs clickhouse (large_event): SLOWER — 0.2x to 0.9x (median 0.5x) over n=9/9, stability noisy/noisy. Warm cache, one host, this query shape.",
"p50_ratio": 0.4585902948338546,
"conservative_ratio": 0.2211279055643479,
"optimistic_ratio": 0.8908260446992852,
"ranges_overlap": false,
"baseline": {
"n": 9,
"min": 74.52815499999997,
"p50": 93.95668399999977,
"p95": 142.78732200000013,
"p99": 142.78732200000013,
"max": 142.78732200000013,
"mean": 100.9339597777778,
"stdev": 23.374602687288522,
"cv": 0.23158313355337923,
"iqr": 30.73406299999988,
"spread_ratio": 1.9158842990276652,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 160.28642499804846,
"p50": 204.8815359994478,
"p95": 337.0364080001309,
"p99": 337.0364080001309,
"max": 337.0364080001309,
"mean": 224.21176633320607,
"stdev": 63.65742323059742,
"cv": 0.28391651460429923,
"iqr": 54.67570199834881,
"spread_ratio": 2.1027133645549485,
"stability": "noisy"
},
"verdict": "slower"
},
{
"dataset": "wide_table",
"query_label": "three columns of fifty, grouped",
"description": "duckdb vs clickhouse (wide_table): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 16.9-32.1ms, candidate 14.4-24.5ms; n=9/9, CV 0.20/0.18).",
"p50_ratio": 1.1355837608784018,
"conservative_ratio": 0.688356429888648,
"optimistic_ratio": 2.232738739269697,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 16.8507410000002,
"p50": 20.52187000000049,
"p95": 32.125191999999515,
"p99": 32.125191999999515,
"max": 32.125191999999515,
"mean": 22.00341511111098,
"stdev": 4.451179566867264,
"cv": 0.2022949412348072,
"iqr": 5.08774799999992,
"spread_ratio": 1.9064557457739775,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 14.388244999281596,
"p50": 18.07164799902239,
"p95": 24.479674000758678,
"p99": 24.479674000758678,
"max": 24.479674000758678,
"mean": 18.796886999350843,
"stdev": 3.4151315245862714,
"cv": 0.1816860166635153,
"iqr": 6.081141000322532,
"spread_ratio": 1.7013662195758374,
"stability": "noisy"
},
"verdict": "indistinguishable"
},
{
"dataset": "geospatial",
"query_label": "bounding box, aggregated to a grid cell",
"description": "duckdb vs clickhouse (geospatial): INDISTINGUISHABLE at this sample size. Medians differ by 1.1x but the observed ranges overlap (baseline 21.3-36.3ms, candidate 20.0-34.5ms; n=9/9, CV 0.17/0.20).",
"p50_ratio": 1.1476612983542442,
"conservative_ratio": 0.619136273621902,
"optimistic_ratio": 1.8180789932751718,
"ranges_overlap": true,
"baseline": {
"n": 9,
"min": 21.336174000000028,
"p50": 27.89697900000101,
"p95": 36.31744599999911,
"p99": 36.31744599999911,
"max": 36.31744599999911,
"mean": 27.244908555555714,
"stdev": 4.498748211227398,
"cv": 0.16512252929951657,
"iqr": 7.097248000000036,
"spread_ratio": 1.7021536288558137,
"stability": "noisy"
},
"candidate": {
"n": 9,
"min": 19.97572500113165,
"p50": 24.307676001626533,
"p95": 34.46119200089015,
"p99": 34.46119200089015,
"max": 34.46119200089015,
"mean": 26.572125556008864,
"stdev": 5.191758162396437,
"cv": 0.19538362301703047,
"iqr": 8.748693999223178,
"spread_ratio": 1.7251535050136042,
"stability": "noisy"
},
"verdict": "indistinguishable"
}
],
"not_exercised": [
{
"engine": "trino",
"available": false,
"reason": "a Trino is running on this host but belongs to another lane (st0re-verify-trino); benchmarking through someone else’s container would measure their configuration"
},
{
"engine": "semantic_query",
"available": false,
"reason": "MetricFlow has never been executed in 0pt (BI-P029 honest status)"
},
{
"engine": "superset",
"available": false,
"reason": "no Superset deployment in the estate yet"
},
{
"engine": "native_charts",
"available": false,
"reason": "render-side; DES1GN owns it (BI-P051..P057)"
},
{
"engine": "perspective",
"available": false,
"reason": "not deployed"
}
]
}