/** * BI-P005 conformance against an INDEPENDENT Arrow implementation. * * The unit suite (tests/analytics/arrow-ipc.test.ts) only proves 0pt's * encoder agrees with 0pt's decoder — which it would even if both shared the * same misreading of the spec. Here, ClickHouse's Arrow C++ implementation is * the second opinion, in BOTH directions: * * ClickHouse -> Arrow -> 0pt decoder (their encoder, our reader) * 0pt encoder -> Arrow -> ClickHouse (our encoder, their reader) * * The second direction is the load-bearing one: ClickHouse INSERTing our * bytes and then answering a SELECT with the exact values back is a claim * that cannot be satisfied by a subtly wrong buffer layout. * * The DoD's literal chain (ClickHouse -> Arrow -> DuckDB/Python) additionally * ran against real pyarrow + DuckDB; that transcript is committed at * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json. */ import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { arrowStreamSettings, clickhouseArrowStreamFormat, decodeArrowStream, encodeArrowStream, } from "@/src/analytics/arrow/ipc"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_arrow_${Date.now().toString(36)}`; function authHeaders(): Record { return CLICKHOUSE_AUTH ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` } : {}; } async function chBytes(sql: string, settings: Record = {}): Promise { const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" }); const buf = new Uint8Array(await res.arrayBuffer()); if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`); return buf; } async function chText(sql: string, body?: BodyInit): Promise { const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); url.searchParams.set("query", sql); const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" }); const text = await res.text(); if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`); return text; } run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => { beforeAll(async () => { await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`); await chText( `CREATE TABLE IF NOT EXISTS ${DB}.arrow_roundtrip ( tenant_id String, n Int64, v Float64, flag Bool, ts DateTime64(3, 'UTC') ) ENGINE = MergeTree() ORDER BY (tenant_id, n)`, ); }, 60_000); afterAll(async () => { if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`); }, 60_000); it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => { const bytes = await chBytes( `SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag, toDateTime64(ts, 3, 'UTC') AS ts FROM ( SELECT arrayJoin([ ('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'), ('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'), ('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789') ]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts ) FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings(), ); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(3); expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); // Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys), // Int=2, FloatingPoint=3, Bool=6, Timestamp=10. expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); }); it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => { const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, { output_format_arrow_string_as_string: "0", }); // Arrow Type union: Binary = 4, Utf8 = 5. expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4); const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings()); expect(decodeArrowStream(asText).fields[0].typeType).toBe(5); }); it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => { const bytes = encodeArrowStream([ { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, { name: "n", type: "int64", values: [1, 2, 3] }, { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, { name: "flag", type: "bool", values: [true, false, true] }, { name: "ts", type: "timestamp_ms", values: [ new Date("2026-01-01T00:00:00.000Z"), new Date("2026-06-15T08:09:10.123Z"), new Date("2026-09-06T12:34:56.789Z"), ], }, ]); await chText(`INSERT INTO ${DB}.arrow_roundtrip FORMAT ArrowStream`, bytes as unknown as BodyInit); const back = await chText( `SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`, ); const rows = (JSON.parse(back) as { data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[]; }).data; expect(rows).toHaveLength(3); expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]); expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]); expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]); expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]); expect(rows.map((r) => r.ts)).toEqual([ "2026-01-01 00:00:00.000", "2026-06-15 08:09:10.123", "2026-09-06 12:34:56.789", ]); }); it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => { const exported = await chBytes( `SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`, arrowStreamSettings(), ); expect(decodeArrowStream(exported).rowCount).toBe(3); await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_circle AS ${DB}.arrow_roundtrip`); await chText(`INSERT INTO ${DB}.arrow_circle FORMAT ArrowStream`, exported as unknown as BodyInit); const diff = await chText( `SELECT count() AS c FROM ( SELECT * FROM ${DB}.arrow_roundtrip EXCEPT SELECT * FROM ${DB}.arrow_circle ) FORMAT JSON`, ); expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0); }); it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => { const n = 20_000; const bytes = encodeArrowStream([ { name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, { name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) }, { name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) }, { name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, { name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) }, ]); expect(decodeArrowStream(bytes).rowCount).toBe(n); await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_big AS ${DB}.arrow_roundtrip`); await chText(`INSERT INTO ${DB}.arrow_big FORMAT ArrowStream`, bytes as unknown as BodyInit); const agg = await chText( `SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`, ); const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0]; expect(Number(d.c)).toBe(n); expect(Number(d.s)).toBe((n * (n - 1)) / 2); expect(Number(d.t)).toBe(7); expect(Number(d.f)).toBe(n / 2); }, 60_000); });