Files
0pt/tests/integration/analytics-arrow-clickhouse.integration.test.ts
T

184 lines
8.2 KiB
TypeScript

/**
* BI-P005 conformance against an INDEPENDENT Arrow implementation.
*
* The unit suite (tests/analytics/arrow-ipc.test.ts) only proves 0pt's
* encoder agrees with 0pt's decoder — which it would even if both shared the
* same misreading of the spec. Here, ClickHouse's Arrow C++ implementation is
* the second opinion, in BOTH directions:
*
* ClickHouse -> Arrow -> 0pt decoder (their encoder, our reader)
* 0pt encoder -> Arrow -> ClickHouse (our encoder, their reader)
*
* The second direction is the load-bearing one: ClickHouse INSERTing our
* bytes and then answering a SELECT with the exact values back is a claim
* that cannot be satisfied by a subtly wrong buffer layout.
*
* The DoD's literal chain (ClickHouse -> Arrow -> DuckDB/Python) additionally
* ran against real pyarrow + DuckDB; that transcript is committed at
* evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json.
*/
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import {
arrowStreamSettings,
clickhouseArrowStreamFormat,
decodeArrowStream,
encodeArrowStream,
} from "@/src/analytics/arrow/ipc";
const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;
const DB = `opt_arrow_${Date.now().toString(36)}`;
function authHeaders(): Record<string, string> {
return CLICKHOUSE_AUTH
? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` }
: {};
}
async function chBytes(sql: string, settings: Record<string, string> = {}): Promise<Uint8Array> {
const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" });
const buf = new Uint8Array(await res.arrayBuffer());
if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`);
return buf;
}
async function chText(sql: string, body?: BodyInit): Promise<string> {
const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
url.searchParams.set("query", sql);
const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" });
const text = await res.text();
if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`);
return text;
}
run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => {
beforeAll(async () => {
await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`);
await chText(
`CREATE TABLE IF NOT EXISTS ${DB}.arrow_roundtrip (
tenant_id String, n Int64, v Float64, flag Bool, ts DateTime64(3, 'UTC')
) ENGINE = MergeTree() ORDER BY (tenant_id, n)`,
);
}, 60_000);
afterAll(async () => {
if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`);
}, 60_000);
it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => {
const bytes = await chBytes(
`SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag,
toDateTime64(ts, 3, 'UTC') AS ts
FROM (
SELECT arrayJoin([
('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'),
('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'),
('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789')
]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts
)
FORMAT ${clickhouseArrowStreamFormat()}`,
arrowStreamSettings(),
);
const info = decodeArrowStream(bytes);
expect(info.rowCount).toBe(3);
expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]);
// Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys),
// Int=2, FloatingPoint=3, Bool=6, Timestamp=10.
expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]);
});
it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => {
const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, {
output_format_arrow_string_as_string: "0",
});
// Arrow Type union: Binary = 4, Utf8 = 5.
expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4);
const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings());
expect(decodeArrowStream(asText).fields[0].typeType).toBe(5);
});
it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => {
const bytes = encodeArrowStream([
{ name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] },
{ name: "n", type: "int64", values: [1, 2, 3] },
{ name: "v", type: "float64", values: [1.5, 2.5, 3.25] },
{ name: "flag", type: "bool", values: [true, false, true] },
{
name: "ts",
type: "timestamp_ms",
values: [
new Date("2026-01-01T00:00:00.000Z"),
new Date("2026-06-15T08:09:10.123Z"),
new Date("2026-09-06T12:34:56.789Z"),
],
},
]);
await chText(`INSERT INTO ${DB}.arrow_roundtrip FORMAT ArrowStream`, bytes as unknown as BodyInit);
const back = await chText(
`SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`,
);
const rows = (JSON.parse(back) as {
data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[];
}).data;
expect(rows).toHaveLength(3);
expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]);
expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]);
expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]);
expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]);
expect(rows.map((r) => r.ts)).toEqual([
"2026-01-01 00:00:00.000",
"2026-06-15 08:09:10.123",
"2026-09-06 12:34:56.789",
]);
});
it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => {
const exported = await chBytes(
`SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`,
arrowStreamSettings(),
);
expect(decodeArrowStream(exported).rowCount).toBe(3);
await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_circle AS ${DB}.arrow_roundtrip`);
await chText(`INSERT INTO ${DB}.arrow_circle FORMAT ArrowStream`, exported as unknown as BodyInit);
const diff = await chText(
`SELECT count() AS c FROM (
SELECT * FROM ${DB}.arrow_roundtrip
EXCEPT
SELECT * FROM ${DB}.arrow_circle
) FORMAT JSON`,
);
expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0);
});
it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => {
const n = 20_000;
const bytes = encodeArrowStream([
{ name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) },
{ name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) },
{ name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) },
{ name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) },
{ name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) },
]);
expect(decodeArrowStream(bytes).rowCount).toBe(n);
await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_big AS ${DB}.arrow_roundtrip`);
await chText(`INSERT INTO ${DB}.arrow_big FORMAT ArrowStream`, bytes as unknown as BodyInit);
const agg = await chText(
`SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`,
);
const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0];
expect(Number(d.c)).toBe(n);
expect(Number(d.s)).toBe((n * (n - 1)) / 2);
expect(Number(d.t)).toBe(7);
expect(Number(d.f)).toBe(n / 2);
}, 60_000);
});