Files
0pt/tests/integration/analytics-arrow-clickhouse.integration.test.ts
Claude Opus 5 c864c8c417
CI (SIAX Cloud) / integration (push) Successful in 40s
CI (SIAX Cloud) / quality (push) Successful in 55s
CI (SIAX Cloud) / security (push) Successful in 21s
Security Scan (SIAX Cloud) / security (push) Successful in 22s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 54s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 21s
CI (SIAX Cloud) / security (pull_request) Successful in 25s
fix(analytics): revert 18 files reformatted by mistake, and actually apply the Iceberg fix a previous message claimed
Two corrections, both of my own errors.

1. SCOPE CREEP, NOW REVERTED. Running `prettier --write` on whole directories
   reformatted 18 files this branch had no business touching — the entire
   merged BI-P002..P012 core (arrow/, domain/, engine/, repositories/, five
   analytics integration suites) and, absurdly,
   search-opensearch.integration.test.ts, which has nothing to do with this
   work. The changes were purely stylistic quote churn, but they added ~1,400
   lines of noise to the diff, made the PR harder to review honestly, and
   invited conflicts with the four other lanes working in this estate. All 18
   are now byte-identical to origin/main again (verified with `git diff
   origin/main --name-only`, which reports nothing for them). Formatting is now
   applied only to the files this branch actually owns.

2. A COMMIT MESSAGE THAT DESCRIBED A FIX THAT WAS NOT IN THE CODE. Commit
   3ee0230 said the "does NOT ship the Iceberg table" assertion had been moved
   from a filesystem check to `git ls-files`. It had not. The patch anchor used
   double quotes, prettier had already converted that file to single quotes, so
   the replacement silently matched nothing and the script printed its success
   line anyway. The suite then passed — but only because I had just deleted the
   fixture directory, so the OLD filesystem assertion happened to be true.

   That is precisely the failure mode this branch keeps writing tests against:
   a green run that means something different from what it appears to mean. The
   fix is now genuinely applied and verified the way it should have been the
   first time — by running the suite WITH the Iceberg fixture present on disk,
   which is the case that previously failed. It passes, and it now also asserts
   that the parquet/csv fixtures ARE tracked, so a version shipping no fixture
   at all would fail rather than pass vacuously.

957 tests pass with ClickHouse, Postgres and DuckDB live. typecheck, eslint and
prettier clean on the files this branch owns.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
2026-09-07 03:16:30 +02:00

184 lines
8.2 KiB
TypeScript

/**
* BI-P005 conformance against an INDEPENDENT Arrow implementation.
*
* The unit suite (tests/analytics/arrow-ipc.test.ts) only proves 0pt's
* encoder agrees with 0pt's decoder — which it would even if both shared the
* same misreading of the spec. Here, ClickHouse's Arrow C++ implementation is
* the second opinion, in BOTH directions:
*
* ClickHouse -> Arrow -> 0pt decoder (their encoder, our reader)
* 0pt encoder -> Arrow -> ClickHouse (our encoder, their reader)
*
* The second direction is the load-bearing one: ClickHouse INSERTing our
* bytes and then answering a SELECT with the exact values back is a claim
* that cannot be satisfied by a subtly wrong buffer layout.
*
* The DoD's literal chain (ClickHouse -> Arrow -> DuckDB/Python) additionally
* ran against real pyarrow + DuckDB; that transcript is committed at
* evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json.
*/
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import {
arrowStreamSettings,
clickhouseArrowStreamFormat,
decodeArrowStream,
encodeArrowStream,
} from "@/src/analytics/arrow/ipc";
const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;
const DB = `opt_arrow_${Date.now().toString(36)}`;
function authHeaders(): Record<string, string> {
return CLICKHOUSE_AUTH
? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` }
: {};
}
async function chBytes(sql: string, settings: Record<string, string> = {}): Promise<Uint8Array> {
const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" });
const buf = new Uint8Array(await res.arrayBuffer());
if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`);
return buf;
}
async function chText(sql: string, body?: BodyInit): Promise<string> {
const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
url.searchParams.set("query", sql);
const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" });
const text = await res.text();
if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`);
return text;
}
run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => {
beforeAll(async () => {
await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`);
await chText(
`CREATE TABLE IF NOT EXISTS ${DB}.arrow_roundtrip (
tenant_id String, n Int64, v Float64, flag Bool, ts DateTime64(3, 'UTC')
) ENGINE = MergeTree() ORDER BY (tenant_id, n)`,
);
}, 60_000);
afterAll(async () => {
if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`);
}, 60_000);
it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => {
const bytes = await chBytes(
`SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag,
toDateTime64(ts, 3, 'UTC') AS ts
FROM (
SELECT arrayJoin([
('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'),
('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'),
('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789')
]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts
)
FORMAT ${clickhouseArrowStreamFormat()}`,
arrowStreamSettings(),
);
const info = decodeArrowStream(bytes);
expect(info.rowCount).toBe(3);
expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]);
// Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys),
// Int=2, FloatingPoint=3, Bool=6, Timestamp=10.
expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]);
});
it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => {
const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, {
output_format_arrow_string_as_string: "0",
});
// Arrow Type union: Binary = 4, Utf8 = 5.
expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4);
const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings());
expect(decodeArrowStream(asText).fields[0].typeType).toBe(5);
});
it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => {
const bytes = encodeArrowStream([
{ name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] },
{ name: "n", type: "int64", values: [1, 2, 3] },
{ name: "v", type: "float64", values: [1.5, 2.5, 3.25] },
{ name: "flag", type: "bool", values: [true, false, true] },
{
name: "ts",
type: "timestamp_ms",
values: [
new Date("2026-01-01T00:00:00.000Z"),
new Date("2026-06-15T08:09:10.123Z"),
new Date("2026-09-06T12:34:56.789Z"),
],
},
]);
await chText(`INSERT INTO ${DB}.arrow_roundtrip FORMAT ArrowStream`, bytes as unknown as BodyInit);
const back = await chText(
`SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`,
);
const rows = (JSON.parse(back) as {
data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[];
}).data;
expect(rows).toHaveLength(3);
expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]);
expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]);
expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]);
expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]);
expect(rows.map((r) => r.ts)).toEqual([
"2026-01-01 00:00:00.000",
"2026-06-15 08:09:10.123",
"2026-09-06 12:34:56.789",
]);
});
it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => {
const exported = await chBytes(
`SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`,
arrowStreamSettings(),
);
expect(decodeArrowStream(exported).rowCount).toBe(3);
await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_circle AS ${DB}.arrow_roundtrip`);
await chText(`INSERT INTO ${DB}.arrow_circle FORMAT ArrowStream`, exported as unknown as BodyInit);
const diff = await chText(
`SELECT count() AS c FROM (
SELECT * FROM ${DB}.arrow_roundtrip
EXCEPT
SELECT * FROM ${DB}.arrow_circle
) FORMAT JSON`,
);
expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0);
});
it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => {
const n = 20_000;
const bytes = encodeArrowStream([
{ name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) },
{ name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) },
{ name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) },
{ name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) },
{ name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) },
]);
expect(decodeArrowStream(bytes).rowCount).toBe(n);
await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_big AS ${DB}.arrow_roundtrip`);
await chText(`INSERT INTO ${DB}.arrow_big FORMAT ArrowStream`, bytes as unknown as BodyInit);
const agg = await chText(
`SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`,
);
const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0];
expect(Number(d.c)).toBe(n);
expect(Number(d.s)).toBe((n * (n - 1)) / 2);
expect(Number(d.t)).toBe(7);
expect(Number(d.f)).toBe(n / 2);
}, 60_000);
});