Files
0pt/scripts/bi-p005-arrow-roundtrip.mjs

162 lines
6.4 KiB
JavaScript
Executable File

#!/usr/bin/env node
/**
* BI-P005 DoD, executed literally:
*
* "Same query result roundtrips ClickHouse -> Arrow -> DuckDB/Python
* without semantic loss for supported types."
*
* Pulls a result from a LIVE ClickHouse as Arrow IPC, hands the identical
* bytes to pyarrow and DuckDB, and records what each of them read back. Also
* runs the reverse direction — 0pt's own encoder's bytes decoded by pyarrow +
* DuckDB — because the encoder is hand-rolled and the only conformance
* evidence worth having comes from somebody else's parser.
*
* CLICKHOUSE_URL=http://localhost:18123 CLICKHOUSE_AUTH=user:pass \
* node scripts/bi-p005-arrow-roundtrip.mjs
*
* pyarrow/DuckDB run in a throwaway `python:3.12-slim` container, so the host
* needs docker but no Python packages. Output:
* evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json
*
* The 0pt-encoded fixture is produced by `pnpm vitest run
* tests/analytics/arrow-fixture.emit.ts`-style code inlined here via a tiny
* tsx-free path: the encoder is re-implemented ONLY in the sense that this
* script shells out to vitest to write the file, so the bytes tested are the
* bytes the library produces.
*/
import fs from "node:fs";
import path from "node:path";
import os from "node:os";
import { execFileSync } from "node:child_process";
const URL_ = process.env.CLICKHOUSE_URL;
const AUTH = process.env.CLICKHOUSE_AUTH;
if (!URL_) {
console.error("CLICKHOUSE_URL is required — this script measures, it does not simulate.");
process.exit(2);
}
const work = fs.mkdtempSync(path.join(os.tmpdir(), "bi-p005-"));
const headers = AUTH ? { Authorization: `Basic ${Buffer.from(AUTH).toString("base64")}` } : {};
const QUERY = `
SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag,
toDateTime64(ts, 3, 'UTC') AS ts
FROM (
SELECT arrayJoin([
('t_alpha', 1, 1.5, true, '2026-01-01 00:00:00.000'),
('t_beta', 2, 2.5, false, '2026-06-15 08:09:10.123'),
('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789')
]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts
)`;
async function chArrow(sql, settings) {
const u = new URL(URL_);
u.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings)) u.searchParams.set(k, v);
const res = await fetch(u.toString(), { method: "POST", headers, body: "" });
const buf = Buffer.from(await res.arrayBuffer());
if (!res.ok) throw new Error(`CH ${res.status}: ${buf.toString("utf8").slice(0, 300)}`);
return buf;
}
const PY = `
import pyarrow as pa, pyarrow.ipc as ipc, duckdb, json, sys, os
out = {}
for name in sorted(os.listdir('/w')):
if not name.endswith('.arrow'):
continue
tbl = ipc.open_stream(pa.BufferReader(open('/w/'+name,'rb').read())).read_all()
con = duckdb.connect(); con.register('t', tbl)
out[name] = {
"pyarrow_version": pa.__version__,
"duckdb_version": duckdb.__version__,
"schema": [{"name": f.name, "type": str(f.type)} for f in tbl.schema],
"num_rows": tbl.num_rows,
"pyarrow_values": json.loads(json.dumps(tbl.to_pydict(), default=str)),
"duckdb_types": [(r[0], r[1]) for r in con.execute("describe select * from t").fetchall()],
"duckdb_rows": [[str(c) for c in row] for row in con.execute("select * from t order by n").fetchall()],
}
print(json.dumps(out))
`;
async function main() {
const version = (await (async () => {
const u = new URL(URL_);
u.searchParams.set("query", "SELECT version() FORMAT TSVRaw");
const r = await fetch(u.toString(), { method: "POST", headers, body: "" });
return (await r.text()).trim();
})());
// Direction 1: ClickHouse's own Arrow C++ output.
const chBytes = await chArrow(`${QUERY} FORMAT ArrowStream`, {
output_format_arrow_string_as_string: "1",
output_format_arrow_low_cardinality_as_dictionary: "0",
});
fs.writeFileSync(path.join(work, "01_clickhouse_encoded.arrow"), chBytes);
// Direction 2: 0pt's own encoder. Driven through the SHIPPED module via
// tests/analytics/arrow-fixture.test.ts (ARROW_FIXTURE_DIR) so the bytes
// pyarrow sees are the library's, not a copy pasted into this script.
execFileSync(
"npx",
["vitest", "run", "tests/analytics/arrow-fixture.test.ts"],
{ stdio: "inherit", env: { ...process.env, ARROW_FIXTURE_DIR: work } },
);
// The reader script goes in as a FILE, not as `python -c "..."`: passing a
// multi-line program through a shell argument mangles the newlines.
fs.writeFileSync(path.join(work, "read_arrow.py"), PY);
const raw = execFileSync(
"docker",
[
"run", "--rm", "-v", `${work}:/w`, "-w", "/w", "python:3.12-slim",
"bash", "-lc",
"pip install --quiet pyarrow duckdb pytz >/dev/null 2>&1 && python /w/read_arrow.py",
],
{ encoding: "utf8", maxBuffer: 32 * 1024 * 1024 },
);
const decoded = JSON.parse(raw);
const evidence = {
generated_at: new Date().toISOString(),
clickhouse_version: version,
query: QUERY.trim(),
directions: {
"clickhouse -> arrow -> pyarrow/duckdb": decoded["01_clickhouse_encoded.arrow"],
"0pt encoder -> arrow -> pyarrow/duckdb": decoded["02_0pt_encoded.arrow"],
},
byte_sizes: {
clickhouse_encoded: chBytes.length,
opt_encoded: fs.statSync(path.join(work, "02_0pt_encoded.arrow")).size,
},
verdict: null,
};
// The DoD is "without semantic loss": the two independent readers must agree
// on the SAME values for both encoders.
const a = evidence.directions["clickhouse -> arrow -> pyarrow/duckdb"];
const b = evidence.directions["0pt encoder -> arrow -> pyarrow/duckdb"];
const same =
JSON.stringify(a.pyarrow_values) === JSON.stringify(b.pyarrow_values) &&
JSON.stringify(a.duckdb_rows) === JSON.stringify(b.duckdb_rows) &&
JSON.stringify(a.duckdb_types) === JSON.stringify(b.duckdb_types);
evidence.verdict = same
? "PASS — pyarrow and DuckDB read identical values and types from BOTH encoders"
: "FAIL — the two encoders do not round-trip to the same values";
const out = path.join(process.cwd(), "evidence", "bi-p002-p012", "bi-p005-arrow-roundtrip.json");
fs.mkdirSync(path.dirname(out), { recursive: true });
fs.writeFileSync(out, JSON.stringify(evidence, null, 2) + "\n");
console.log(evidence.verdict);
console.log(`wrote ${out}`);
if (!same) process.exitCode = 1;
}
main()
.catch((e) => {
console.error(e);
process.exitCode = 1;
})
.finally(() => fs.rmSync(work, { recursive: true, force: true }));