0pt/tests/integration/analytics-arrow-clickhouse.integration.test.ts

/**
 * BI-P005 conformance against an INDEPENDENT Arrow implementation.
 *
 * The unit suite (tests/analytics/arrow-ipc.test.ts) only proves 0pt's
 * encoder agrees with 0pt's decoder — which it would even if both shared the
 * same misreading of the spec. Here, ClickHouse's Arrow C++ implementation is
 * the second opinion, in BOTH directions:
 *
 *   ClickHouse -> Arrow -> 0pt decoder   (their encoder, our reader)
 *   0pt encoder -> Arrow -> ClickHouse   (our encoder, their reader)
 *
 * The second direction is the load-bearing one: ClickHouse INSERTing our
 * bytes and then answering a SELECT with the exact values back is a claim
 * that cannot be satisfied by a subtly wrong buffer layout.
 *
 * The DoD's literal chain (ClickHouse -> Arrow -> DuckDB/Python) additionally
 * ran against real pyarrow + DuckDB; that transcript is committed at
 * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json.
 */
import { describe, it, expect, beforeAll, afterAll } from "vitest";
import {
  arrowStreamSettings,
  clickhouseArrowStreamFormat,
  decodeArrowStream,
  encodeArrowStream,
} from "@/src/analytics/arrow/ipc";

const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;

const DB = `opt_arrow_${Date.now().toString(36)}`;

function authHeaders(): Record<string, string> {
  return CLICKHOUSE_AUTH
    ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` }
    : {};
}

async function chBytes(sql: string, settings: Record<string, string> = {}): Promise<Uint8Array> {
  const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
  url.searchParams.set("query", sql);
  for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
  const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" });
  const buf = new Uint8Array(await res.arrayBuffer());
  if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`);
  return buf;
}

async function chText(sql: string, body?: BodyInit): Promise<string> {
  const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid");
  url.searchParams.set("query", sql);
  const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" });
  const text = await res.text();
  if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`);
  return text;
}

run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => {
  beforeAll(async () => {
    await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`);
    await chText(
      `CREATE TABLE IF NOT EXISTS ${DB}.arrow_roundtrip (
         tenant_id String, n Int64, v Float64, flag Bool, ts DateTime64(3, 'UTC')
       ) ENGINE = MergeTree() ORDER BY (tenant_id, n)`,
    );
  }, 60_000);

  afterAll(async () => {
    if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`);
  }, 60_000);

  it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => {
    const bytes = await chBytes(
      `SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag,
              toDateTime64(ts, 3, 'UTC') AS ts
       FROM (
         SELECT arrayJoin([
           ('t_alpha', 1, 1.5, true,  '2026-01-01 00:00:00.000'),
           ('t_beta',  2, 2.5, false, '2026-06-15 08:09:10.123'),
           ('t_gamma', 3, 3.25, true, '2026-09-06 12:34:56.789')
         ]) AS tup, tup.1 AS t, tup.2 AS n, tup.3 AS v, tup.4 AS b, tup.5 AS ts
       )
       FORMAT ${clickhouseArrowStreamFormat()}`,
      arrowStreamSettings(),
    );
    const info = decodeArrowStream(bytes);
    expect(info.rowCount).toBe(3);
    expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]);
    // Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys),
    // Int=2, FloatingPoint=3, Bool=6, Timestamp=10.
    expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]);
  });

  it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => {
    const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, {
      output_format_arrow_string_as_string: "0",
    });
    // Arrow Type union: Binary = 4, Utf8 = 5.
    expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4);
    const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings());
    expect(decodeArrowStream(asText).fields[0].typeType).toBe(5);
  });

  it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => {
    const bytes = encodeArrowStream([
      { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] },
      { name: "n", type: "int64", values: [1, 2, 3] },
      { name: "v", type: "float64", values: [1.5, 2.5, 3.25] },
      { name: "flag", type: "bool", values: [true, false, true] },
      {
        name: "ts",
        type: "timestamp_ms",
        values: [
          new Date("2026-01-01T00:00:00.000Z"),
          new Date("2026-06-15T08:09:10.123Z"),
          new Date("2026-09-06T12:34:56.789Z"),
        ],
      },
    ]);
    await chText(`INSERT INTO ${DB}.arrow_roundtrip FORMAT ArrowStream`, bytes as unknown as BodyInit);

    const back = await chText(
      `SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`,
    );
    const rows = (JSON.parse(back) as {
      data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[];
    }).data;
    expect(rows).toHaveLength(3);
    expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]);
    expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]);
    expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]);
    expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]);
    expect(rows.map((r) => r.ts)).toEqual([
      "2026-01-01 00:00:00.000",
      "2026-06-15 08:09:10.123",
      "2026-09-06 12:34:56.789",
    ]);
  });

  it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => {
    const exported = await chBytes(
      `SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`,
      arrowStreamSettings(),
    );
    expect(decodeArrowStream(exported).rowCount).toBe(3);

    await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_circle AS ${DB}.arrow_roundtrip`);
    await chText(`INSERT INTO ${DB}.arrow_circle FORMAT ArrowStream`, exported as unknown as BodyInit);

    const diff = await chText(
      `SELECT count() AS c FROM (
         SELECT * FROM ${DB}.arrow_roundtrip
         EXCEPT
         SELECT * FROM ${DB}.arrow_circle
       ) FORMAT JSON`,
    );
    expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0);
  });

  it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => {
    const n = 20_000;
    const bytes = encodeArrowStream([
      { name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) },
      { name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) },
      { name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) },
      { name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) },
      { name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) },
    ]);
    expect(decodeArrowStream(bytes).rowCount).toBe(n);

    await chText(`CREATE TABLE IF NOT EXISTS ${DB}.arrow_big AS ${DB}.arrow_roundtrip`);
    await chText(`INSERT INTO ${DB}.arrow_big FORMAT ArrowStream`, bytes as unknown as BodyInit);
    const agg = await chText(
      `SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`,
    );
    const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0];
    expect(Number(d.c)).toBe(n);
    expect(Number(d.s)).toBe((n * (n - 1)) / 2);
    expect(Number(d.t)).toBe(7);
    expect(Number(d.f)).toBe(n / 2);
  }, 60_000);
});