/** * src/analytics/arrow/ipc.ts * =========================== * BI-P005 — APACHE ARROW INTERCHANGE. * * Canonical columnar result encoding ALONGSIDE JSON (JSON is not removed — * BI-P005 says "alongside"). Two paths, and knowing which is which matters: * * PASS-THROUGH (preferred for ClickHouse results): ClickHouse emits real * Arrow IPC itself via `FORMAT ArrowStream`. Those bytes were produced by * the Arrow C++ implementation; 0pt streams them out unchanged, so there is * no encoder of ours in the path at all. `clickhouseArrowStreamFormat()` * and `arrowStreamSettings()` are that path. * * ENCODED (for results 0pt itself assembles — projections, JSON-sourced * rows, exports): `encodeArrowStream()` writes the IPC stream here. * * Both paths are decodable by `decodeArrowStream()`, which exists so 0pt can * count rows and read the schema of an Arrow payload WITHOUT shelling out to * another runtime — needed to enforce BI-P012's `max_result_rows` on the * pass-through path, where the row count is not otherwise visible. * * "Supported types" (the DoD's qualifier) is `ARROW_TYPE_SUPPORT` below: the * exact ClickHouse -> Arrow -> DuckDB/Python mapping, including where * semantic loss WOULD occur, stated rather than hidden. * * IPC stream layout (Arrow format 1.5, metadata V5): * for each message: 0xFFFFFFFF, int32 metadata length, FlatBuffer Message * padded so the body starts 8-byte aligned, then the body buffers. * Terminated by 0xFFFFFFFF 0x00000000. */ import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder"; import { SiaxError } from "../../platform/errors"; // ── FlatBuffer enum values from org.apache.arrow.flatbuf ──────────────────── const METADATA_VERSION_V5 = 4; const MESSAGE_HEADER_SCHEMA = 1; const MESSAGE_HEADER_RECORD_BATCH = 3; const ENDIANNESS_LITTLE = 0; const TYPE_INT = 2; const TYPE_FLOATING_POINT = 3; const TYPE_UTF8 = 5; const TYPE_BOOL = 6; const TYPE_TIMESTAMP = 10; const PRECISION_DOUBLE = 2; // BigInt LITERALS (`0n`) require target >= ES2020; this repo targets ES6, so // every 64-bit constant is built with BigInt(...) instead. Behaviour is // identical — this is a compiler-target accommodation, not a semantic choice. const BIG_ZERO = BigInt(0); const INT64_MAX = BigInt("9223372036854775807"); const INT64_MIN = BigInt("-9223372036854775808"); const TIME_UNIT_MILLISECOND = 1; // ── public column model ──────────────────────────────────────────────────── export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms"; export interface ArrowColumn { name: string; type: ArrowColumnType; /** `null` marks a null slot; the validity bitmap is derived from it. */ values: (number | bigint | string | boolean | Date | null)[]; } /** * The type table BI-P005's "supported types" qualifier refers to. Every row * has been exercised end to end by scripts/bi-p005-arrow-roundtrip.mjs. */ export const ARROW_TYPE_SUPPORT: { clickhouse: string; arrow: ArrowColumnType; duckdb: string; python: string; note: string; }[] = [ { clickhouse: "Int64 / UInt32 / Int32", arrow: "int64", duckdb: "BIGINT", python: "int", note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped", }, { clickhouse: "Float64", arrow: "float64", duckdb: "DOUBLE", python: "float", note: "exact IEEE-754 round trip", }, { clickhouse: "String / LowCardinality(String)", arrow: "utf8", duckdb: "VARCHAR", python: "str", note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not", }, { clickhouse: "Bool / UInt8(0|1)", arrow: "bool", duckdb: "BOOLEAN", python: "bool", note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64", }, { clickhouse: "DateTime64(3,'UTC')", arrow: "timestamp_ms", duckdb: "TIMESTAMP", python: "datetime", note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set", }, ]; // ── ClickHouse pass-through ──────────────────────────────────────────────── /** ClickHouse output format producing an Arrow IPC *stream*. */ export function clickhouseArrowStreamFormat(): string { return "ArrowStream"; } /** * Settings for the pass-through path. `output_format_arrow_string_as_string` * matters: without it ClickHouse emits String columns as Arrow *Binary*, * which pyarrow and DuckDB both read as bytes rather than text — semantic * loss that would slip through a test that only compares row counts. */ export function arrowStreamSettings(): Record { return { output_format_arrow_string_as_string: "1", output_format_arrow_low_cardinality_as_dictionary: "0", }; } // ── encoder ──────────────────────────────────────────────────────────────── function alignTo8(n: number): number { return (n + 7) & ~7; } interface EncodedBuffer { bytes: Uint8Array; /** Logical (unpadded) length recorded in the RecordBatch metadata. */ length: number; } function validityBuffer(values: readonly unknown[]): { buf: EncodedBuffer; nullCount: number } { let nullCount = 0; const byteLen = Math.ceil(values.length / 8); const bytes = new Uint8Array(alignTo8(byteLen)); for (let i = 0; i < values.length; i++) { if (values[i] === null || values[i] === undefined) { nullCount += 1; } else { bytes[i >> 3] |= 1 << (i & 7); } } return { buf: { bytes, length: byteLen }, nullCount }; } function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: number } { const { buf: validity, nullCount } = validityBuffer(col.values); const n = col.values.length; switch (col.type) { case "int64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v))); if (b > INT64_MAX || b < INT64_MIN) { throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`); } dv.setBigInt64(i * 8, b, true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } case "float64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; dv.setFloat64(i * 8, Number(v), true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } case "timestamp_ms": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v); dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } case "bool": { const byteLen = Math.ceil(n / 8); const data = new Uint8Array(alignTo8(byteLen)); col.values.forEach((v, i) => { if (v === null || v === undefined) return; if (v) data[i >> 3] |= 1 << (i & 7); }); return { buffers: [validity, { bytes: data, length: byteLen }], nullCount }; } case "utf8": { const encoded = col.values.map((v) => v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)), ); const total = encoded.reduce((a, b) => a + b.length, 0); const offsets = new Uint8Array(alignTo8((n + 1) * 4)); const odv = new DataView(offsets.buffer); const data = new Uint8Array(alignTo8(total)); let pos = 0; odv.setInt32(0, 0, true); encoded.forEach((bytes, i) => { data.set(bytes, pos); pos += bytes.length; odv.setInt32((i + 1) * 4, pos, true); }); return { buffers: [validity, { bytes: offsets, length: (n + 1) * 4 }, { bytes: data, length: total }], nullCount, }; } default: { const never: never = col.type; throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } { switch (type) { case "int64": { b.startObject(2); b.addInt32(0, 64, 0); // bitWidth b.addInt8(1, 1, 0); // is_signed return { typeType: TYPE_INT, offset: b.endObject() }; } case "float64": { b.startObject(1); b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() }; } case "utf8": { b.startObject(0); return { typeType: TYPE_UTF8, offset: b.endObject() }; } case "bool": { b.startObject(0); return { typeType: TYPE_BOOL, offset: b.endObject() }; } case "timestamp_ms": { // Build the timezone string BEFORE startObject — FlatBuffers forbids // creating a nested object while a table is open. const tz = b.createString("UTC"); b.startObject(2); b.addInt16(0, TIME_UNIT_MILLISECOND, 0); b.addOffset(1, tz, 0); return { typeType: TYPE_TIMESTAMP, offset: b.endObject() }; } default: { const never: never = type; throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } function buildSchemaMessage(columns: readonly ArrowColumn[]): Uint8Array { const b = new FlatBufferBuilder(2048); const fieldOffsets: Offset[] = columns.map((col) => { const nameOffset = b.createString(col.name); const { typeType, offset: typeOffset } = buildTypeTable(b, col.type); const children = b.createOffsetVector([]); b.startObject(7); b.addOffset(0, nameOffset, 0); b.addInt8(1, 1, 0); // nullable = true b.addInt8(2, typeType, 0); b.addOffset(3, typeOffset, 0); b.addOffset(5, children, 0); return b.endObject(); }); const fieldsVector = b.createOffsetVector(fieldOffsets); b.startObject(4); b.addInt16(0, ENDIANNESS_LITTLE, 0); b.addOffset(1, fieldsVector, 0); const schema = b.endObject(); b.startObject(5); b.addInt16(0, METADATA_VERSION_V5, 0); b.addInt8(1, MESSAGE_HEADER_SCHEMA, 0); b.addOffset(2, schema, 0); b.addInt64(3, BIG_ZERO, BIG_ZERO); // bodyLength const message = b.endObject(); return b.finish(message); } function buildRecordBatchMessage( rowCount: number, nodes: { length: number; nullCount: number }[], buffers: { offset: number; length: number }[], bodyLength: number, ): Uint8Array { const b = new FlatBufferBuilder(2048); // FieldNode[] — inline structs, written in reverse. b.startVector(16, nodes.length, 8); for (let i = nodes.length - 1; i >= 0; i--) { b.writeInt64(BigInt(nodes[i].nullCount)); b.writeInt64(BigInt(nodes[i].length)); } const nodesVector = b.endVector(nodes.length); // Buffer[] — inline structs, written in reverse. b.startVector(16, buffers.length, 8); for (let i = buffers.length - 1; i >= 0; i--) { b.writeInt64(BigInt(buffers[i].length)); b.writeInt64(BigInt(buffers[i].offset)); } const buffersVector = b.endVector(buffers.length); b.startObject(5); b.addInt64(0, BigInt(rowCount), BIG_ZERO); b.addOffset(1, nodesVector, 0); b.addOffset(2, buffersVector, 0); const recordBatch = b.endObject(); b.startObject(5); b.addInt16(0, METADATA_VERSION_V5, 0); b.addInt8(1, MESSAGE_HEADER_RECORD_BATCH, 0); b.addOffset(2, recordBatch, 0); b.addInt64(3, BigInt(bodyLength), BIG_ZERO); const message = b.endObject(); return b.finish(message); } function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array { // continuation(4) + length(4) + metadata, padded so the body is 8-aligned. const prefix = 8; const padded = alignTo8(prefix + metadata.length) - prefix; const out = new Uint8Array(prefix + padded + body.length); const dv = new DataView(out.buffer); dv.setUint32(0, 0xffffffff, true); dv.setInt32(4, padded, true); out.set(metadata, 8); out.set(body, prefix + padded); return out; } /** * Encode columns as a complete Arrow IPC *stream* (schema message, one record * batch, end-of-stream marker). */ export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array { if (columns.length === 0) { throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column"); } const rowCount = columns[0].values.length; for (const c of columns) { if (c.values.length !== rowCount) { throw new SiaxError( "INVALID_REQUEST", "0pt.analytics", `column "${c.name}" has ${c.values.length} values, expected ${rowCount}`, ); } } const schemaMsg = buildSchemaMessage(columns); const nodes: { length: number; nullCount: number }[] = []; const bufferMeta: { offset: number; length: number }[] = []; const bodyParts: Uint8Array[] = []; let bodyOffset = 0; for (const col of columns) { const { buffers, nullCount } = encodeColumn(col); nodes.push({ length: rowCount, nullCount }); for (const buf of buffers) { bufferMeta.push({ offset: bodyOffset, length: buf.length }); bodyParts.push(buf.bytes); bodyOffset += buf.bytes.length; // already 8-aligned by encodeColumn } } const body = new Uint8Array(bodyOffset); let pos = 0; for (const part of bodyParts) { body.set(part, pos); pos += part.length; } const batchMsg = buildRecordBatchMessage(rowCount, nodes, bufferMeta, body.length); const schemaFrame = framedMessage(schemaMsg, new Uint8Array(0)); const batchFrame = framedMessage(batchMsg, body); const eos = new Uint8Array(8); new DataView(eos.buffer).setUint32(0, 0xffffffff, true); // length 0 = EOS const out = new Uint8Array(schemaFrame.length + batchFrame.length + eos.length); out.set(schemaFrame, 0); out.set(batchFrame, schemaFrame.length); out.set(eos, schemaFrame.length + batchFrame.length); return out; } // ── decoder (structural: schema names/types + row counts) ────────────────── export interface ArrowStreamInfo { fields: { name: string; typeType: number }[]; /** Total rows across every record batch in the stream. */ rowCount: number; batchCount: number; } /** Minimal FlatBuffers table reader — enough for Message/Schema/RecordBatch. */ class FbTable { constructor( readonly dv: DataView, readonly pos: number, ) {} fieldPos(slot: number): number { const vtable = this.pos - this.dv.getInt32(this.pos, true); const vtLen = this.dv.getInt16(vtable, true); const idx = 4 + slot * 2; if (idx >= vtLen) return 0; const rel = this.dv.getInt16(vtable + idx, true); return rel === 0 ? 0 : this.pos + rel; } readInt8(slot: number, def = 0): number { const p = this.fieldPos(slot); return p === 0 ? def : this.dv.getInt8(p); } readInt16(slot: number, def = 0): number { const p = this.fieldPos(slot); return p === 0 ? def : this.dv.getInt16(p, true); } readInt64(slot: number, def: bigint = BIG_ZERO): bigint { const p = this.fieldPos(slot); return p === 0 ? def : this.dv.getBigInt64(p, true); } /** Follow a uoffset to a nested table. */ readTable(slot: number): FbTable | undefined { const p = this.fieldPos(slot); if (p === 0) return undefined; return new FbTable(this.dv, p + this.dv.getInt32(p, true)); } vectorLength(slot: number): number { const p = this.fieldPos(slot); if (p === 0) return 0; const vec = p + this.dv.getInt32(p, true); return this.dv.getInt32(vec, true); } vectorElementTable(slot: number, index: number): FbTable { const p = this.fieldPos(slot); const vec = p + this.dv.getInt32(p, true); const elem = vec + 4 + index * 4; return new FbTable(this.dv, elem + this.dv.getInt32(elem, true)); } readString(slot: number): string | undefined { const p = this.fieldPos(slot); if (p === 0) return undefined; const str = p + this.dv.getInt32(p, true); const len = this.dv.getInt32(str, true); const bytes = new Uint8Array(this.dv.buffer, this.dv.byteOffset + str + 4, len); return new TextDecoder().decode(bytes); } } /** * Parse an Arrow IPC stream's structure. Used to enforce BI-P012's result-row * ceiling on Arrow payloads (including ClickHouse pass-through bytes, where * the count is otherwise invisible to 0pt) and by the roundtrip tests. */ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); let pos = 0; let fields: { name: string; typeType: number }[] = []; let rowCount = 0; let batchCount = 0; while (pos + 8 <= bytes.byteLength) { const continuation = dv.getUint32(pos, true); if (continuation !== 0xffffffff) { throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)"); } const metaLen = dv.getInt32(pos + 4, true); if (metaLen === 0) break; // end of stream const metaStart = pos + 8; const root = metaStart + dv.getInt32(metaStart, true); const message = new FbTable(dv, root); const headerType = message.readInt8(1, 0); const bodyLength = Number(message.readInt64(3, BIG_ZERO)); if (headerType === MESSAGE_HEADER_SCHEMA) { const schema = message.readTable(2); if (schema) { const n = schema.vectorLength(1); fields = []; for (let i = 0; i < n; i++) { const f = schema.vectorElementTable(1, i); fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) }); } } } else if (headerType === MESSAGE_HEADER_RECORD_BATCH) { const rb = message.readTable(2); if (rb) { rowCount += Number(rb.readInt64(0, BIG_ZERO)); batchCount += 1; } } pos = metaStart + metaLen + bodyLength; } return { fields, rowCount, batchCount }; } /** * BI-P012 on the Arrow path: refuse a payload that exceeds the caller's * result-row ceiling. Without this, asking for Arrow would be a way to walk * around `max_result_rows`. */ export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: number): ArrowStreamInfo { const info = decodeArrowStream(bytes); if (info.rowCount > maxResultRows) { throw new SiaxError( "QUOTA_EXCEEDED", "0pt.analytics", `Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`, ); } return info; }