Files
0pt/src/analytics/arrow/ipc.ts
T

548 lines
19 KiB
TypeScript

/**
* src/analytics/arrow/ipc.ts
* ===========================
* BI-P005 — APACHE ARROW INTERCHANGE.
*
* Canonical columnar result encoding ALONGSIDE JSON (JSON is not removed —
* BI-P005 says "alongside"). Two paths, and knowing which is which matters:
*
* PASS-THROUGH (preferred for ClickHouse results): ClickHouse emits real
* Arrow IPC itself via `FORMAT ArrowStream`. Those bytes were produced by
* the Arrow C++ implementation; 0pt streams them out unchanged, so there is
* no encoder of ours in the path at all. `clickhouseArrowStreamFormat()`
* and `arrowStreamSettings()` are that path.
*
* ENCODED (for results 0pt itself assembles — projections, JSON-sourced
* rows, exports): `encodeArrowStream()` writes the IPC stream here.
*
* Both paths are decodable by `decodeArrowStream()`, which exists so 0pt can
* count rows and read the schema of an Arrow payload WITHOUT shelling out to
* another runtime — needed to enforce BI-P012's `max_result_rows` on the
* pass-through path, where the row count is not otherwise visible.
*
* "Supported types" (the DoD's qualifier) is `ARROW_TYPE_SUPPORT` below: the
* exact ClickHouse -> Arrow -> DuckDB/Python mapping, including where
* semantic loss WOULD occur, stated rather than hidden.
*
* IPC stream layout (Arrow format 1.5, metadata V5):
* for each message: 0xFFFFFFFF, int32 metadata length, FlatBuffer Message
* padded so the body starts 8-byte aligned, then the body buffers.
* Terminated by 0xFFFFFFFF 0x00000000.
*/
import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder";
import { SiaxError } from "../../platform/errors";
// ── FlatBuffer enum values from org.apache.arrow.flatbuf ────────────────────
const METADATA_VERSION_V5 = 4;
const MESSAGE_HEADER_SCHEMA = 1;
const MESSAGE_HEADER_RECORD_BATCH = 3;
const ENDIANNESS_LITTLE = 0;
const TYPE_INT = 2;
const TYPE_FLOATING_POINT = 3;
const TYPE_UTF8 = 5;
const TYPE_BOOL = 6;
const TYPE_TIMESTAMP = 10;
const PRECISION_DOUBLE = 2;
// BigInt LITERALS (`0n`) require target >= ES2020; this repo targets ES6, so
// every 64-bit constant is built with BigInt(...) instead. Behaviour is
// identical — this is a compiler-target accommodation, not a semantic choice.
const BIG_ZERO = BigInt(0);
const INT64_MAX = BigInt("9223372036854775807");
const INT64_MIN = BigInt("-9223372036854775808");
const TIME_UNIT_MILLISECOND = 1;
// ── public column model ────────────────────────────────────────────────────
export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms";
export interface ArrowColumn {
name: string;
type: ArrowColumnType;
/** `null` marks a null slot; the validity bitmap is derived from it. */
values: (number | bigint | string | boolean | Date | null)[];
}
/**
* The type table BI-P005's "supported types" qualifier refers to. Every row
* has been exercised end to end by scripts/bi-p005-arrow-roundtrip.mjs.
*/
export const ARROW_TYPE_SUPPORT: {
clickhouse: string;
arrow: ArrowColumnType;
duckdb: string;
python: string;
note: string;
}[] = [
{
clickhouse: "Int64 / UInt32 / Int32",
arrow: "int64",
duckdb: "BIGINT",
python: "int",
note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped",
},
{
clickhouse: "Float64",
arrow: "float64",
duckdb: "DOUBLE",
python: "float",
note: "exact IEEE-754 round trip",
},
{
clickhouse: "String / LowCardinality(String)",
arrow: "utf8",
duckdb: "VARCHAR",
python: "str",
note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not",
},
{
clickhouse: "Bool / UInt8(0|1)",
arrow: "bool",
duckdb: "BOOLEAN",
python: "bool",
note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64",
},
{
clickhouse: "DateTime64(3,'UTC')",
arrow: "timestamp_ms",
duckdb: "TIMESTAMP",
python: "datetime",
note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set",
},
];
// ── ClickHouse pass-through ────────────────────────────────────────────────
/** ClickHouse output format producing an Arrow IPC *stream*. */
export function clickhouseArrowStreamFormat(): string {
return "ArrowStream";
}
/**
* Settings for the pass-through path. `output_format_arrow_string_as_string`
* matters: without it ClickHouse emits String columns as Arrow *Binary*,
* which pyarrow and DuckDB both read as bytes rather than text — semantic
* loss that would slip through a test that only compares row counts.
*/
export function arrowStreamSettings(): Record<string, string> {
return {
output_format_arrow_string_as_string: "1",
output_format_arrow_low_cardinality_as_dictionary: "0",
};
}
// ── encoder ────────────────────────────────────────────────────────────────
function alignTo8(n: number): number {
return (n + 7) & ~7;
}
interface EncodedBuffer {
bytes: Uint8Array;
/** Logical (unpadded) length recorded in the RecordBatch metadata. */
length: number;
}
function validityBuffer(values: readonly unknown[]): { buf: EncodedBuffer; nullCount: number } {
let nullCount = 0;
const byteLen = Math.ceil(values.length / 8);
const bytes = new Uint8Array(alignTo8(byteLen));
for (let i = 0; i < values.length; i++) {
if (values[i] === null || values[i] === undefined) {
nullCount += 1;
} else {
bytes[i >> 3] |= 1 << (i & 7);
}
}
return { buf: { bytes, length: byteLen }, nullCount };
}
function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: number } {
const { buf: validity, nullCount } = validityBuffer(col.values);
const n = col.values.length;
switch (col.type) {
case "int64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v)));
if (b > INT64_MAX || b < INT64_MIN) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`);
}
dv.setBigInt64(i * 8, b, true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "float64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
dv.setFloat64(i * 8, Number(v), true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "timestamp_ms": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v);
dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "bool": {
const byteLen = Math.ceil(n / 8);
const data = new Uint8Array(alignTo8(byteLen));
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
if (v) data[i >> 3] |= 1 << (i & 7);
});
return { buffers: [validity, { bytes: data, length: byteLen }], nullCount };
}
case "utf8": {
const encoded = col.values.map((v) =>
v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)),
);
const total = encoded.reduce((a, b) => a + b.length, 0);
const offsets = new Uint8Array(alignTo8((n + 1) * 4));
const odv = new DataView(offsets.buffer);
const data = new Uint8Array(alignTo8(total));
let pos = 0;
odv.setInt32(0, 0, true);
encoded.forEach((bytes, i) => {
data.set(bytes, pos);
pos += bytes.length;
odv.setInt32((i + 1) * 4, pos, true);
});
return {
buffers: [validity, { bytes: offsets, length: (n + 1) * 4 }, { bytes: data, length: total }],
nullCount,
};
}
default: {
const never: never = col.type;
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } {
switch (type) {
case "int64": {
b.startObject(2);
b.addInt32(0, 64, 0); // bitWidth
b.addInt8(1, 1, 0); // is_signed
return { typeType: TYPE_INT, offset: b.endObject() };
}
case "float64": {
b.startObject(1);
b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE
return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() };
}
case "utf8": {
b.startObject(0);
return { typeType: TYPE_UTF8, offset: b.endObject() };
}
case "bool": {
b.startObject(0);
return { typeType: TYPE_BOOL, offset: b.endObject() };
}
case "timestamp_ms": {
// Build the timezone string BEFORE startObject — FlatBuffers forbids
// creating a nested object while a table is open.
const tz = b.createString("UTC");
b.startObject(2);
b.addInt16(0, TIME_UNIT_MILLISECOND, 0);
b.addOffset(1, tz, 0);
return { typeType: TYPE_TIMESTAMP, offset: b.endObject() };
}
default: {
const never: never = type;
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
function buildSchemaMessage(columns: readonly ArrowColumn[]): Uint8Array {
const b = new FlatBufferBuilder(2048);
const fieldOffsets: Offset[] = columns.map((col) => {
const nameOffset = b.createString(col.name);
const { typeType, offset: typeOffset } = buildTypeTable(b, col.type);
const children = b.createOffsetVector([]);
b.startObject(7);
b.addOffset(0, nameOffset, 0);
b.addInt8(1, 1, 0); // nullable = true
b.addInt8(2, typeType, 0);
b.addOffset(3, typeOffset, 0);
b.addOffset(5, children, 0);
return b.endObject();
});
const fieldsVector = b.createOffsetVector(fieldOffsets);
b.startObject(4);
b.addInt16(0, ENDIANNESS_LITTLE, 0);
b.addOffset(1, fieldsVector, 0);
const schema = b.endObject();
b.startObject(5);
b.addInt16(0, METADATA_VERSION_V5, 0);
b.addInt8(1, MESSAGE_HEADER_SCHEMA, 0);
b.addOffset(2, schema, 0);
b.addInt64(3, BIG_ZERO, BIG_ZERO); // bodyLength
const message = b.endObject();
return b.finish(message);
}
function buildRecordBatchMessage(
rowCount: number,
nodes: { length: number; nullCount: number }[],
buffers: { offset: number; length: number }[],
bodyLength: number,
): Uint8Array {
const b = new FlatBufferBuilder(2048);
// FieldNode[] — inline structs, written in reverse.
b.startVector(16, nodes.length, 8);
for (let i = nodes.length - 1; i >= 0; i--) {
b.writeInt64(BigInt(nodes[i].nullCount));
b.writeInt64(BigInt(nodes[i].length));
}
const nodesVector = b.endVector(nodes.length);
// Buffer[] — inline structs, written in reverse.
b.startVector(16, buffers.length, 8);
for (let i = buffers.length - 1; i >= 0; i--) {
b.writeInt64(BigInt(buffers[i].length));
b.writeInt64(BigInt(buffers[i].offset));
}
const buffersVector = b.endVector(buffers.length);
b.startObject(5);
b.addInt64(0, BigInt(rowCount), BIG_ZERO);
b.addOffset(1, nodesVector, 0);
b.addOffset(2, buffersVector, 0);
const recordBatch = b.endObject();
b.startObject(5);
b.addInt16(0, METADATA_VERSION_V5, 0);
b.addInt8(1, MESSAGE_HEADER_RECORD_BATCH, 0);
b.addOffset(2, recordBatch, 0);
b.addInt64(3, BigInt(bodyLength), BIG_ZERO);
const message = b.endObject();
return b.finish(message);
}
function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array {
// continuation(4) + length(4) + metadata, padded so the body is 8-aligned.
const prefix = 8;
const padded = alignTo8(prefix + metadata.length) - prefix;
const out = new Uint8Array(prefix + padded + body.length);
const dv = new DataView(out.buffer);
dv.setUint32(0, 0xffffffff, true);
dv.setInt32(4, padded, true);
out.set(metadata, 8);
out.set(body, prefix + padded);
return out;
}
/**
* Encode columns as a complete Arrow IPC *stream* (schema message, one record
* batch, end-of-stream marker).
*/
export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array {
if (columns.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column");
}
const rowCount = columns[0].values.length;
for (const c of columns) {
if (c.values.length !== rowCount) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`column "${c.name}" has ${c.values.length} values, expected ${rowCount}`,
);
}
}
const schemaMsg = buildSchemaMessage(columns);
const nodes: { length: number; nullCount: number }[] = [];
const bufferMeta: { offset: number; length: number }[] = [];
const bodyParts: Uint8Array[] = [];
let bodyOffset = 0;
for (const col of columns) {
const { buffers, nullCount } = encodeColumn(col);
nodes.push({ length: rowCount, nullCount });
for (const buf of buffers) {
bufferMeta.push({ offset: bodyOffset, length: buf.length });
bodyParts.push(buf.bytes);
bodyOffset += buf.bytes.length; // already 8-aligned by encodeColumn
}
}
const body = new Uint8Array(bodyOffset);
let pos = 0;
for (const part of bodyParts) {
body.set(part, pos);
pos += part.length;
}
const batchMsg = buildRecordBatchMessage(rowCount, nodes, bufferMeta, body.length);
const schemaFrame = framedMessage(schemaMsg, new Uint8Array(0));
const batchFrame = framedMessage(batchMsg, body);
const eos = new Uint8Array(8);
new DataView(eos.buffer).setUint32(0, 0xffffffff, true); // length 0 = EOS
const out = new Uint8Array(schemaFrame.length + batchFrame.length + eos.length);
out.set(schemaFrame, 0);
out.set(batchFrame, schemaFrame.length);
out.set(eos, schemaFrame.length + batchFrame.length);
return out;
}
// ── decoder (structural: schema names/types + row counts) ──────────────────
export interface ArrowStreamInfo {
fields: { name: string; typeType: number }[];
/** Total rows across every record batch in the stream. */
rowCount: number;
batchCount: number;
}
/** Minimal FlatBuffers table reader — enough for Message/Schema/RecordBatch. */
class FbTable {
constructor(
readonly dv: DataView,
readonly pos: number,
) {}
fieldPos(slot: number): number {
const vtable = this.pos - this.dv.getInt32(this.pos, true);
const vtLen = this.dv.getInt16(vtable, true);
const idx = 4 + slot * 2;
if (idx >= vtLen) return 0;
const rel = this.dv.getInt16(vtable + idx, true);
return rel === 0 ? 0 : this.pos + rel;
}
readInt8(slot: number, def = 0): number {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getInt8(p);
}
readInt16(slot: number, def = 0): number {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getInt16(p, true);
}
readInt64(slot: number, def: bigint = BIG_ZERO): bigint {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getBigInt64(p, true);
}
/** Follow a uoffset to a nested table. */
readTable(slot: number): FbTable | undefined {
const p = this.fieldPos(slot);
if (p === 0) return undefined;
return new FbTable(this.dv, p + this.dv.getInt32(p, true));
}
vectorLength(slot: number): number {
const p = this.fieldPos(slot);
if (p === 0) return 0;
const vec = p + this.dv.getInt32(p, true);
return this.dv.getInt32(vec, true);
}
vectorElementTable(slot: number, index: number): FbTable {
const p = this.fieldPos(slot);
const vec = p + this.dv.getInt32(p, true);
const elem = vec + 4 + index * 4;
return new FbTable(this.dv, elem + this.dv.getInt32(elem, true));
}
readString(slot: number): string | undefined {
const p = this.fieldPos(slot);
if (p === 0) return undefined;
const str = p + this.dv.getInt32(p, true);
const len = this.dv.getInt32(str, true);
const bytes = new Uint8Array(this.dv.buffer, this.dv.byteOffset + str + 4, len);
return new TextDecoder().decode(bytes);
}
}
/**
* Parse an Arrow IPC stream's structure. Used to enforce BI-P012's result-row
* ceiling on Arrow payloads (including ClickHouse pass-through bytes, where
* the count is otherwise invisible to 0pt) and by the roundtrip tests.
*/
export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo {
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
let pos = 0;
let fields: { name: string; typeType: number }[] = [];
let rowCount = 0;
let batchCount = 0;
while (pos + 8 <= bytes.byteLength) {
const continuation = dv.getUint32(pos, true);
if (continuation !== 0xffffffff) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)");
}
const metaLen = dv.getInt32(pos + 4, true);
if (metaLen === 0) break; // end of stream
const metaStart = pos + 8;
const root = metaStart + dv.getInt32(metaStart, true);
const message = new FbTable(dv, root);
const headerType = message.readInt8(1, 0);
const bodyLength = Number(message.readInt64(3, BIG_ZERO));
if (headerType === MESSAGE_HEADER_SCHEMA) {
const schema = message.readTable(2);
if (schema) {
const n = schema.vectorLength(1);
fields = [];
for (let i = 0; i < n; i++) {
const f = schema.vectorElementTable(1, i);
fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) });
}
}
} else if (headerType === MESSAGE_HEADER_RECORD_BATCH) {
const rb = message.readTable(2);
if (rb) {
rowCount += Number(rb.readInt64(0, BIG_ZERO));
batchCount += 1;
}
}
pos = metaStart + metaLen + bodyLength;
}
return { fields, rowCount, batchCount };
}
/**
* BI-P012 on the Arrow path: refuse a payload that exceeds the caller's
* result-row ceiling. Without this, asking for Arrow would be a way to walk
* around `max_result_rows`.
*/
export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: number): ArrowStreamInfo {
const info = decodeArrowStream(bytes);
if (info.rowCount > maxResultRows) {
throw new SiaxError(
"QUOTA_EXCEEDED",
"0pt.analytics",
`Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`,
);
}
return info;
}