/** * src/analytics/arrow/flatbuffer-builder.ts * ========================================== * A minimal FlatBuffers writer, sufficient for the Arrow IPC metadata * messages this repo emits (BI-P005). * * WHY HAND-ROLLED rather than `apache-arrow` from npm: adding a runtime * dependency to 0pt's blocking-typechecked capability layer means a lockfile * change, a new supply-chain surface, and a new CVE surface for every * consumer of this repo — for a format whose METADATA side is a few hundred * bytes of well-specified binary and whose DATA side is plain columnar memory * either way. * * The correctness bar is NOT "this looks right". The BI-P005 evidence script * (scripts/bi-p005-arrow-roundtrip.mjs) decodes this writer's output with * pyarrow AND DuckDB — two independent implementations of the specification — * so a mistake here surfaces as a failed decode by somebody else's parser, * not as a green test of my own assumptions. * * Layout facts implemented (FlatBuffers binary format, little-endian): * * the buffer is built BACK TO FRONT; `offset` counts bytes from the end; * * a uoffset stored at a field site holds `site_offset + 4 - target_offset` * so a reader resolves it as `position + value`; * * a table begins with an soffset `table_start - vtable_start`; * * a vtable is `[vtable_len:i16][table_len:i16][field_offsets:i16...]`, * with 0 meaning "field absent, use the default". */ export type Offset = number; const SIZEOF_INT = 4; export class FlatBufferBuilder { private buf: Uint8Array; private view: DataView; /** Bytes written so far, counted from the END of `buf`. */ private used = 0; private minAlign = 1; private vtable: number[] = []; private objectStart = 0; private inObject = false; /** Offsets of previously written vtables, for deduplication. */ private vtables: number[] = []; constructor(initial = 1024) { this.buf = new Uint8Array(initial); this.view = new DataView(this.buf.buffer); } /** Bytes written so far (FlatBuffers "offset"). */ get offset(): number { return this.used; } private ensure(extra: number): void { if (this.used + extra <= this.buf.length) return; let size = this.buf.length; while (size < this.used + extra) size *= 2; const next = new Uint8Array(size); next.set(this.buf.subarray(this.buf.length - this.used), size - this.used); this.buf = next; this.view = new DataView(this.buf.buffer); } private pad(n: number): void { if (n <= 0) return; this.ensure(n); for (let i = 0; i < n; i++) { this.used += 1; this.buf[this.buf.length - this.used] = 0; } } /** FlatBuffers `prep`: pad so that `additional` more bytes land aligned. */ private prep(size: number, additional = 0): void { if (size > this.minAlign) this.minAlign = size; const alignSize = (~(this.used + additional) + 1) & (size - 1); this.pad(alignSize); } writeInt8(v: number): void { this.ensure(1); this.used += 1; this.view.setInt8(this.buf.length - this.used, v); } writeInt16(v: number): void { this.ensure(2); this.used += 2; this.view.setInt16(this.buf.length - this.used, v, true); } writeInt32(v: number): void { this.ensure(4); this.used += 4; this.view.setInt32(this.buf.length - this.used, v, true); } writeInt64(v: bigint): void { this.ensure(8); this.used += 8; this.view.setBigInt64(this.buf.length - this.used, v, true); } writeBytes(bytes: Uint8Array): void { this.ensure(bytes.length); this.used += bytes.length; this.buf.set(bytes, this.buf.length - this.used); } addInt8(slot: number, value: number, def: number): void { if (value === def) return; this.prep(1); this.writeInt8(value); this.slot(slot); } addInt16(slot: number, value: number, def: number): void { if (value === def) return; this.prep(2); this.writeInt16(value); this.slot(slot); } addInt32(slot: number, value: number, def: number): void { if (value === def) return; this.prep(4); this.writeInt32(value); this.slot(slot); } addInt64(slot: number, value: bigint, def: bigint): void { if (value === def) return; this.prep(8); this.writeInt64(value); this.slot(slot); } addOffset(slot: number, value: Offset, def: number): void { if (value === def) return; this.prep(SIZEOF_INT); this.writeInt32(this.used + SIZEOF_INT - value); this.slot(slot); } private slot(slotNum: number): void { this.vtable[slotNum] = this.used; } startObject(numFields: number): void { if (this.inObject) throw new Error('nested startObject'); this.vtable = new Array(numFields).fill(0); this.objectStart = this.used; this.inObject = true; } endObject(): Offset { if (!this.inObject) throw new Error('endObject outside an object'); this.prep(SIZEOF_INT); this.writeInt32(0); // soffset placeholder, patched below const objectOffset = this.used; let trimmed = this.vtable.length; while (trimmed > 0 && this.vtable[trimmed - 1] === 0) trimmed -= 1; for (let i = trimmed - 1; i >= 0; i--) { this.writeInt16(this.vtable[i] !== 0 ? objectOffset - this.vtable[i] : 0); } this.writeInt16(objectOffset - this.objectStart); // table byte length const vtLen = (trimmed + 2) * 2; this.writeInt16(vtLen); // vtable byte length — lands at the vtable start const vtStart = this.used; const vtAbs = this.buf.length - vtStart; let existing = 0; for (const candidate of this.vtables) { const candAbs = this.buf.length - candidate; if (this.view.getInt16(candAbs, true) !== vtLen) continue; let same = true; for (let i = 2; i < vtLen; i += 2) { if (this.view.getInt16(vtAbs + i, true) !== this.view.getInt16(candAbs + i, true)) { same = false; break; } } if (same) { existing = candidate; break; } } const objAbs = this.buf.length - objectOffset; if (existing !== 0) { this.used -= vtLen; // discard the duplicate vtable just written this.view.setInt32(objAbs, existing - objectOffset, true); } else { this.vtables.push(vtStart); this.view.setInt32(objAbs, this.used - objectOffset, true); } this.inObject = false; return objectOffset; } createString(s: string): Offset { const bytes = new TextEncoder().encode(s); this.writeInt8(0); // null terminator (readers rely on it) this.startVector(1, bytes.length, 1); this.writeBytes(bytes); return this.endVector(bytes.length); } startVector(elemSize: number, numElems: number, alignment: number): void { this.prep(SIZEOF_INT, elemSize * numElems); this.prep(alignment, elemSize * numElems); } endVector(numElems: number): Offset { this.writeInt32(numElems); return this.used; } /** Vector of table offsets. Elements are written in reverse build order. */ createOffsetVector(items: Offset[]): Offset { this.startVector(SIZEOF_INT, items.length, SIZEOF_INT); for (let i = items.length - 1; i >= 0; i--) { this.writeInt32(this.used + SIZEOF_INT - items[i]); } return this.endVector(items.length); } /** Finish the buffer with `root` as the root table. Returns the bytes. */ finish(root: Offset): Uint8Array { this.prep(this.minAlign, SIZEOF_INT); this.prep(SIZEOF_INT); this.writeInt32(this.used + SIZEOF_INT - root); return this.buf.slice(this.buf.length - this.used); } }