/** * BI-P005 — APACHE ARROW INTERCHANGE, the part provable without a live engine. * * Honest scope note: the tests here run 0pt's encoder against 0pt's decoder, * which proves internal consistency and NOT specification conformance. The * conformance evidence comes from two places where an INDEPENDENT * implementation reads the bytes: * * * tests/integration/analytics-arrow-clickhouse.integration.test.ts — * ClickHouse (Arrow C++) both produces Arrow this decoder reads and * consumes Arrow this encoder writes. Runs on every push in CI. * * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json — pyarrow and DuckDB * decoding the same bytes, executed against real containers. * * That distinction is the point: a self-roundtrip that agrees with its own * mistakes is exactly the kind of green this estate has been burned by. */ import { describe, it, expect } from 'vitest'; import { ARROW_TYPE_SUPPORT, arrowStreamSettings, assertArrowWithinResultBudget, clickhouseArrowStreamFormat, decodeArrowStream, encodeArrowStream, } from '@/src/analytics/arrow/ipc'; describe('BI-P005 Arrow IPC', () => { it('emits a well-formed IPC stream: continuation markers, schema, batch, EOS', () => { const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3] }]); const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength); expect(dv.getUint32(0, true)).toBe(0xffffffff); // Last 8 bytes are the end-of-stream marker: 0xFFFFFFFF then length 0. expect(dv.getUint32(bytes.length - 8, true)).toBe(0xffffffff); expect(dv.getInt32(bytes.length - 4, true)).toBe(0); }); it('round-trips every supported column type with nulls', () => { const bytes = encodeArrowStream([ { name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', null] }, { name: 'n', type: 'int64', values: [1, 2, 3] }, { name: 'v', type: 'float64', values: [1.5, 2.5, null] }, { name: 'flag', type: 'bool', values: [true, false, true] }, { name: 'ts', type: 'timestamp_ms', values: [new Date('2026-01-01T00:00:00Z'), null, new Date('2026-09-06T12:34:56.789Z')], }, ]); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(3); expect(info.batchCount).toBe(1); expect(info.fields.map((f) => f.name)).toEqual(['tenant_id', 'n', 'v', 'flag', 'ts']); // Arrow Type union tags: Int=2, FloatingPoint=3, Utf8=5, Bool=6, Timestamp=10. expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); }); it('handles an empty result set without producing a malformed stream', () => { const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [] }]); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(0); expect(info.fields).toEqual([{ name: 'n', typeType: 2 }]); }); it('rejects ragged columns rather than emitting a stream that decodes to garbage', () => { expect(() => encodeArrowStream([ { name: 'a', type: 'int64', values: [1, 2] }, { name: 'b', type: 'int64', values: [1] }, ]), ).toThrowError(/expected 2/); expect(() => encodeArrowStream([])).toThrowError(/at least one column/); }); it('refuses an int64 value outside the representable range instead of wrapping it', () => { expect(() => encodeArrowStream([{ name: 'n', type: 'int64', values: [BigInt('9223372036854775808')] }]), ).toThrowError(/out of int64 range/); }); it('rejects bytes that are not an Arrow IPC stream', () => { expect(() => decodeArrowStream(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrowError( /not an Arrow IPC stream/, ); }); it('BI-P012: Arrow is not a way around max_result_rows', () => { const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3, 4, 5] }]); expect(() => assertArrowWithinResultBudget(bytes, 3)).toThrowError(/over the 3-row budget/); expect(assertArrowWithinResultBudget(bytes, 5).rowCount).toBe(5); }); it('the ClickHouse pass-through asks for text Strings, not Binary', () => { expect(clickhouseArrowStreamFormat()).toBe('ArrowStream'); // Without this setting ClickHouse emits String as Arrow Binary, which // pyarrow and DuckDB read as bytes — silent semantic loss. expect(arrowStreamSettings().output_format_arrow_string_as_string).toBe('1'); }); it('documents the supported type mapping including where loss WOULD occur', () => { expect(ARROW_TYPE_SUPPORT.length).toBeGreaterThanOrEqual(5); const arrowTypes = ARROW_TYPE_SUPPORT.map((r) => r.arrow); expect(new Set(arrowTypes)).toEqual(new Set(['int64', 'float64', 'utf8', 'bool', 'timestamp_ms'])); // Every row states its caveat; an empty note would be a silent promise. for (const row of ARROW_TYPE_SUPPORT) expect(row.note.length).toBeGreaterThan(10); }); });