108 lines
4.8 KiB
TypeScript
108 lines
4.8 KiB
TypeScript
/**
|
|
* BI-P005 — APACHE ARROW INTERCHANGE, the part provable without a live engine.
|
|
*
|
|
* Honest scope note: the tests here run 0pt's encoder against 0pt's decoder,
|
|
* which proves internal consistency and NOT specification conformance. The
|
|
* conformance evidence comes from two places where an INDEPENDENT
|
|
* implementation reads the bytes:
|
|
*
|
|
* * tests/integration/analytics-arrow-clickhouse.integration.test.ts —
|
|
* ClickHouse (Arrow C++) both produces Arrow this decoder reads and
|
|
* consumes Arrow this encoder writes. Runs on every push in CI.
|
|
* * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json — pyarrow and DuckDB
|
|
* decoding the same bytes, executed against real containers.
|
|
*
|
|
* That distinction is the point: a self-roundtrip that agrees with its own
|
|
* mistakes is exactly the kind of green this estate has been burned by.
|
|
*/
|
|
import { describe, it, expect } from 'vitest';
|
|
import {
|
|
ARROW_TYPE_SUPPORT,
|
|
arrowStreamSettings,
|
|
assertArrowWithinResultBudget,
|
|
clickhouseArrowStreamFormat,
|
|
decodeArrowStream,
|
|
encodeArrowStream,
|
|
} from '@/src/analytics/arrow/ipc';
|
|
|
|
describe('BI-P005 Arrow IPC', () => {
|
|
it('emits a well-formed IPC stream: continuation markers, schema, batch, EOS', () => {
|
|
const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3] }]);
|
|
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
|
expect(dv.getUint32(0, true)).toBe(0xffffffff);
|
|
// Last 8 bytes are the end-of-stream marker: 0xFFFFFFFF then length 0.
|
|
expect(dv.getUint32(bytes.length - 8, true)).toBe(0xffffffff);
|
|
expect(dv.getInt32(bytes.length - 4, true)).toBe(0);
|
|
});
|
|
|
|
it('round-trips every supported column type with nulls', () => {
|
|
const bytes = encodeArrowStream([
|
|
{ name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', null] },
|
|
{ name: 'n', type: 'int64', values: [1, 2, 3] },
|
|
{ name: 'v', type: 'float64', values: [1.5, 2.5, null] },
|
|
{ name: 'flag', type: 'bool', values: [true, false, true] },
|
|
{
|
|
name: 'ts',
|
|
type: 'timestamp_ms',
|
|
values: [new Date('2026-01-01T00:00:00Z'), null, new Date('2026-09-06T12:34:56.789Z')],
|
|
},
|
|
]);
|
|
const info = decodeArrowStream(bytes);
|
|
expect(info.rowCount).toBe(3);
|
|
expect(info.batchCount).toBe(1);
|
|
expect(info.fields.map((f) => f.name)).toEqual(['tenant_id', 'n', 'v', 'flag', 'ts']);
|
|
// Arrow Type union tags: Int=2, FloatingPoint=3, Utf8=5, Bool=6, Timestamp=10.
|
|
expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]);
|
|
});
|
|
|
|
it('handles an empty result set without producing a malformed stream', () => {
|
|
const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [] }]);
|
|
const info = decodeArrowStream(bytes);
|
|
expect(info.rowCount).toBe(0);
|
|
expect(info.fields).toEqual([{ name: 'n', typeType: 2 }]);
|
|
});
|
|
|
|
it('rejects ragged columns rather than emitting a stream that decodes to garbage', () => {
|
|
expect(() =>
|
|
encodeArrowStream([
|
|
{ name: 'a', type: 'int64', values: [1, 2] },
|
|
{ name: 'b', type: 'int64', values: [1] },
|
|
]),
|
|
).toThrowError(/expected 2/);
|
|
expect(() => encodeArrowStream([])).toThrowError(/at least one column/);
|
|
});
|
|
|
|
it('refuses an int64 value outside the representable range instead of wrapping it', () => {
|
|
expect(() =>
|
|
encodeArrowStream([{ name: 'n', type: 'int64', values: [BigInt('9223372036854775808')] }]),
|
|
).toThrowError(/out of int64 range/);
|
|
});
|
|
|
|
it('rejects bytes that are not an Arrow IPC stream', () => {
|
|
expect(() => decodeArrowStream(new Uint8Array([1, 2, 3, 4, 5, 6, 7, 8]))).toThrowError(
|
|
/not an Arrow IPC stream/,
|
|
);
|
|
});
|
|
|
|
it('BI-P012: Arrow is not a way around max_result_rows', () => {
|
|
const bytes = encodeArrowStream([{ name: 'n', type: 'int64', values: [1, 2, 3, 4, 5] }]);
|
|
expect(() => assertArrowWithinResultBudget(bytes, 3)).toThrowError(/over the 3-row budget/);
|
|
expect(assertArrowWithinResultBudget(bytes, 5).rowCount).toBe(5);
|
|
});
|
|
|
|
it('the ClickHouse pass-through asks for text Strings, not Binary', () => {
|
|
expect(clickhouseArrowStreamFormat()).toBe('ArrowStream');
|
|
// Without this setting ClickHouse emits String as Arrow Binary, which
|
|
// pyarrow and DuckDB read as bytes — silent semantic loss.
|
|
expect(arrowStreamSettings().output_format_arrow_string_as_string).toBe('1');
|
|
});
|
|
|
|
it('documents the supported type mapping including where loss WOULD occur', () => {
|
|
expect(ARROW_TYPE_SUPPORT.length).toBeGreaterThanOrEqual(5);
|
|
const arrowTypes = ARROW_TYPE_SUPPORT.map((r) => r.arrow);
|
|
expect(new Set(arrowTypes)).toEqual(new Set(['int64', 'float64', 'utf8', 'bool', 'timestamp_ms']));
|
|
// Every row states its caveat; an empty note would be a silent promise.
|
|
for (const row of ARROW_TYPE_SUPPORT) expect(row.note.length).toBeGreaterThan(10);
|
|
});
|
|
});
|