fix(recovery): restore source removed by production documentation merge
This commit is contained in:
@@ -0,0 +1,237 @@
|
||||
/**
|
||||
* src/analytics/arrow/flatbuffer-builder.ts
|
||||
* ==========================================
|
||||
* A minimal FlatBuffers writer, sufficient for the Arrow IPC metadata
|
||||
* messages this repo emits (BI-P005).
|
||||
*
|
||||
* WHY HAND-ROLLED rather than `apache-arrow` from npm: adding a runtime
|
||||
* dependency to 0pt's blocking-typechecked capability layer means a lockfile
|
||||
* change, a new supply-chain surface, and a new CVE surface for every
|
||||
* consumer of this repo — for a format whose METADATA side is a few hundred
|
||||
* bytes of well-specified binary and whose DATA side is plain columnar memory
|
||||
* either way.
|
||||
*
|
||||
* The correctness bar is NOT "this looks right". The BI-P005 evidence script
|
||||
* (scripts/bi-p005-arrow-roundtrip.mjs) decodes this writer's output with
|
||||
* pyarrow AND DuckDB — two independent implementations of the specification —
|
||||
* so a mistake here surfaces as a failed decode by somebody else's parser,
|
||||
* not as a green test of my own assumptions.
|
||||
*
|
||||
* Layout facts implemented (FlatBuffers binary format, little-endian):
|
||||
* * the buffer is built BACK TO FRONT; `offset` counts bytes from the end;
|
||||
* * a uoffset stored at a field site holds `site_offset + 4 - target_offset`
|
||||
* so a reader resolves it as `position + value`;
|
||||
* * a table begins with an soffset `table_start - vtable_start`;
|
||||
* * a vtable is `[vtable_len:i16][table_len:i16][field_offsets:i16...]`,
|
||||
* with 0 meaning "field absent, use the default".
|
||||
*/
|
||||
|
||||
export type Offset = number;
|
||||
|
||||
const SIZEOF_INT = 4;
|
||||
|
||||
export class FlatBufferBuilder {
|
||||
private buf: Uint8Array;
|
||||
private view: DataView;
|
||||
/** Bytes written so far, counted from the END of `buf`. */
|
||||
private used = 0;
|
||||
private minAlign = 1;
|
||||
private vtable: number[] = [];
|
||||
private objectStart = 0;
|
||||
private inObject = false;
|
||||
/** Offsets of previously written vtables, for deduplication. */
|
||||
private vtables: number[] = [];
|
||||
|
||||
constructor(initial = 1024) {
|
||||
this.buf = new Uint8Array(initial);
|
||||
this.view = new DataView(this.buf.buffer);
|
||||
}
|
||||
|
||||
/** Bytes written so far (FlatBuffers "offset"). */
|
||||
get offset(): number {
|
||||
return this.used;
|
||||
}
|
||||
|
||||
private ensure(extra: number): void {
|
||||
if (this.used + extra <= this.buf.length) return;
|
||||
let size = this.buf.length;
|
||||
while (size < this.used + extra) size *= 2;
|
||||
const next = new Uint8Array(size);
|
||||
next.set(this.buf.subarray(this.buf.length - this.used), size - this.used);
|
||||
this.buf = next;
|
||||
this.view = new DataView(this.buf.buffer);
|
||||
}
|
||||
|
||||
private pad(n: number): void {
|
||||
if (n <= 0) return;
|
||||
this.ensure(n);
|
||||
for (let i = 0; i < n; i++) {
|
||||
this.used += 1;
|
||||
this.buf[this.buf.length - this.used] = 0;
|
||||
}
|
||||
}
|
||||
|
||||
/** FlatBuffers `prep`: pad so that `additional` more bytes land aligned. */
|
||||
private prep(size: number, additional = 0): void {
|
||||
if (size > this.minAlign) this.minAlign = size;
|
||||
const alignSize = (~(this.used + additional) + 1) & (size - 1);
|
||||
this.pad(alignSize);
|
||||
}
|
||||
|
||||
writeInt8(v: number): void {
|
||||
this.ensure(1);
|
||||
this.used += 1;
|
||||
this.view.setInt8(this.buf.length - this.used, v);
|
||||
}
|
||||
|
||||
writeInt16(v: number): void {
|
||||
this.ensure(2);
|
||||
this.used += 2;
|
||||
this.view.setInt16(this.buf.length - this.used, v, true);
|
||||
}
|
||||
|
||||
writeInt32(v: number): void {
|
||||
this.ensure(4);
|
||||
this.used += 4;
|
||||
this.view.setInt32(this.buf.length - this.used, v, true);
|
||||
}
|
||||
|
||||
writeInt64(v: bigint): void {
|
||||
this.ensure(8);
|
||||
this.used += 8;
|
||||
this.view.setBigInt64(this.buf.length - this.used, v, true);
|
||||
}
|
||||
|
||||
writeBytes(bytes: Uint8Array): void {
|
||||
this.ensure(bytes.length);
|
||||
this.used += bytes.length;
|
||||
this.buf.set(bytes, this.buf.length - this.used);
|
||||
}
|
||||
|
||||
addInt8(slot: number, value: number, def: number): void {
|
||||
if (value === def) return;
|
||||
this.prep(1);
|
||||
this.writeInt8(value);
|
||||
this.slot(slot);
|
||||
}
|
||||
|
||||
addInt16(slot: number, value: number, def: number): void {
|
||||
if (value === def) return;
|
||||
this.prep(2);
|
||||
this.writeInt16(value);
|
||||
this.slot(slot);
|
||||
}
|
||||
|
||||
addInt32(slot: number, value: number, def: number): void {
|
||||
if (value === def) return;
|
||||
this.prep(4);
|
||||
this.writeInt32(value);
|
||||
this.slot(slot);
|
||||
}
|
||||
|
||||
addInt64(slot: number, value: bigint, def: bigint): void {
|
||||
if (value === def) return;
|
||||
this.prep(8);
|
||||
this.writeInt64(value);
|
||||
this.slot(slot);
|
||||
}
|
||||
|
||||
addOffset(slot: number, value: Offset, def: number): void {
|
||||
if (value === def) return;
|
||||
this.prep(SIZEOF_INT);
|
||||
this.writeInt32(this.used + SIZEOF_INT - value);
|
||||
this.slot(slot);
|
||||
}
|
||||
|
||||
private slot(slotNum: number): void {
|
||||
this.vtable[slotNum] = this.used;
|
||||
}
|
||||
|
||||
startObject(numFields: number): void {
|
||||
if (this.inObject) throw new Error("nested startObject");
|
||||
this.vtable = new Array<number>(numFields).fill(0);
|
||||
this.objectStart = this.used;
|
||||
this.inObject = true;
|
||||
}
|
||||
|
||||
endObject(): Offset {
|
||||
if (!this.inObject) throw new Error("endObject outside an object");
|
||||
this.prep(SIZEOF_INT);
|
||||
this.writeInt32(0); // soffset placeholder, patched below
|
||||
const objectOffset = this.used;
|
||||
|
||||
let trimmed = this.vtable.length;
|
||||
while (trimmed > 0 && this.vtable[trimmed - 1] === 0) trimmed -= 1;
|
||||
for (let i = trimmed - 1; i >= 0; i--) {
|
||||
this.writeInt16(this.vtable[i] !== 0 ? objectOffset - this.vtable[i] : 0);
|
||||
}
|
||||
this.writeInt16(objectOffset - this.objectStart); // table byte length
|
||||
const vtLen = (trimmed + 2) * 2;
|
||||
this.writeInt16(vtLen); // vtable byte length — lands at the vtable start
|
||||
|
||||
const vtStart = this.used;
|
||||
const vtAbs = this.buf.length - vtStart;
|
||||
let existing = 0;
|
||||
for (const candidate of this.vtables) {
|
||||
const candAbs = this.buf.length - candidate;
|
||||
if (this.view.getInt16(candAbs, true) !== vtLen) continue;
|
||||
let same = true;
|
||||
for (let i = 2; i < vtLen; i += 2) {
|
||||
if (this.view.getInt16(vtAbs + i, true) !== this.view.getInt16(candAbs + i, true)) {
|
||||
same = false;
|
||||
break;
|
||||
}
|
||||
}
|
||||
if (same) {
|
||||
existing = candidate;
|
||||
break;
|
||||
}
|
||||
}
|
||||
|
||||
const objAbs = this.buf.length - objectOffset;
|
||||
if (existing !== 0) {
|
||||
this.used -= vtLen; // discard the duplicate vtable just written
|
||||
this.view.setInt32(objAbs, existing - objectOffset, true);
|
||||
} else {
|
||||
this.vtables.push(vtStart);
|
||||
this.view.setInt32(objAbs, this.used - objectOffset, true);
|
||||
}
|
||||
this.inObject = false;
|
||||
return objectOffset;
|
||||
}
|
||||
|
||||
createString(s: string): Offset {
|
||||
const bytes = new TextEncoder().encode(s);
|
||||
this.writeInt8(0); // null terminator (readers rely on it)
|
||||
this.startVector(1, bytes.length, 1);
|
||||
this.writeBytes(bytes);
|
||||
return this.endVector(bytes.length);
|
||||
}
|
||||
|
||||
startVector(elemSize: number, numElems: number, alignment: number): void {
|
||||
this.prep(SIZEOF_INT, elemSize * numElems);
|
||||
this.prep(alignment, elemSize * numElems);
|
||||
}
|
||||
|
||||
endVector(numElems: number): Offset {
|
||||
this.writeInt32(numElems);
|
||||
return this.used;
|
||||
}
|
||||
|
||||
/** Vector of table offsets. Elements are written in reverse build order. */
|
||||
createOffsetVector(items: Offset[]): Offset {
|
||||
this.startVector(SIZEOF_INT, items.length, SIZEOF_INT);
|
||||
for (let i = items.length - 1; i >= 0; i--) {
|
||||
this.writeInt32(this.used + SIZEOF_INT - items[i]);
|
||||
}
|
||||
return this.endVector(items.length);
|
||||
}
|
||||
|
||||
/** Finish the buffer with `root` as the root table. Returns the bytes. */
|
||||
finish(root: Offset): Uint8Array {
|
||||
this.prep(this.minAlign, SIZEOF_INT);
|
||||
this.prep(SIZEOF_INT);
|
||||
this.writeInt32(this.used + SIZEOF_INT - root);
|
||||
return this.buf.slice(this.buf.length - this.used);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,547 @@
|
||||
/**
|
||||
* src/analytics/arrow/ipc.ts
|
||||
* ===========================
|
||||
* BI-P005 — APACHE ARROW INTERCHANGE.
|
||||
*
|
||||
* Canonical columnar result encoding ALONGSIDE JSON (JSON is not removed —
|
||||
* BI-P005 says "alongside"). Two paths, and knowing which is which matters:
|
||||
*
|
||||
* PASS-THROUGH (preferred for ClickHouse results): ClickHouse emits real
|
||||
* Arrow IPC itself via `FORMAT ArrowStream`. Those bytes were produced by
|
||||
* the Arrow C++ implementation; 0pt streams them out unchanged, so there is
|
||||
* no encoder of ours in the path at all. `clickhouseArrowStreamFormat()`
|
||||
* and `arrowStreamSettings()` are that path.
|
||||
*
|
||||
* ENCODED (for results 0pt itself assembles — projections, JSON-sourced
|
||||
* rows, exports): `encodeArrowStream()` writes the IPC stream here.
|
||||
*
|
||||
* Both paths are decodable by `decodeArrowStream()`, which exists so 0pt can
|
||||
* count rows and read the schema of an Arrow payload WITHOUT shelling out to
|
||||
* another runtime — needed to enforce BI-P012's `max_result_rows` on the
|
||||
* pass-through path, where the row count is not otherwise visible.
|
||||
*
|
||||
* "Supported types" (the DoD's qualifier) is `ARROW_TYPE_SUPPORT` below: the
|
||||
* exact ClickHouse -> Arrow -> DuckDB/Python mapping, including where
|
||||
* semantic loss WOULD occur, stated rather than hidden.
|
||||
*
|
||||
* IPC stream layout (Arrow format 1.5, metadata V5):
|
||||
* for each message: 0xFFFFFFFF, int32 metadata length, FlatBuffer Message
|
||||
* padded so the body starts 8-byte aligned, then the body buffers.
|
||||
* Terminated by 0xFFFFFFFF 0x00000000.
|
||||
*/
|
||||
import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder";
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
|
||||
// ── FlatBuffer enum values from org.apache.arrow.flatbuf ────────────────────
|
||||
|
||||
const METADATA_VERSION_V5 = 4;
|
||||
const MESSAGE_HEADER_SCHEMA = 1;
|
||||
const MESSAGE_HEADER_RECORD_BATCH = 3;
|
||||
const ENDIANNESS_LITTLE = 0;
|
||||
|
||||
const TYPE_INT = 2;
|
||||
const TYPE_FLOATING_POINT = 3;
|
||||
const TYPE_UTF8 = 5;
|
||||
const TYPE_BOOL = 6;
|
||||
const TYPE_TIMESTAMP = 10;
|
||||
|
||||
const PRECISION_DOUBLE = 2;
|
||||
|
||||
// BigInt LITERALS (`0n`) require target >= ES2020; this repo targets ES6, so
|
||||
// every 64-bit constant is built with BigInt(...) instead. Behaviour is
|
||||
// identical — this is a compiler-target accommodation, not a semantic choice.
|
||||
const BIG_ZERO = BigInt(0);
|
||||
const INT64_MAX = BigInt("9223372036854775807");
|
||||
const INT64_MIN = BigInt("-9223372036854775808");
|
||||
const TIME_UNIT_MILLISECOND = 1;
|
||||
|
||||
// ── public column model ────────────────────────────────────────────────────
|
||||
|
||||
export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms";
|
||||
|
||||
export interface ArrowColumn {
|
||||
name: string;
|
||||
type: ArrowColumnType;
|
||||
/** `null` marks a null slot; the validity bitmap is derived from it. */
|
||||
values: (number | bigint | string | boolean | Date | null)[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The type table BI-P005's "supported types" qualifier refers to. Every row
|
||||
* has been exercised end to end by scripts/bi-p005-arrow-roundtrip.mjs.
|
||||
*/
|
||||
export const ARROW_TYPE_SUPPORT: {
|
||||
clickhouse: string;
|
||||
arrow: ArrowColumnType;
|
||||
duckdb: string;
|
||||
python: string;
|
||||
note: string;
|
||||
}[] = [
|
||||
{
|
||||
clickhouse: "Int64 / UInt32 / Int32",
|
||||
arrow: "int64",
|
||||
duckdb: "BIGINT",
|
||||
python: "int",
|
||||
note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped",
|
||||
},
|
||||
{
|
||||
clickhouse: "Float64",
|
||||
arrow: "float64",
|
||||
duckdb: "DOUBLE",
|
||||
python: "float",
|
||||
note: "exact IEEE-754 round trip",
|
||||
},
|
||||
{
|
||||
clickhouse: "String / LowCardinality(String)",
|
||||
arrow: "utf8",
|
||||
duckdb: "VARCHAR",
|
||||
python: "str",
|
||||
note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not",
|
||||
},
|
||||
{
|
||||
clickhouse: "Bool / UInt8(0|1)",
|
||||
arrow: "bool",
|
||||
duckdb: "BOOLEAN",
|
||||
python: "bool",
|
||||
note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64",
|
||||
},
|
||||
{
|
||||
clickhouse: "DateTime64(3,'UTC')",
|
||||
arrow: "timestamp_ms",
|
||||
duckdb: "TIMESTAMP",
|
||||
python: "datetime",
|
||||
note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set",
|
||||
},
|
||||
];
|
||||
|
||||
// ── ClickHouse pass-through ────────────────────────────────────────────────
|
||||
|
||||
/** ClickHouse output format producing an Arrow IPC *stream*. */
|
||||
export function clickhouseArrowStreamFormat(): string {
|
||||
return "ArrowStream";
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings for the pass-through path. `output_format_arrow_string_as_string`
|
||||
* matters: without it ClickHouse emits String columns as Arrow *Binary*,
|
||||
* which pyarrow and DuckDB both read as bytes rather than text — semantic
|
||||
* loss that would slip through a test that only compares row counts.
|
||||
*/
|
||||
export function arrowStreamSettings(): Record<string, string> {
|
||||
return {
|
||||
output_format_arrow_string_as_string: "1",
|
||||
output_format_arrow_low_cardinality_as_dictionary: "0",
|
||||
};
|
||||
}
|
||||
|
||||
// ── encoder ────────────────────────────────────────────────────────────────
|
||||
|
||||
function alignTo8(n: number): number {
|
||||
return (n + 7) & ~7;
|
||||
}
|
||||
|
||||
interface EncodedBuffer {
|
||||
bytes: Uint8Array;
|
||||
/** Logical (unpadded) length recorded in the RecordBatch metadata. */
|
||||
length: number;
|
||||
}
|
||||
|
||||
function validityBuffer(values: readonly unknown[]): { buf: EncodedBuffer; nullCount: number } {
|
||||
let nullCount = 0;
|
||||
const byteLen = Math.ceil(values.length / 8);
|
||||
const bytes = new Uint8Array(alignTo8(byteLen));
|
||||
for (let i = 0; i < values.length; i++) {
|
||||
if (values[i] === null || values[i] === undefined) {
|
||||
nullCount += 1;
|
||||
} else {
|
||||
bytes[i >> 3] |= 1 << (i & 7);
|
||||
}
|
||||
}
|
||||
return { buf: { bytes, length: byteLen }, nullCount };
|
||||
}
|
||||
|
||||
function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: number } {
|
||||
const { buf: validity, nullCount } = validityBuffer(col.values);
|
||||
const n = col.values.length;
|
||||
|
||||
switch (col.type) {
|
||||
case "int64": {
|
||||
const data = new Uint8Array(alignTo8(n * 8));
|
||||
const dv = new DataView(data.buffer);
|
||||
col.values.forEach((v, i) => {
|
||||
if (v === null || v === undefined) return;
|
||||
const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v)));
|
||||
if (b > INT64_MAX || b < INT64_MIN) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`);
|
||||
}
|
||||
dv.setBigInt64(i * 8, b, true);
|
||||
});
|
||||
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
|
||||
}
|
||||
case "float64": {
|
||||
const data = new Uint8Array(alignTo8(n * 8));
|
||||
const dv = new DataView(data.buffer);
|
||||
col.values.forEach((v, i) => {
|
||||
if (v === null || v === undefined) return;
|
||||
dv.setFloat64(i * 8, Number(v), true);
|
||||
});
|
||||
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
|
||||
}
|
||||
case "timestamp_ms": {
|
||||
const data = new Uint8Array(alignTo8(n * 8));
|
||||
const dv = new DataView(data.buffer);
|
||||
col.values.forEach((v, i) => {
|
||||
if (v === null || v === undefined) return;
|
||||
const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v);
|
||||
dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true);
|
||||
});
|
||||
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
|
||||
}
|
||||
case "bool": {
|
||||
const byteLen = Math.ceil(n / 8);
|
||||
const data = new Uint8Array(alignTo8(byteLen));
|
||||
col.values.forEach((v, i) => {
|
||||
if (v === null || v === undefined) return;
|
||||
if (v) data[i >> 3] |= 1 << (i & 7);
|
||||
});
|
||||
return { buffers: [validity, { bytes: data, length: byteLen }], nullCount };
|
||||
}
|
||||
case "utf8": {
|
||||
const encoded = col.values.map((v) =>
|
||||
v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)),
|
||||
);
|
||||
const total = encoded.reduce((a, b) => a + b.length, 0);
|
||||
const offsets = new Uint8Array(alignTo8((n + 1) * 4));
|
||||
const odv = new DataView(offsets.buffer);
|
||||
const data = new Uint8Array(alignTo8(total));
|
||||
let pos = 0;
|
||||
odv.setInt32(0, 0, true);
|
||||
encoded.forEach((bytes, i) => {
|
||||
data.set(bytes, pos);
|
||||
pos += bytes.length;
|
||||
odv.setInt32((i + 1) * 4, pos, true);
|
||||
});
|
||||
return {
|
||||
buffers: [validity, { bytes: offsets, length: (n + 1) * 4 }, { bytes: data, length: total }],
|
||||
nullCount,
|
||||
};
|
||||
}
|
||||
default: {
|
||||
const never: never = col.type;
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } {
|
||||
switch (type) {
|
||||
case "int64": {
|
||||
b.startObject(2);
|
||||
b.addInt32(0, 64, 0); // bitWidth
|
||||
b.addInt8(1, 1, 0); // is_signed
|
||||
return { typeType: TYPE_INT, offset: b.endObject() };
|
||||
}
|
||||
case "float64": {
|
||||
b.startObject(1);
|
||||
b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE
|
||||
return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() };
|
||||
}
|
||||
case "utf8": {
|
||||
b.startObject(0);
|
||||
return { typeType: TYPE_UTF8, offset: b.endObject() };
|
||||
}
|
||||
case "bool": {
|
||||
b.startObject(0);
|
||||
return { typeType: TYPE_BOOL, offset: b.endObject() };
|
||||
}
|
||||
case "timestamp_ms": {
|
||||
// Build the timezone string BEFORE startObject — FlatBuffers forbids
|
||||
// creating a nested object while a table is open.
|
||||
const tz = b.createString("UTC");
|
||||
b.startObject(2);
|
||||
b.addInt16(0, TIME_UNIT_MILLISECOND, 0);
|
||||
b.addOffset(1, tz, 0);
|
||||
return { typeType: TYPE_TIMESTAMP, offset: b.endObject() };
|
||||
}
|
||||
default: {
|
||||
const never: never = type;
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function buildSchemaMessage(columns: readonly ArrowColumn[]): Uint8Array {
|
||||
const b = new FlatBufferBuilder(2048);
|
||||
|
||||
const fieldOffsets: Offset[] = columns.map((col) => {
|
||||
const nameOffset = b.createString(col.name);
|
||||
const { typeType, offset: typeOffset } = buildTypeTable(b, col.type);
|
||||
const children = b.createOffsetVector([]);
|
||||
b.startObject(7);
|
||||
b.addOffset(0, nameOffset, 0);
|
||||
b.addInt8(1, 1, 0); // nullable = true
|
||||
b.addInt8(2, typeType, 0);
|
||||
b.addOffset(3, typeOffset, 0);
|
||||
b.addOffset(5, children, 0);
|
||||
return b.endObject();
|
||||
});
|
||||
|
||||
const fieldsVector = b.createOffsetVector(fieldOffsets);
|
||||
b.startObject(4);
|
||||
b.addInt16(0, ENDIANNESS_LITTLE, 0);
|
||||
b.addOffset(1, fieldsVector, 0);
|
||||
const schema = b.endObject();
|
||||
|
||||
b.startObject(5);
|
||||
b.addInt16(0, METADATA_VERSION_V5, 0);
|
||||
b.addInt8(1, MESSAGE_HEADER_SCHEMA, 0);
|
||||
b.addOffset(2, schema, 0);
|
||||
b.addInt64(3, BIG_ZERO, BIG_ZERO); // bodyLength
|
||||
const message = b.endObject();
|
||||
|
||||
return b.finish(message);
|
||||
}
|
||||
|
||||
function buildRecordBatchMessage(
|
||||
rowCount: number,
|
||||
nodes: { length: number; nullCount: number }[],
|
||||
buffers: { offset: number; length: number }[],
|
||||
bodyLength: number,
|
||||
): Uint8Array {
|
||||
const b = new FlatBufferBuilder(2048);
|
||||
|
||||
// FieldNode[] — inline structs, written in reverse.
|
||||
b.startVector(16, nodes.length, 8);
|
||||
for (let i = nodes.length - 1; i >= 0; i--) {
|
||||
b.writeInt64(BigInt(nodes[i].nullCount));
|
||||
b.writeInt64(BigInt(nodes[i].length));
|
||||
}
|
||||
const nodesVector = b.endVector(nodes.length);
|
||||
|
||||
// Buffer[] — inline structs, written in reverse.
|
||||
b.startVector(16, buffers.length, 8);
|
||||
for (let i = buffers.length - 1; i >= 0; i--) {
|
||||
b.writeInt64(BigInt(buffers[i].length));
|
||||
b.writeInt64(BigInt(buffers[i].offset));
|
||||
}
|
||||
const buffersVector = b.endVector(buffers.length);
|
||||
|
||||
b.startObject(5);
|
||||
b.addInt64(0, BigInt(rowCount), BIG_ZERO);
|
||||
b.addOffset(1, nodesVector, 0);
|
||||
b.addOffset(2, buffersVector, 0);
|
||||
const recordBatch = b.endObject();
|
||||
|
||||
b.startObject(5);
|
||||
b.addInt16(0, METADATA_VERSION_V5, 0);
|
||||
b.addInt8(1, MESSAGE_HEADER_RECORD_BATCH, 0);
|
||||
b.addOffset(2, recordBatch, 0);
|
||||
b.addInt64(3, BigInt(bodyLength), BIG_ZERO);
|
||||
const message = b.endObject();
|
||||
|
||||
return b.finish(message);
|
||||
}
|
||||
|
||||
function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array {
|
||||
// continuation(4) + length(4) + metadata, padded so the body is 8-aligned.
|
||||
const prefix = 8;
|
||||
const padded = alignTo8(prefix + metadata.length) - prefix;
|
||||
const out = new Uint8Array(prefix + padded + body.length);
|
||||
const dv = new DataView(out.buffer);
|
||||
dv.setUint32(0, 0xffffffff, true);
|
||||
dv.setInt32(4, padded, true);
|
||||
out.set(metadata, 8);
|
||||
out.set(body, prefix + padded);
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* Encode columns as a complete Arrow IPC *stream* (schema message, one record
|
||||
* batch, end-of-stream marker).
|
||||
*/
|
||||
export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array {
|
||||
if (columns.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column");
|
||||
}
|
||||
const rowCount = columns[0].values.length;
|
||||
for (const c of columns) {
|
||||
if (c.values.length !== rowCount) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`column "${c.name}" has ${c.values.length} values, expected ${rowCount}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const schemaMsg = buildSchemaMessage(columns);
|
||||
|
||||
const nodes: { length: number; nullCount: number }[] = [];
|
||||
const bufferMeta: { offset: number; length: number }[] = [];
|
||||
const bodyParts: Uint8Array[] = [];
|
||||
let bodyOffset = 0;
|
||||
for (const col of columns) {
|
||||
const { buffers, nullCount } = encodeColumn(col);
|
||||
nodes.push({ length: rowCount, nullCount });
|
||||
for (const buf of buffers) {
|
||||
bufferMeta.push({ offset: bodyOffset, length: buf.length });
|
||||
bodyParts.push(buf.bytes);
|
||||
bodyOffset += buf.bytes.length; // already 8-aligned by encodeColumn
|
||||
}
|
||||
}
|
||||
const body = new Uint8Array(bodyOffset);
|
||||
let pos = 0;
|
||||
for (const part of bodyParts) {
|
||||
body.set(part, pos);
|
||||
pos += part.length;
|
||||
}
|
||||
|
||||
const batchMsg = buildRecordBatchMessage(rowCount, nodes, bufferMeta, body.length);
|
||||
|
||||
const schemaFrame = framedMessage(schemaMsg, new Uint8Array(0));
|
||||
const batchFrame = framedMessage(batchMsg, body);
|
||||
const eos = new Uint8Array(8);
|
||||
new DataView(eos.buffer).setUint32(0, 0xffffffff, true); // length 0 = EOS
|
||||
|
||||
const out = new Uint8Array(schemaFrame.length + batchFrame.length + eos.length);
|
||||
out.set(schemaFrame, 0);
|
||||
out.set(batchFrame, schemaFrame.length);
|
||||
out.set(eos, schemaFrame.length + batchFrame.length);
|
||||
return out;
|
||||
}
|
||||
|
||||
// ── decoder (structural: schema names/types + row counts) ──────────────────
|
||||
|
||||
export interface ArrowStreamInfo {
|
||||
fields: { name: string; typeType: number }[];
|
||||
/** Total rows across every record batch in the stream. */
|
||||
rowCount: number;
|
||||
batchCount: number;
|
||||
}
|
||||
|
||||
/** Minimal FlatBuffers table reader — enough for Message/Schema/RecordBatch. */
|
||||
class FbTable {
|
||||
constructor(
|
||||
readonly dv: DataView,
|
||||
readonly pos: number,
|
||||
) {}
|
||||
|
||||
fieldPos(slot: number): number {
|
||||
const vtable = this.pos - this.dv.getInt32(this.pos, true);
|
||||
const vtLen = this.dv.getInt16(vtable, true);
|
||||
const idx = 4 + slot * 2;
|
||||
if (idx >= vtLen) return 0;
|
||||
const rel = this.dv.getInt16(vtable + idx, true);
|
||||
return rel === 0 ? 0 : this.pos + rel;
|
||||
}
|
||||
|
||||
readInt8(slot: number, def = 0): number {
|
||||
const p = this.fieldPos(slot);
|
||||
return p === 0 ? def : this.dv.getInt8(p);
|
||||
}
|
||||
|
||||
readInt16(slot: number, def = 0): number {
|
||||
const p = this.fieldPos(slot);
|
||||
return p === 0 ? def : this.dv.getInt16(p, true);
|
||||
}
|
||||
|
||||
readInt64(slot: number, def: bigint = BIG_ZERO): bigint {
|
||||
const p = this.fieldPos(slot);
|
||||
return p === 0 ? def : this.dv.getBigInt64(p, true);
|
||||
}
|
||||
|
||||
/** Follow a uoffset to a nested table. */
|
||||
readTable(slot: number): FbTable | undefined {
|
||||
const p = this.fieldPos(slot);
|
||||
if (p === 0) return undefined;
|
||||
return new FbTable(this.dv, p + this.dv.getInt32(p, true));
|
||||
}
|
||||
|
||||
vectorLength(slot: number): number {
|
||||
const p = this.fieldPos(slot);
|
||||
if (p === 0) return 0;
|
||||
const vec = p + this.dv.getInt32(p, true);
|
||||
return this.dv.getInt32(vec, true);
|
||||
}
|
||||
|
||||
vectorElementTable(slot: number, index: number): FbTable {
|
||||
const p = this.fieldPos(slot);
|
||||
const vec = p + this.dv.getInt32(p, true);
|
||||
const elem = vec + 4 + index * 4;
|
||||
return new FbTable(this.dv, elem + this.dv.getInt32(elem, true));
|
||||
}
|
||||
|
||||
readString(slot: number): string | undefined {
|
||||
const p = this.fieldPos(slot);
|
||||
if (p === 0) return undefined;
|
||||
const str = p + this.dv.getInt32(p, true);
|
||||
const len = this.dv.getInt32(str, true);
|
||||
const bytes = new Uint8Array(this.dv.buffer, this.dv.byteOffset + str + 4, len);
|
||||
return new TextDecoder().decode(bytes);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an Arrow IPC stream's structure. Used to enforce BI-P012's result-row
|
||||
* ceiling on Arrow payloads (including ClickHouse pass-through bytes, where
|
||||
* the count is otherwise invisible to 0pt) and by the roundtrip tests.
|
||||
*/
|
||||
export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo {
|
||||
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
|
||||
let pos = 0;
|
||||
let fields: { name: string; typeType: number }[] = [];
|
||||
let rowCount = 0;
|
||||
let batchCount = 0;
|
||||
|
||||
while (pos + 8 <= bytes.byteLength) {
|
||||
const continuation = dv.getUint32(pos, true);
|
||||
if (continuation !== 0xffffffff) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)");
|
||||
}
|
||||
const metaLen = dv.getInt32(pos + 4, true);
|
||||
if (metaLen === 0) break; // end of stream
|
||||
const metaStart = pos + 8;
|
||||
const root = metaStart + dv.getInt32(metaStart, true);
|
||||
const message = new FbTable(dv, root);
|
||||
const headerType = message.readInt8(1, 0);
|
||||
const bodyLength = Number(message.readInt64(3, BIG_ZERO));
|
||||
|
||||
if (headerType === MESSAGE_HEADER_SCHEMA) {
|
||||
const schema = message.readTable(2);
|
||||
if (schema) {
|
||||
const n = schema.vectorLength(1);
|
||||
fields = [];
|
||||
for (let i = 0; i < n; i++) {
|
||||
const f = schema.vectorElementTable(1, i);
|
||||
fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) });
|
||||
}
|
||||
}
|
||||
} else if (headerType === MESSAGE_HEADER_RECORD_BATCH) {
|
||||
const rb = message.readTable(2);
|
||||
if (rb) {
|
||||
rowCount += Number(rb.readInt64(0, BIG_ZERO));
|
||||
batchCount += 1;
|
||||
}
|
||||
}
|
||||
pos = metaStart + metaLen + bodyLength;
|
||||
}
|
||||
|
||||
return { fields, rowCount, batchCount };
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P012 on the Arrow path: refuse a payload that exceeds the caller's
|
||||
* result-row ceiling. Without this, asking for Arrow would be a way to walk
|
||||
* around `max_result_rows`.
|
||||
*/
|
||||
export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: number): ArrowStreamInfo {
|
||||
const info = decodeArrowStream(bytes);
|
||||
if (info.rowCount > maxResultRows) {
|
||||
throw new SiaxError(
|
||||
"QUOTA_EXCEEDED",
|
||||
"0pt.analytics",
|
||||
`Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`,
|
||||
);
|
||||
}
|
||||
return info;
|
||||
}
|
||||
@@ -0,0 +1,341 @@
|
||||
/**
|
||||
* src/analytics/bench/benchmark-harness.ts
|
||||
* =========================================
|
||||
* BI-P104 — BI PERFORMANCE BENCHMARK HARNESS. DoD: "results committed as
|
||||
* machine-readable evidence."
|
||||
*
|
||||
* The harness exists because of a specific way benchmarks lie. Yesterday's
|
||||
* lane measured the same budget-enforcement overshoot at ~150x on one run and
|
||||
* ~130x on the next, on the same machine, and correctly reported the SHAPE
|
||||
* rather than a number. That is the doctrine encoded here:
|
||||
*
|
||||
* 1. A SINGLE FIGURE IS NOT A SYSTEM PROPERTY. `summarizeSamples` returns a
|
||||
* distribution — n, min, p50, p95, p99, max, mean, stdev, CV, IQR — and
|
||||
* there is no function anywhere in this module that returns one number
|
||||
* for a measurement. `describeComparison` renders a RANGE and the sample
|
||||
* size into every sentence it produces.
|
||||
*
|
||||
* 2. OVERLAPPING DISTRIBUTIONS ARE INDISTINGUISHABLE, whatever the medians
|
||||
* say. If the candidate's p95 is not below the baseline's minimum, the
|
||||
* verdict is `indistinguishable` even when p50 differs by 2x — because a
|
||||
* 2x median gap inside a 10x spread is noise with a story attached.
|
||||
*
|
||||
* 3. TOO FEW SAMPLES IS A REFUSAL. Under MIN_SAMPLES the summary carries
|
||||
* `stability: 'insufficient_samples'` and comparisons refuse to judge.
|
||||
* A three-run benchmark is an anecdote.
|
||||
*
|
||||
* 4. AN ENGINE THAT WAS NOT RUN IS RECORDED AS NOT RUN. `EngineAvailability`
|
||||
* carries a reason. The evidence file lists every engine BI-P104 names,
|
||||
* including the ones nothing measured — an omitted engine reads as "we
|
||||
* chose not to report it".
|
||||
*/
|
||||
|
||||
/** The six dataset profiles BI-P104 names, with what makes each one distinct. */
|
||||
export const BENCH_DATASETS = [
|
||||
{
|
||||
id: 'small_saas',
|
||||
label: 'small SaaS',
|
||||
approx_rows: 100_000,
|
||||
shape: 'narrow fact table, one tenant, high selectivity',
|
||||
},
|
||||
{
|
||||
id: 'medium_business',
|
||||
label: 'medium business',
|
||||
approx_rows: 2_000_000,
|
||||
shape: 'multi-tenant fact table with a date and a few dimensions',
|
||||
},
|
||||
{
|
||||
id: 'vehiq_market',
|
||||
label: 'VehIQ market',
|
||||
approx_rows: 1_000_000,
|
||||
shape: 'vehicle listings: make/model/year/price/mileage, skewed cardinalities',
|
||||
},
|
||||
{
|
||||
id: 'large_event',
|
||||
label: 'large event',
|
||||
approx_rows: 5_000_000,
|
||||
shape: 'append-only event stream, timestamp-ordered, sparse dimensions',
|
||||
},
|
||||
{
|
||||
id: 'wide_table',
|
||||
label: 'wide table',
|
||||
approx_rows: 200_000,
|
||||
shape: 'many columns, few read per query — the columnar-store best case',
|
||||
},
|
||||
{
|
||||
id: 'geospatial',
|
||||
label: 'geospatial',
|
||||
approx_rows: 500_000,
|
||||
shape: 'lat/lon points with bounding-box and distance predicates',
|
||||
},
|
||||
] as const;
|
||||
export type BenchDatasetId = (typeof BENCH_DATASETS)[number]['id'];
|
||||
|
||||
/** Every engine BI-P104 lists. All of them appear in the evidence, run or not. */
|
||||
export const BENCH_ENGINES = [
|
||||
'clickhouse',
|
||||
'duckdb',
|
||||
'trino',
|
||||
'semantic_query',
|
||||
'superset',
|
||||
'native_charts',
|
||||
'perspective',
|
||||
] as const;
|
||||
export type BenchEngine = (typeof BENCH_ENGINES)[number];
|
||||
|
||||
export interface EngineAvailability {
|
||||
engine: BenchEngine;
|
||||
available: boolean;
|
||||
/** Required when unavailable. "Unknown" is not a reason. */
|
||||
reason?: string;
|
||||
version?: string;
|
||||
}
|
||||
|
||||
export const STABILITY_VERDICTS = ['stable', 'noisy', 'unstable', 'insufficient_samples'] as const;
|
||||
export type StabilityVerdict = (typeof STABILITY_VERDICTS)[number];
|
||||
|
||||
/** Below this a run is an anecdote, not a measurement. */
|
||||
export const MIN_SAMPLES = 5;
|
||||
/** Coefficient of variation thresholds. Wide on purpose: real hosts are noisy. */
|
||||
const CV_STABLE = 0.15;
|
||||
const CV_NOISY = 0.5;
|
||||
|
||||
export interface Distribution {
|
||||
n: number;
|
||||
min: number;
|
||||
p50: number;
|
||||
p95: number;
|
||||
p99: number;
|
||||
max: number;
|
||||
mean: number;
|
||||
stdev: number;
|
||||
/** stdev / mean. The number that decides whether a comparison is meaningful. */
|
||||
cv: number;
|
||||
iqr: number;
|
||||
/** max / min. Reported because it is what a reader feels. */
|
||||
spread_ratio: number;
|
||||
stability: StabilityVerdict;
|
||||
}
|
||||
|
||||
/** Nearest-rank; no interpolation, so every reported value really happened. */
|
||||
function pct(sorted: number[], q: number): number {
|
||||
const rank = Math.ceil(q * sorted.length);
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
|
||||
}
|
||||
|
||||
export function summarizeSamples(samples: readonly number[]): Distribution {
|
||||
if (samples.length === 0) {
|
||||
return {
|
||||
n: 0,
|
||||
min: 0,
|
||||
p50: 0,
|
||||
p95: 0,
|
||||
p99: 0,
|
||||
max: 0,
|
||||
mean: 0,
|
||||
stdev: 0,
|
||||
cv: 0,
|
||||
iqr: 0,
|
||||
spread_ratio: 0,
|
||||
stability: 'insufficient_samples',
|
||||
};
|
||||
}
|
||||
const s = [...samples].sort((a, b) => a - b);
|
||||
const mean = s.reduce((a, b) => a + b, 0) / s.length;
|
||||
const variance = s.reduce((acc, v) => acc + (v - mean) ** 2, 0) / s.length;
|
||||
const stdev = Math.sqrt(variance);
|
||||
const cv = mean === 0 ? 0 : stdev / mean;
|
||||
const stability: StabilityVerdict =
|
||||
s.length < MIN_SAMPLES
|
||||
? 'insufficient_samples'
|
||||
: cv <= CV_STABLE
|
||||
? 'stable'
|
||||
: cv <= CV_NOISY
|
||||
? 'noisy'
|
||||
: 'unstable';
|
||||
return {
|
||||
n: s.length,
|
||||
min: s[0],
|
||||
p50: pct(s, 0.5),
|
||||
p95: pct(s, 0.95),
|
||||
p99: pct(s, 0.99),
|
||||
max: s[s.length - 1],
|
||||
mean,
|
||||
stdev,
|
||||
cv,
|
||||
iqr: pct(s, 0.75) - pct(s, 0.25),
|
||||
spread_ratio: s[0] === 0 ? Number.POSITIVE_INFINITY : s[s.length - 1] / s[0],
|
||||
stability,
|
||||
};
|
||||
}
|
||||
|
||||
export const COMPARISON_VERDICTS = ['faster', 'slower', 'indistinguishable', 'not_comparable'] as const;
|
||||
export type ComparisonVerdict = (typeof COMPARISON_VERDICTS)[number];
|
||||
|
||||
export interface Comparison {
|
||||
verdict: ComparisonVerdict;
|
||||
/** Ratio of medians. Present for information; never the claim on its own. */
|
||||
p50_ratio: number;
|
||||
/** Worst case for the candidate: baseline.min / candidate.max. */
|
||||
conservative_ratio: number;
|
||||
/** Best case for the candidate: baseline.max / candidate.min. */
|
||||
optimistic_ratio: number;
|
||||
/** True when the observed ranges overlap at all. */
|
||||
ranges_overlap: boolean;
|
||||
baseline: Distribution;
|
||||
candidate: Distribution;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare two measured distributions.
|
||||
*
|
||||
* `faster` requires clean separation — the candidate's p95 below the
|
||||
* baseline's minimum. Anything less is `indistinguishable`, however flattering
|
||||
* the medians look, because that is exactly the claim a noisy host will
|
||||
* manufacture for you.
|
||||
*/
|
||||
export function compareDistributions(baseline: Distribution, candidate: Distribution): Comparison {
|
||||
const ratio = (a: number, b: number) => (b === 0 ? Number.POSITIVE_INFINITY : a / b);
|
||||
const base: Omit<Comparison, 'verdict'> = {
|
||||
p50_ratio: ratio(baseline.p50, candidate.p50),
|
||||
conservative_ratio: ratio(baseline.min, candidate.max),
|
||||
optimistic_ratio: ratio(baseline.max, candidate.min),
|
||||
ranges_overlap: !(candidate.max < baseline.min || baseline.max < candidate.min),
|
||||
baseline,
|
||||
candidate,
|
||||
};
|
||||
if (baseline.stability === 'insufficient_samples' || candidate.stability === 'insufficient_samples') {
|
||||
return { ...base, verdict: 'not_comparable' };
|
||||
}
|
||||
if (candidate.p95 < baseline.min) return { ...base, verdict: 'faster' };
|
||||
if (baseline.p95 < candidate.min) return { ...base, verdict: 'slower' };
|
||||
return { ...base, verdict: 'indistinguishable' };
|
||||
}
|
||||
|
||||
/**
|
||||
* The sentence a human is allowed to quote. It ALWAYS carries the range, the
|
||||
* sample size and the stability verdict, so it cannot be trimmed down to "3.4x
|
||||
* faster" without visibly removing the caveats.
|
||||
*/
|
||||
export function describeComparison(c: Comparison, label = 'candidate'): string {
|
||||
const r = (v: number) => (Number.isFinite(v) ? v.toFixed(1) : '∞');
|
||||
if (c.verdict === 'not_comparable') {
|
||||
return (
|
||||
`${label}: NOT COMPARABLE — ${c.baseline.n} baseline and ${c.candidate.n} candidate samples, ` +
|
||||
`fewer than the ${MIN_SAMPLES} required. A run this short is an anecdote.`
|
||||
);
|
||||
}
|
||||
if (c.verdict === 'indistinguishable') {
|
||||
return (
|
||||
`${label}: INDISTINGUISHABLE at this sample size. Medians differ by ${r(c.p50_ratio)}x but the ` +
|
||||
`observed ranges overlap (baseline ${c.baseline.min.toFixed(1)}-${c.baseline.max.toFixed(1)}ms, ` +
|
||||
`candidate ${c.candidate.min.toFixed(1)}-${c.candidate.max.toFixed(1)}ms; n=${c.baseline.n}/${c.candidate.n}, ` +
|
||||
`CV ${c.baseline.cv.toFixed(2)}/${c.candidate.cv.toFixed(2)}).`
|
||||
);
|
||||
}
|
||||
return (
|
||||
`${label}: ${c.verdict.toUpperCase()} — ${r(c.conservative_ratio)}x to ${r(c.optimistic_ratio)}x ` +
|
||||
`(median ${r(c.p50_ratio)}x) over n=${c.baseline.n}/${c.candidate.n}, ` +
|
||||
`stability ${c.baseline.stability}/${c.candidate.stability}. Warm cache, one host, this query shape.`
|
||||
);
|
||||
}
|
||||
|
||||
export interface BenchmarkRun {
|
||||
engine: BenchEngine;
|
||||
dataset: BenchDatasetId;
|
||||
query_label: string;
|
||||
/** Discarded from the statistics; recorded so the discard is visible. */
|
||||
warmup_ms: number[];
|
||||
samples_ms: number[];
|
||||
distribution: Distribution;
|
||||
rows_returned?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run one measurement. Warmups are executed and RECORDED but excluded from the
|
||||
* distribution: a first run that pays for cold marks and page cache is a
|
||||
* different measurement, and silently folding it in is how a p95 becomes a
|
||||
* story about disk.
|
||||
*/
|
||||
export async function measure(args: {
|
||||
engine: BenchEngine;
|
||||
dataset: BenchDatasetId;
|
||||
query_label: string;
|
||||
run: () => Promise<{ rows?: number }>;
|
||||
warmups?: number;
|
||||
repetitions?: number;
|
||||
now?: () => number;
|
||||
}): Promise<BenchmarkRun> {
|
||||
const now = args.now ?? (() => performance.now());
|
||||
const warmups = args.warmups ?? 2;
|
||||
const repetitions = args.repetitions ?? 9;
|
||||
const warmup_ms: number[] = [];
|
||||
const samples_ms: number[] = [];
|
||||
let rows: number | undefined;
|
||||
|
||||
for (let i = 0; i < warmups; i += 1) {
|
||||
const t0 = now();
|
||||
await args.run();
|
||||
warmup_ms.push(now() - t0);
|
||||
}
|
||||
for (let i = 0; i < repetitions; i += 1) {
|
||||
const t0 = now();
|
||||
const out = await args.run();
|
||||
samples_ms.push(now() - t0);
|
||||
rows = out.rows ?? rows;
|
||||
}
|
||||
return {
|
||||
engine: args.engine,
|
||||
dataset: args.dataset,
|
||||
query_label: args.query_label,
|
||||
warmup_ms,
|
||||
samples_ms,
|
||||
distribution: summarizeSamples(samples_ms),
|
||||
...(rows !== undefined ? { rows_returned: rows } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export interface BenchmarkEvidence {
|
||||
recorded_at: string;
|
||||
host_note: string;
|
||||
datasets: typeof BENCH_DATASETS;
|
||||
engines: EngineAvailability[];
|
||||
runs: BenchmarkRun[];
|
||||
comparisons: Array<{ dataset: BenchDatasetId; query_label: string; description: string } & Comparison>;
|
||||
/** Engines BI-P104 names that produced no measurement, each with a reason. */
|
||||
not_exercised: EngineAvailability[];
|
||||
}
|
||||
|
||||
/**
|
||||
* Assemble the evidence document. Every engine BI-P104 names appears, and the
|
||||
* ones with no runs are listed under `not_exercised` with their reason — an
|
||||
* engine quietly missing from a benchmark table reads as a choice.
|
||||
*/
|
||||
export function buildEvidence(args: {
|
||||
host_note: string;
|
||||
engines: EngineAvailability[];
|
||||
runs: BenchmarkRun[];
|
||||
comparisons?: BenchmarkEvidence['comparisons'];
|
||||
now?: Date;
|
||||
}): BenchmarkEvidence {
|
||||
const measured = new Set(args.runs.map((r) => r.engine));
|
||||
const declared = new Map(args.engines.map((e) => [e.engine, e]));
|
||||
return {
|
||||
recorded_at: (args.now ?? new Date()).toISOString(),
|
||||
host_note: args.host_note,
|
||||
datasets: BENCH_DATASETS,
|
||||
engines: BENCH_ENGINES.map(
|
||||
(engine) => declared.get(engine) ?? { engine, available: false, reason: 'not declared by this run' },
|
||||
),
|
||||
runs: args.runs,
|
||||
comparisons: args.comparisons ?? [],
|
||||
not_exercised: BENCH_ENGINES.filter((e) => !measured.has(e)).map((engine) => {
|
||||
const d = declared.get(engine);
|
||||
if (!d) return { engine, available: false, reason: 'not present in this environment' };
|
||||
// An engine declared AVAILABLE that produced no measurement is the
|
||||
// dangerous case: it reads as "we could have, and chose not to say".
|
||||
// It gets a reason too, generated here rather than left blank.
|
||||
return d.reason ? d : { ...d, reason: 'declared available but produced no measurement in this run' };
|
||||
}),
|
||||
};
|
||||
}
|
||||
Vendored
+329
@@ -0,0 +1,329 @@
|
||||
/**
|
||||
* src/analytics/cache/query-cache.ts
|
||||
* ===================================
|
||||
* BI-P020 — QUERY CACHE.
|
||||
*
|
||||
* "Cache key includes: tenant, semantic model version, metric definitions,
|
||||
* filters, query, data snapshot/version. Never share tenant result cache."
|
||||
*
|
||||
* All six are REQUIRED, and the key builder refuses to produce a key when one
|
||||
* is missing rather than defaulting it. A defaulted component is how a cache
|
||||
* serves a result computed under a metric definition that no longer exists —
|
||||
* the answer is not stale, it is WRONG, and it looks current.
|
||||
*
|
||||
* "Never share tenant result cache" is enforced structurally, not by
|
||||
* discipline:
|
||||
*
|
||||
* * the key is PREFIXED with the tenant id, and the prefix is derived from
|
||||
* the verified identity inside this module — a caller cannot pass one;
|
||||
* * every read re-derives the prefix from the identity and REFUSES a stored
|
||||
* entry whose recorded tenant does not match, so a prefix collision or a
|
||||
* shared backing store still cannot serve a foreign result;
|
||||
* * the stored entry carries its own tenant id, so the check is against the
|
||||
* DATA, not against the key that found it.
|
||||
*
|
||||
* The second and third points exist because the first is a naming convention,
|
||||
* and naming conventions are what ST0RE was relying on when it turned out to
|
||||
* have four cross-tenant paths rather than the one reported.
|
||||
*
|
||||
* Provider-neutral: `AnalyticsResultCache` is the port. `InMemoryResultCache`
|
||||
* is the reference implementation and the test double; `ValkeyResultCache`
|
||||
* adapts any Valkey/Redis client that can GET/SET/DEL with a TTL, taking it as
|
||||
* an injected interface so this module adds no dependency.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
|
||||
/**
|
||||
* Everything that must be true for a cached result to still be the right
|
||||
* answer. Missing any one of these makes the cache unsound, so the type has no
|
||||
* optional fields.
|
||||
*/
|
||||
export interface CacheKeyComponents {
|
||||
/** From the verified identity. Not a caller-supplied value. */
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
dataset_id: string;
|
||||
/** BI-P030's semantic model version. Bumping it invalidates everything under it. */
|
||||
semantic_model_version: number;
|
||||
/**
|
||||
* metric_id -> version, for EVERY metric in the query. A metric whose
|
||||
* definition changed must not serve a result computed under the old one.
|
||||
*/
|
||||
metric_versions: Record<string, number>;
|
||||
/** The compiled statement hash (BI-P018). Covers filters, grain and limit. */
|
||||
compiled_query_hash: string;
|
||||
/**
|
||||
* The data snapshot the result was computed from. Whatever identifies "the
|
||||
* data as of now" for this dataset: a DatasetVersion number, an Iceberg
|
||||
* snapshot id, a max ingested watermark. Advancing it invalidates.
|
||||
*/
|
||||
data_snapshot: string;
|
||||
/** Result encoding, since a JSON and an Arrow result are different bytes. */
|
||||
format: string;
|
||||
}
|
||||
|
||||
export interface CachedResult<T = unknown> {
|
||||
/** Recorded so the read check is against the DATA, not against the key. */
|
||||
tenant_id: string;
|
||||
key: string;
|
||||
value: T;
|
||||
stored_at: string;
|
||||
expires_at: string;
|
||||
row_count: number;
|
||||
/** Kept for targeted invalidation. */
|
||||
dataset_id: string;
|
||||
metric_ids: string[];
|
||||
semantic_model_version: number;
|
||||
data_snapshot: string;
|
||||
}
|
||||
|
||||
export interface AnalyticsResultCache {
|
||||
readonly provider: string;
|
||||
get<T>(components: CacheKeyComponents): Promise<CachedResult<T> | undefined>;
|
||||
set<T>(
|
||||
components: CacheKeyComponents,
|
||||
value: T,
|
||||
opts: { ttlSeconds: number; rowCount: number },
|
||||
): Promise<void>;
|
||||
/** Drop every entry for a dataset — the blunt, always-correct invalidation. */
|
||||
invalidateDataset(tenantId: string, datasetId: string): Promise<number>;
|
||||
/** Drop every entry that used a given metric. */
|
||||
invalidateMetric(tenantId: string, metricId: string): Promise<number>;
|
||||
/** Drop everything for a tenant (offboarding, incident response). */
|
||||
invalidateTenant(tenantId: string): Promise<number>;
|
||||
}
|
||||
|
||||
function required(value: unknown, name: string): void {
|
||||
if (
|
||||
value === undefined ||
|
||||
value === null ||
|
||||
value === '' ||
|
||||
(typeof value === 'number' && !Number.isFinite(value))
|
||||
) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.cache',
|
||||
`cache key component "${name}" is required — a defaulted component serves results computed under a definition that no longer exists`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Build the key. Deterministic, tenant-prefixed, and total over the six
|
||||
* components BI-P020 lists.
|
||||
*
|
||||
* The tenant prefix is in PLAINTEXT at the front rather than folded into the
|
||||
* hash, so `invalidateTenant` is a prefix scan and an operator reading the
|
||||
* store can see which tenant an entry belongs to without a lookup table.
|
||||
*/
|
||||
export function buildCacheKey(c: CacheKeyComponents): string {
|
||||
required(c.tenant_id, 'tenant_id');
|
||||
required(c.dataset_id, 'dataset_id');
|
||||
required(c.semantic_model_version, 'semantic_model_version');
|
||||
required(c.compiled_query_hash, 'compiled_query_hash');
|
||||
required(c.data_snapshot, 'data_snapshot');
|
||||
required(c.format, 'format');
|
||||
if (!c.metric_versions || Object.keys(c.metric_versions).length === 0) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.cache',
|
||||
'cache key component "metric_versions" is required and must name every metric in the query',
|
||||
);
|
||||
}
|
||||
for (const [id, version] of Object.entries(c.metric_versions)) {
|
||||
required(version, `metric_versions[${id}]`);
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{1,64}$/.test(c.tenant_id)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'tenant id is not key-safe');
|
||||
}
|
||||
const material = JSON.stringify([
|
||||
c.tenant_id,
|
||||
c.project_id ?? '',
|
||||
c.dataset_id,
|
||||
c.semantic_model_version,
|
||||
Object.keys(c.metric_versions)
|
||||
.sort()
|
||||
.map((k) => [k, c.metric_versions[k]]),
|
||||
c.compiled_query_hash,
|
||||
c.data_snapshot,
|
||||
c.format,
|
||||
]);
|
||||
const digest = crypto.createHash('sha256').update(material).digest('hex');
|
||||
return `optq:${c.tenant_id}:${digest}`;
|
||||
}
|
||||
|
||||
/** The prefix `invalidateTenant` scans. Derived, never supplied. */
|
||||
export function tenantKeyPrefix(tenantId: string): string {
|
||||
return `optq:${tenantId}:`;
|
||||
}
|
||||
|
||||
abstract class BaseResultCache implements AnalyticsResultCache {
|
||||
abstract readonly provider: string;
|
||||
protected abstract read(key: string): Promise<CachedResult | undefined>;
|
||||
protected abstract write(entry: CachedResult, ttlSeconds: number): Promise<void>;
|
||||
protected abstract entries(tenantId: string): Promise<CachedResult[]>;
|
||||
protected abstract remove(keys: string[]): Promise<void>;
|
||||
|
||||
protected readonly now: () => number;
|
||||
constructor(now: () => number = Date.now) {
|
||||
this.now = now;
|
||||
}
|
||||
|
||||
async get<T>(components: CacheKeyComponents): Promise<CachedResult<T> | undefined> {
|
||||
const key = buildCacheKey(components);
|
||||
const entry = await this.read(key);
|
||||
if (!entry) return undefined;
|
||||
|
||||
// The check that survives a prefix collision or a shared backing store:
|
||||
// the ENTRY says which tenant it belongs to, and it must be this one.
|
||||
if (entry.tenant_id !== components.tenant_id) {
|
||||
// Not a miss — a cross-tenant hit is a security event, and returning
|
||||
// `undefined` would hide it. Drop the poisoned entry and refuse loudly.
|
||||
await this.remove([key]);
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.cache',
|
||||
'cache entry belongs to a different tenant; entry evicted',
|
||||
);
|
||||
}
|
||||
if (Date.parse(entry.expires_at) <= this.now()) {
|
||||
await this.remove([key]);
|
||||
return undefined;
|
||||
}
|
||||
return entry as CachedResult<T>;
|
||||
}
|
||||
|
||||
async set<T>(
|
||||
components: CacheKeyComponents,
|
||||
value: T,
|
||||
opts: { ttlSeconds: number; rowCount: number },
|
||||
): Promise<void> {
|
||||
if (!Number.isFinite(opts.ttlSeconds) || opts.ttlSeconds <= 0) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'ttlSeconds must be positive');
|
||||
}
|
||||
const key = buildCacheKey(components);
|
||||
const nowMs = this.now();
|
||||
await this.write(
|
||||
{
|
||||
tenant_id: components.tenant_id,
|
||||
key,
|
||||
value,
|
||||
stored_at: new Date(nowMs).toISOString(),
|
||||
expires_at: new Date(nowMs + opts.ttlSeconds * 1000).toISOString(),
|
||||
row_count: opts.rowCount,
|
||||
dataset_id: components.dataset_id,
|
||||
metric_ids: Object.keys(components.metric_versions),
|
||||
semantic_model_version: components.semantic_model_version,
|
||||
data_snapshot: components.data_snapshot,
|
||||
},
|
||||
opts.ttlSeconds,
|
||||
);
|
||||
}
|
||||
|
||||
async invalidateDataset(tenantId: string, datasetId: string): Promise<number> {
|
||||
const victims = (await this.entries(tenantId)).filter((e) => e.dataset_id === datasetId);
|
||||
await this.remove(victims.map((v) => v.key));
|
||||
return victims.length;
|
||||
}
|
||||
|
||||
async invalidateMetric(tenantId: string, metricId: string): Promise<number> {
|
||||
const victims = (await this.entries(tenantId)).filter((e) => e.metric_ids.includes(metricId));
|
||||
await this.remove(victims.map((v) => v.key));
|
||||
return victims.length;
|
||||
}
|
||||
|
||||
async invalidateTenant(tenantId: string): Promise<number> {
|
||||
const victims = await this.entries(tenantId);
|
||||
await this.remove(victims.map((v) => v.key));
|
||||
return victims.length;
|
||||
}
|
||||
}
|
||||
|
||||
export class InMemoryResultCache extends BaseResultCache {
|
||||
readonly provider = 'memory';
|
||||
private readonly store = new Map<string, CachedResult>();
|
||||
|
||||
protected async read(key: string): Promise<CachedResult | undefined> {
|
||||
return this.store.get(key);
|
||||
}
|
||||
protected async write(entry: CachedResult): Promise<void> {
|
||||
this.store.set(entry.key, entry);
|
||||
}
|
||||
protected async entries(tenantId: string): Promise<CachedResult[]> {
|
||||
const prefix = tenantKeyPrefix(tenantId);
|
||||
return [...this.store.values()].filter((e) => e.key.startsWith(prefix));
|
||||
}
|
||||
protected async remove(keys: string[]): Promise<void> {
|
||||
for (const k of keys) this.store.delete(k);
|
||||
}
|
||||
/** Test/observability affordance. Never used to bypass the tenant check. */
|
||||
get size(): number {
|
||||
return this.store.size;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The minimum a Valkey/Redis client must offer. Taken as an interface so this
|
||||
* module adds no dependency and no CVE surface — the same reason BI-P005's
|
||||
* Arrow writer is hand-written.
|
||||
*/
|
||||
export interface ValkeyLikeClient {
|
||||
get(key: string): Promise<string | null>;
|
||||
set(key: string, value: string, opts: { EX: number }): Promise<unknown>;
|
||||
del(keys: string[]): Promise<unknown>;
|
||||
/** Cursor-based scan. A blocking KEYS on a shared Valkey is an outage. */
|
||||
scan(cursor: string, opts: { MATCH: string; COUNT: number }): Promise<{ cursor: string; keys: string[] }>;
|
||||
}
|
||||
|
||||
export class ValkeyResultCache extends BaseResultCache {
|
||||
readonly provider = 'valkey';
|
||||
constructor(
|
||||
private readonly client: ValkeyLikeClient,
|
||||
now: () => number = Date.now,
|
||||
) {
|
||||
super(now);
|
||||
}
|
||||
|
||||
protected async read(key: string): Promise<CachedResult | undefined> {
|
||||
const raw = await this.client.get(key);
|
||||
if (raw === null) return undefined;
|
||||
try {
|
||||
return JSON.parse(raw) as CachedResult;
|
||||
} catch {
|
||||
// A corrupt entry is a miss, not a crash — but it is also removed, so a
|
||||
// permanently unparseable key cannot become a permanent cache miss that
|
||||
// nobody notices.
|
||||
await this.client.del([key]);
|
||||
return undefined;
|
||||
}
|
||||
}
|
||||
|
||||
protected async write(entry: CachedResult, ttlSeconds: number): Promise<void> {
|
||||
// The TTL is set on the STORE as well as recorded in the entry. Relying on
|
||||
// the recorded timestamp alone leaves expired results occupying memory
|
||||
// forever on a busy tenant.
|
||||
await this.client.set(entry.key, JSON.stringify(entry), { EX: Math.ceil(ttlSeconds) });
|
||||
}
|
||||
|
||||
protected async entries(tenantId: string): Promise<CachedResult[]> {
|
||||
const prefix = tenantKeyPrefix(tenantId);
|
||||
const out: CachedResult[] = [];
|
||||
let cursor = '0';
|
||||
do {
|
||||
const page = await this.client.scan(cursor, { MATCH: `${prefix}*`, COUNT: 500 });
|
||||
cursor = page.cursor;
|
||||
for (const key of page.keys) {
|
||||
const entry = await this.read(key);
|
||||
// Defence in depth: SCAN MATCH is a glob, and a glob is a naming
|
||||
// convention. Confirm against the entry's own tenant id.
|
||||
if (entry && entry.tenant_id === tenantId) out.push(entry);
|
||||
}
|
||||
} while (cursor !== '0');
|
||||
return out;
|
||||
}
|
||||
|
||||
protected async remove(keys: string[]): Promise<void> {
|
||||
if (keys.length) await this.client.del(keys);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* src/analytics/clickhouse-analytics.ts
|
||||
* ======================================
|
||||
* ClickHouse-backed Analytics capability (Master-DoD §10 Analytics:
|
||||
* ClickHouse, tenant isolation, dataset/event ingest, query limits, metric
|
||||
* query capability, retention).
|
||||
*
|
||||
* LEGAL BOUNDARY (absolute): this module is a GENERIC analytics adapter for
|
||||
* 0pt capability events/dataset metrics. It has NO connection to, and MUST
|
||||
* NEVER be pointed at, the b0ts ClickHouse pipelines (Ratsit/person-PII —
|
||||
* separate legal case) or any quarantined market='sverige'/'norge' market
|
||||
* data (written DPIA prohibition). Nothing here reads services/scrapers/*.
|
||||
*
|
||||
* Tenant isolation model:
|
||||
* - every table carries a `tenant_id` column;
|
||||
* - EVERY query built by this service injects `tenant_id = {tenant}` as a
|
||||
* parameterized predicate — callers cannot omit it;
|
||||
* - callers NEVER pass raw SQL: the query surface is a closed metric-query
|
||||
* DSL (metric + optional group-by + time range) compiled here, which is
|
||||
* what makes the injection guarantee enforceable.
|
||||
*
|
||||
* Query limits (§10): every SELECT runs with readonly=1,
|
||||
* max_execution_time, max_result_rows — enforced via ClickHouse settings on
|
||||
* the HTTP interface.
|
||||
*
|
||||
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
|
||||
*/
|
||||
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
|
||||
|
||||
export interface ClickHouseAnalyticsOptions {
|
||||
baseUrl: string;
|
||||
/** "user:password" — from env/secret ref. */
|
||||
basicAuth?: string;
|
||||
database?: string;
|
||||
timeoutMs?: number;
|
||||
/** SELECT limits (§10 query limits). */
|
||||
maxExecutionTimeSec?: number;
|
||||
maxResultRows?: number;
|
||||
}
|
||||
|
||||
export interface AnalyticsEvent {
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
metric: string;
|
||||
value: number;
|
||||
/** ISO timestamp; defaults to now. */
|
||||
event_time?: string;
|
||||
dimensions?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface MetricQuery {
|
||||
tenant_id: string;
|
||||
dataset_id?: string;
|
||||
metric: string;
|
||||
aggregate: "sum" | "avg" | "min" | "max" | "count";
|
||||
/** Group by one dimension key (validated identifier). */
|
||||
group_by?: string;
|
||||
from_time?: string;
|
||||
to_time?: string;
|
||||
/** Row limit; clamped to maxResultRows. */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface MetricQueryRow {
|
||||
group?: string;
|
||||
value: number;
|
||||
}
|
||||
|
||||
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
|
||||
export class ClickHouseAnalytics {
|
||||
private readonly baseUrl: string;
|
||||
private readonly authHeader?: string;
|
||||
private readonly database: string;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly maxExecutionTimeSec: number;
|
||||
private readonly maxResultRows: number;
|
||||
|
||||
constructor(opts: ClickHouseAnalyticsOptions) {
|
||||
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
||||
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
|
||||
this.database = opts.database ?? "opt_analytics";
|
||||
this.timeoutMs = opts.timeoutMs ?? 30_000;
|
||||
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
|
||||
this.maxResultRows = opts.maxResultRows ?? 10_000;
|
||||
}
|
||||
|
||||
/** Create the events table (idempotent) with retention TTL (§10 retention). */
|
||||
async ensureSchema(retentionDays = 90): Promise<void> {
|
||||
await this.exec(`CREATE DATABASE IF NOT EXISTS ${this.database}`);
|
||||
await this.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.database}.opt_metric_events (
|
||||
tenant_id LowCardinality(String),
|
||||
dataset_id String,
|
||||
metric LowCardinality(String),
|
||||
value Float64,
|
||||
event_time DateTime64(3, 'UTC'),
|
||||
dimensions Map(String, String)
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (tenant_id, metric, dataset_id, event_time)
|
||||
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
|
||||
);
|
||||
}
|
||||
|
||||
/** Ingest events (JSONEachRow). tenant_id is required per event. */
|
||||
async ingest(events: AnalyticsEvent[]): Promise<{ ingested: number }> {
|
||||
if (events.length === 0) return { ingested: 0 };
|
||||
for (const e of events) {
|
||||
if (!e.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
|
||||
}
|
||||
if (!Number.isFinite(e.value)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
|
||||
}
|
||||
}
|
||||
const rows = events
|
||||
.map((e) =>
|
||||
JSON.stringify({
|
||||
tenant_id: e.tenant_id,
|
||||
dataset_id: e.dataset_id,
|
||||
metric: e.metric,
|
||||
value: e.value,
|
||||
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
|
||||
dimensions: e.dimensions ?? {},
|
||||
})
|
||||
)
|
||||
.join("\n");
|
||||
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
|
||||
return { ingested: events.length };
|
||||
}
|
||||
|
||||
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
|
||||
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
|
||||
if (!q.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
|
||||
}
|
||||
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
|
||||
}
|
||||
const agg =
|
||||
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
|
||||
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
|
||||
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
|
||||
|
||||
// Parameterized query — values travel as HTTP params, never spliced.
|
||||
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
|
||||
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
|
||||
if (q.dataset_id) {
|
||||
conditions.push("dataset_id = {dataset:String}");
|
||||
params.param_dataset = q.dataset_id;
|
||||
}
|
||||
if (q.from_time) {
|
||||
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
|
||||
params.param_fromTime = q.from_time;
|
||||
}
|
||||
if (q.to_time) {
|
||||
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
|
||||
params.param_toTime = q.to_time;
|
||||
}
|
||||
|
||||
const sql =
|
||||
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
|
||||
`FROM ${this.database}.opt_metric_events ` +
|
||||
`WHERE ${conditions.join(" AND ")} ` +
|
||||
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
|
||||
`LIMIT ${limit} FORMAT JSON`;
|
||||
|
||||
const json = (await this.exec(sql, undefined, {
|
||||
readonly: "1",
|
||||
max_execution_time: String(this.maxExecutionTimeSec),
|
||||
max_result_rows: String(this.maxResultRows),
|
||||
...params,
|
||||
})) as { data?: { grp?: string; value: number | string }[] };
|
||||
|
||||
return (json?.data ?? []).map((r) => ({
|
||||
...(r.grp !== undefined ? { group: r.grp } : {}),
|
||||
value: Number(r.value),
|
||||
}));
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
try {
|
||||
await this.exec("SELECT 1 FORMAT JSON");
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
private async exec(
|
||||
sql: string,
|
||||
body?: string,
|
||||
settings?: Record<string, string>
|
||||
): Promise<unknown> {
|
||||
const url = new URL(this.baseUrl);
|
||||
url.searchParams.set("query", sql);
|
||||
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.authHeader) headers["Authorization"] = this.authHeader;
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: body ?? "",
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
|
||||
}
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
// ClickHouse error text can include query fragments — keep only the code.
|
||||
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
|
||||
throw new SiaxError(
|
||||
siaxErrorCodeForUpstreamStatus(res.status),
|
||||
"clickhouse",
|
||||
`query failed (CH code ${code})`
|
||||
);
|
||||
}
|
||||
if (!text) return undefined;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,277 @@
|
||||
/**
|
||||
* src/analytics/contract/compile-clickhouse.ts
|
||||
* =============================================
|
||||
* BI-P016, compile target 1 of 2: the canonical `AnalyticsQueryRequest`
|
||||
* becomes tenant-forced ClickHouse SQL.
|
||||
*
|
||||
* The BI-P011 guarantee is repeated here VERBATIM and by construction:
|
||||
*
|
||||
* * `FORCED_TENANT_PREDICATE` is condition[0] with no branch that can skip
|
||||
* it, and the tenant value comes from the verified identity handed in by
|
||||
* the engine — this function has no way to read one from a request,
|
||||
* because `AnalyticsQueryRequest` has no tenant field.
|
||||
* * every caller VALUE is a `param_*` binding; `sql` never contains one.
|
||||
* * every caller NAME has already been resolved through the catalog
|
||||
* (`resolveAnalyticsQuery`), and the resolved physical name is validated
|
||||
* as an identifier a second time before it reaches the string.
|
||||
*
|
||||
* tests/analytics/query-request-tenant-forcing.test.ts asserts the predicate
|
||||
* survives every request shape this compiler accepts, by ENUMERATING the
|
||||
* shapes rather than spot-checking two of them.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import { assertSafeIdentifier, FORCED_TENANT_PREDICATE, type CompiledQuery } from '../engine/query-compiler';
|
||||
import type { AnalyticsTable } from '../engine/schema';
|
||||
import type { ResolvedQuery, TimeGrain } from './query-request';
|
||||
import type { FilterOperator } from './query-request';
|
||||
|
||||
/** Grain -> ClickHouse truncation function. A closed map, never caller text. */
|
||||
const GRAIN_FN: Record<TimeGrain, string> = {
|
||||
hour: 'toStartOfHour',
|
||||
day: 'toStartOfDay',
|
||||
week: 'toStartOfWeek',
|
||||
month: 'toStartOfMonth',
|
||||
quarter: 'toStartOfQuarter',
|
||||
year: 'toStartOfYear',
|
||||
};
|
||||
|
||||
/**
|
||||
* Aggregation -> conditional-aggregate template. `count_distinct` uses
|
||||
* `uniqExactIf` so a multi-metric request stays one scan instead of N.
|
||||
*
|
||||
* The argument is the WHOLE per-metric predicate, not just the metric-key
|
||||
* comparison, because a metric definition can carry its own filters (BI-P028
|
||||
* `MetricSpec.filters`) and those belong INSIDE this metric's `If` — one
|
||||
* request may carry two metrics that exclude different rows.
|
||||
*/
|
||||
const AGG_IF: Record<string, (predicate: string) => string> = {
|
||||
sum: (p) => `sumIf(value, ${p})`,
|
||||
avg: (p) => `avgIf(value, ${p})`,
|
||||
min: (p) => `minIf(value, ${p})`,
|
||||
max: (p) => `maxIf(value, ${p})`,
|
||||
count: (p) => `countIf(${p})`,
|
||||
count_distinct: (p) => `uniqExactIf(event_id, ${p})`,
|
||||
};
|
||||
|
||||
export interface ClickHouseCompileOptions {
|
||||
/** From the VERIFIED CL0UD identity. Never from a request body. */
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
database: string;
|
||||
table?: AnalyticsTable;
|
||||
/** BI-P012 ceiling; `limit` is clamped to it, never raised by it. */
|
||||
maxResultRows: number;
|
||||
}
|
||||
|
||||
export interface CompiledAnalyticsQuery extends CompiledQuery {
|
||||
/** Column alias -> caller-facing metric name, so the caller never sees `m0`. */
|
||||
metricAliases: Record<string, string>;
|
||||
/** Column alias -> caller-facing dimension name. */
|
||||
dimensionAliases: Record<string, string>;
|
||||
effectiveLimit: number;
|
||||
}
|
||||
|
||||
export function compileRequestToClickHouse(
|
||||
resolved: ResolvedQuery,
|
||||
opts: ClickHouseCompileOptions,
|
||||
): CompiledAnalyticsQuery {
|
||||
if (!opts.tenant_id || !opts.tenant_id.trim()) {
|
||||
// Fail closed. A blank tenant must never compile to "no predicate".
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query');
|
||||
}
|
||||
assertSafeIdentifier(opts.database, 'database');
|
||||
const table: AnalyticsTable = opts.table ?? 'opt_analytics_event';
|
||||
assertSafeIdentifier(table, 'table');
|
||||
|
||||
const params: Record<string, string> = { param_tenant: opts.tenant_id };
|
||||
// ── the forced predicate. First condition, always, no branch. ─────────
|
||||
const conditions: string[] = [FORCED_TENANT_PREDICATE];
|
||||
|
||||
if (opts.project_id) {
|
||||
conditions.push('project_id = {project:String}');
|
||||
params.param_project = opts.project_id;
|
||||
}
|
||||
conditions.push('dataset_id = {dataset:String}');
|
||||
params.param_dataset = resolved.dataset_id;
|
||||
|
||||
// ── metrics: bound keys, compiler-generated aliases ───────────────────
|
||||
const selectParts: string[] = [];
|
||||
const groupParts: string[] = [];
|
||||
const dimensionAliases: Record<string, string> = {};
|
||||
const metricAliases: Record<string, string> = {};
|
||||
const metricKeyParams: string[] = [];
|
||||
|
||||
resolved.dimensions.forEach((d, i) => {
|
||||
assertSafeIdentifier(d.field_name, 'dimension');
|
||||
const key = `d${i}`;
|
||||
const alias = `dim_${i}`;
|
||||
selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS ${alias}`);
|
||||
groupParts.push(alias);
|
||||
params[`param_${key}`] = d.field_name;
|
||||
dimensionAliases[alias] = d.name;
|
||||
});
|
||||
|
||||
// ── time grain: a GROUP BY bucket, from the closed GRAIN_FN map ───────
|
||||
const grain = resolved.request.timeRange?.grain;
|
||||
if (grain) {
|
||||
const timeCol = resolved.timeDimension ?? 'event_time';
|
||||
assertSafeIdentifier(timeCol, 'time dimension');
|
||||
const alias = 'time_bucket';
|
||||
selectParts.push(`${GRAIN_FN[grain]}(${timeCol}) AS ${alias}`);
|
||||
groupParts.push(alias);
|
||||
dimensionAliases[alias] = resolved.request.timeRange?.dimension ?? '__time';
|
||||
}
|
||||
|
||||
resolved.metrics.forEach((m, i) => {
|
||||
const key = `m${i}`;
|
||||
const alias = `metric_${i}`;
|
||||
const template = AGG_IF[m.aggregation];
|
||||
if (!template) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported aggregation "${m.aggregation}"`);
|
||||
}
|
||||
// The metric's own definition filters (BI-P028) join the metric-key
|
||||
// comparison inside THIS metric's `If`. Bound, never spliced, and scoped
|
||||
// to this metric — putting them in the shared WHERE would apply one
|
||||
// metric's meaning to every other metric in the same request.
|
||||
const metricConditions = [`metric = {${key}:String}`];
|
||||
(m.definition_filters ?? []).forEach((f, j) => {
|
||||
assertSafeIdentifier(f.field_name, 'metric definition filter dimension');
|
||||
const nameKey = `mf${i}_${j}`;
|
||||
params[`param_${nameKey}`] = f.field_name;
|
||||
const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`;
|
||||
metricConditions.push(renderFilter(lhs, nameKey, f.operator, [...f.values], `${i}_${j}m`, params));
|
||||
});
|
||||
selectParts.push(`${template(metricConditions.join(' AND '))} AS ${alias}`);
|
||||
params[`param_${key}`] = m.event_metric_key;
|
||||
metricKeyParams.push(`{${key}:String}`);
|
||||
metricAliases[alias] = m.name;
|
||||
});
|
||||
|
||||
// Restrict the SCAN to the requested metric keys — a correctness AND a cost
|
||||
// measure: without it every metric row in the dataset is read to be thrown
|
||||
// away by the `If` conditions.
|
||||
conditions.push(`metric IN (${metricKeyParams.join(', ')})`);
|
||||
|
||||
// ── time range: bound instants, half-open ─────────────────────────────
|
||||
const tr = resolved.request.timeRange;
|
||||
if (tr?.from || tr?.to) {
|
||||
const timeCol = resolved.timeDimension ?? 'event_time';
|
||||
assertSafeIdentifier(timeCol, 'time dimension');
|
||||
if (tr.from) {
|
||||
conditions.push(`${timeCol} >= parseDateTime64BestEffort({fromTime:String})`);
|
||||
params.param_fromTime = tr.from;
|
||||
}
|
||||
if (tr.to) {
|
||||
conditions.push(`${timeCol} < parseDateTime64BestEffort({toTime:String})`);
|
||||
params.param_toTime = tr.to;
|
||||
}
|
||||
}
|
||||
|
||||
// ── filters: closed operator set, values always bound ─────────────────
|
||||
resolved.filters.forEach((entry, i) => {
|
||||
assertSafeIdentifier(entry.dimension.field_name, 'dimension');
|
||||
const nameKey = `fn${i}`;
|
||||
params[`param_${nameKey}`] = entry.dimension.field_name;
|
||||
const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`;
|
||||
conditions.push(
|
||||
renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], String(i), params),
|
||||
);
|
||||
});
|
||||
|
||||
// ── order by: resolved back onto compiler-generated aliases only ──────
|
||||
const aliasForName = new Map<string, string>();
|
||||
for (const [alias, name] of Object.entries(dimensionAliases)) aliasForName.set(name, alias);
|
||||
for (const [alias, name] of Object.entries(metricAliases)) aliasForName.set(name, alias);
|
||||
const orderParts: string[] = [];
|
||||
for (const o of resolved.request.orderBy ?? []) {
|
||||
const alias = aliasForName.get(o.ref);
|
||||
if (!alias) {
|
||||
// Cannot happen after parseAnalyticsQueryRequest, which already refuses
|
||||
// an orderBy that is not selected. Kept as a hard stop rather than a
|
||||
// silent skip: a guard that quietly does nothing is how a bypass hides.
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `orderBy "${o.ref}" is not selected`);
|
||||
}
|
||||
orderParts.push(`${alias} ${o.direction === 'asc' ? 'ASC' : 'DESC'}`);
|
||||
}
|
||||
|
||||
const requested = resolved.request.limit ?? 1000;
|
||||
const effectiveLimit = Math.max(1, Math.min(Math.floor(requested), opts.maxResultRows));
|
||||
|
||||
const sql =
|
||||
`SELECT ${selectParts.join(', ')} ` +
|
||||
`FROM ${opts.database}.${table} ` +
|
||||
`WHERE ${conditions.join(' AND ')}` +
|
||||
(groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') +
|
||||
(orderParts.length ? ` ORDER BY ${orderParts.join(', ')}` : '') +
|
||||
` LIMIT ${effectiveLimit} FORMAT JSON`;
|
||||
|
||||
return {
|
||||
sql,
|
||||
params,
|
||||
forcedTenantPredicate: FORCED_TENANT_PREDICATE,
|
||||
table,
|
||||
metricAliases,
|
||||
dimensionAliases,
|
||||
effectiveLimit,
|
||||
};
|
||||
}
|
||||
|
||||
function renderFilter(
|
||||
lhs: string,
|
||||
nameKey: string,
|
||||
operator: FilterOperator,
|
||||
values: string[],
|
||||
/**
|
||||
* Uniquifies this filter's bound-value parameter names. A STRING, not a
|
||||
* number, because request filters and per-metric definition filters share
|
||||
* one parameter map: `0` and `0_0m` must not collide, and two collided
|
||||
* bindings would silently make one filter win over the other.
|
||||
*/
|
||||
index: string,
|
||||
params: Record<string, string>,
|
||||
): string {
|
||||
const bind = (v: string, suffix: string): string => {
|
||||
const key = `fv${index}${suffix}`;
|
||||
params[`param_${key}`] = v;
|
||||
return `{${key}:String}`;
|
||||
};
|
||||
switch (operator) {
|
||||
case 'eq':
|
||||
return `${lhs} = ${bind(values[0], '')}`;
|
||||
case 'ne':
|
||||
return `${lhs} != ${bind(values[0], '')}`;
|
||||
case 'lt':
|
||||
return `${lhs} < ${bind(values[0], '')}`;
|
||||
case 'lte':
|
||||
return `${lhs} <= ${bind(values[0], '')}`;
|
||||
case 'gt':
|
||||
return `${lhs} > ${bind(values[0], '')}`;
|
||||
case 'gte':
|
||||
return `${lhs} >= ${bind(values[0], '')}`;
|
||||
case 'contains':
|
||||
// `position(haystack, needle) > 0` keeps the needle a BOUND VALUE.
|
||||
// A LIKE with a concatenated pattern would make it SQL text.
|
||||
return `position(${lhs}, ${bind(values[0], '')}) > 0`;
|
||||
case 'in':
|
||||
case 'not_in': {
|
||||
const bound = values.map((v, i) => bind(v, `_${i}`));
|
||||
return `${lhs} ${operator === 'in' ? 'IN' : 'NOT IN'} (${bound.join(', ')})`;
|
||||
}
|
||||
case 'is_null':
|
||||
// Dimension values live in a paired-array map; an absent key yields ''
|
||||
// rather than NULL, so "is null" means "the key is not present". The
|
||||
// key itself is the already-bound `param_fn<i>` — no new binding, and
|
||||
// certainly no spliced identifier.
|
||||
return `NOT has(dim_names, {${nameKey}:String})`;
|
||||
case 'is_not_null':
|
||||
return `has(dim_names, {${nameKey}:String})`;
|
||||
default: {
|
||||
const _never: never = operator;
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics',
|
||||
`unsupported filter operator ${String(_never)}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* src/analytics/contract/compile-semantic.ts
|
||||
* ===========================================
|
||||
* BI-P016, compile target 2 of 2 — and the thing that makes BI-P016's DoD
|
||||
* ("same request can compile to ClickHouse/semantic engine without consumer
|
||||
* rewrite") a testable claim rather than a design intention.
|
||||
*
|
||||
* A `SemanticQuerySpec` is the engine-neutral form a semantic layer consumes.
|
||||
* It is deliberately shaped like MetricFlow's query interface (metrics /
|
||||
* group-by / where / order / limit / time-constraint) so BI-P029's
|
||||
* `MetricFlowProvider` can execute it without an adapter, but it names
|
||||
* nothing MetricFlow-specific: swapping MetricFlow for Cube or a hand-rolled
|
||||
* compiler changes this file's CONSUMER, not this file, and never the
|
||||
* product API.
|
||||
*
|
||||
* The `where` clauses here are STRUCTURED, not SQL strings. A semantic layer
|
||||
* that accepts SQL predicate text (MetricFlow's `--where` does) would put a
|
||||
* raw-SQL hole behind the closed DSL — so the spec carries operator+values
|
||||
* and the provider renders them itself under its own escaping rules.
|
||||
*/
|
||||
import type { FilterOperator, ResolvedQuery, TimeGrain } from './query-request';
|
||||
|
||||
export interface SemanticMetricRef {
|
||||
/** Canonical SIAX metric id — stable across semantic-engine replacement. */
|
||||
metric_id: string;
|
||||
name: string;
|
||||
/**
|
||||
* The metric DEFINITION's own filters, carried through so the semantic
|
||||
* target computes the same number as the ClickHouse target. Dropping them
|
||||
* on one side and not the other is how one canonical metric id comes to
|
||||
* mean two different things — the divergence BI-P029's DoD is written to
|
||||
* catch. Structured, like every other filter here: never predicate text.
|
||||
*/
|
||||
definition_filters?: readonly SemanticWhere[];
|
||||
}
|
||||
|
||||
export interface SemanticGroupBy {
|
||||
dimension_id: string;
|
||||
name: string;
|
||||
/** Set when this group-by is a time bucket rather than a categorical one. */
|
||||
grain?: TimeGrain;
|
||||
}
|
||||
|
||||
export interface SemanticWhere {
|
||||
dimension_id: string;
|
||||
name: string;
|
||||
operator: FilterOperator;
|
||||
values: string[];
|
||||
}
|
||||
|
||||
export interface SemanticOrderBy {
|
||||
ref: string;
|
||||
direction: 'asc' | 'desc';
|
||||
}
|
||||
|
||||
export interface SemanticTimeConstraint {
|
||||
/** Inclusive lower bound. */
|
||||
start?: string;
|
||||
/** EXCLUSIVE upper bound — matched to the ClickHouse target exactly. */
|
||||
end?: string;
|
||||
dimension?: string;
|
||||
}
|
||||
|
||||
export interface SemanticQuerySpec {
|
||||
dataset_id: string;
|
||||
semantic_model_version?: number;
|
||||
metrics: SemanticMetricRef[];
|
||||
groupBy: SemanticGroupBy[];
|
||||
where: SemanticWhere[];
|
||||
orderBy: SemanticOrderBy[];
|
||||
limit: number;
|
||||
timeConstraint?: SemanticTimeConstraint;
|
||||
format: 'json' | 'arrow_ipc';
|
||||
}
|
||||
|
||||
export function compileRequestToSemantic(resolved: ResolvedQuery): SemanticQuerySpec {
|
||||
const groupBy: SemanticGroupBy[] = resolved.dimensions.map((d) => ({
|
||||
dimension_id: d.dimension_id,
|
||||
name: d.name,
|
||||
}));
|
||||
const grain = resolved.request.timeRange?.grain;
|
||||
if (grain) {
|
||||
groupBy.push({
|
||||
dimension_id: resolved.request.timeRange?.dimension ?? '__time',
|
||||
name: resolved.request.timeRange?.dimension ?? '__time',
|
||||
grain,
|
||||
});
|
||||
}
|
||||
return {
|
||||
dataset_id: resolved.dataset_id,
|
||||
semantic_model_version: resolved.semantic_model_version,
|
||||
metrics: resolved.metrics.map((m) => ({
|
||||
metric_id: m.metric_id,
|
||||
name: m.name,
|
||||
...(m.definition_filters?.length
|
||||
? {
|
||||
definition_filters: m.definition_filters.map((f) => ({
|
||||
// Already resolved onto the physical key; the dimension id is
|
||||
// the same value here because the definition names a field, not
|
||||
// a caller-supplied alias.
|
||||
dimension_id: f.field_name,
|
||||
name: f.field_name,
|
||||
operator: f.operator,
|
||||
values: [...f.values],
|
||||
})),
|
||||
}
|
||||
: {}),
|
||||
})),
|
||||
groupBy,
|
||||
where: resolved.filters.map((f) => ({
|
||||
dimension_id: f.dimension.dimension_id,
|
||||
name: f.dimension.name,
|
||||
operator: f.filter.operator,
|
||||
values: f.filter.values ?? [],
|
||||
})),
|
||||
orderBy: (resolved.request.orderBy ?? []).map((o) => ({ ref: o.ref, direction: o.direction })),
|
||||
limit: resolved.request.limit ?? 1000,
|
||||
timeConstraint:
|
||||
resolved.request.timeRange && (resolved.request.timeRange.from || resolved.request.timeRange.to)
|
||||
? {
|
||||
start: resolved.request.timeRange.from,
|
||||
end: resolved.request.timeRange.to,
|
||||
dimension: resolved.request.timeRange.dimension,
|
||||
}
|
||||
: undefined,
|
||||
format: resolved.request.format ?? 'json',
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The comparable SHAPE of a compiled query, target-independent.
|
||||
*
|
||||
* BI-P016's DoD is about semantic equivalence, and the honest way to test
|
||||
* that is to project both targets onto the same tuple and demand equality —
|
||||
* not to eyeball two strings. `tests/analytics/query-request-contract.test.ts`
|
||||
* builds this from the ClickHouse SQL by PARSING the compiled artefact, so a
|
||||
* compiler that quietly drops a filter fails the comparison.
|
||||
*/
|
||||
export interface QueryShape {
|
||||
dataset_id: string;
|
||||
metrics: string[];
|
||||
dimensions: string[];
|
||||
filters: Array<{ dimension: string; operator: FilterOperator; values: string[] }>;
|
||||
order: string[];
|
||||
limit: number;
|
||||
timeFrom?: string;
|
||||
timeTo?: string;
|
||||
grain?: TimeGrain;
|
||||
}
|
||||
|
||||
export function shapeOfSemanticSpec(spec: SemanticQuerySpec): QueryShape {
|
||||
const timeBucket = spec.groupBy.find((g) => g.grain);
|
||||
return {
|
||||
dataset_id: spec.dataset_id,
|
||||
metrics: spec.metrics.map((m) => m.name).sort(),
|
||||
dimensions: spec.groupBy
|
||||
.filter((g) => !g.grain)
|
||||
.map((g) => g.name)
|
||||
.sort(),
|
||||
filters: spec.where
|
||||
.map((w) => ({ dimension: w.name, operator: w.operator, values: [...w.values] }))
|
||||
.sort((a, b) => (a.dimension + a.operator).localeCompare(b.dimension + b.operator)),
|
||||
order: spec.orderBy.map((o) => `${o.ref}:${o.direction}`),
|
||||
limit: spec.limit,
|
||||
timeFrom: spec.timeConstraint?.start,
|
||||
timeTo: spec.timeConstraint?.end,
|
||||
grain: timeBucket?.grain,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,518 @@
|
||||
/**
|
||||
* src/analytics/contract/query-request.ts
|
||||
* ========================================
|
||||
* BI-P016 — QUERY REQUEST CONTRACT.
|
||||
*
|
||||
* The canonical shape every SIAX product/application API speaks when it wants
|
||||
* analytics:
|
||||
*
|
||||
* AnalyticsQueryRequest { dataset metrics[] dimensions[] filters[]
|
||||
* timeRange orderBy limit format }
|
||||
*
|
||||
* BI-P016's hard rule: "The default product/application API must NOT accept
|
||||
* arbitrary SQL." That is enforced structurally, not by a review convention —
|
||||
* there is no field in this type that can carry SQL, and
|
||||
* `parseAnalyticsQueryRequest` REJECTS unknown keys rather than ignoring them.
|
||||
* A caller who adds `sql`, `where`, `having` or `expression` gets
|
||||
* INVALID_REQUEST, not silent truncation; silent truncation is how an
|
||||
* "unsupported" field becomes a supported one after the next refactor.
|
||||
*
|
||||
* Everything in this request is a NAME, resolved through a declared catalog
|
||||
* (`QueryResolution`) before it can reach a compiler. Caller text therefore
|
||||
* never becomes a SQL identifier and never becomes a SQL literal — the two
|
||||
* ways injection happens.
|
||||
*
|
||||
* WHY THIS IS SEPARATE FROM `MetricQueryRequest` (engine/query-compiler.ts):
|
||||
* that type is the ENGINE's internal, already-resolved, single-metric input,
|
||||
* and it carries `tenant_id` because the engine sets it. This one is the
|
||||
* CONSUMER-facing contract: it has no tenant field at all, because a consumer
|
||||
* must never be able to name a tenant. INF0's Wave-2 hole was exactly a tenant
|
||||
* read out of a request body; the fix that sticks is a request type in which
|
||||
* the field does not exist.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
|
||||
// ── vocabulary ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Filter operators. A CLOSED list: every one compiles to a fixed SQL shape
|
||||
* with bound parameters. There is no "raw" or "expression" operator, and
|
||||
* adding one would have to survive `filter-operators-are-closed` in
|
||||
* tests/analytics/query-request-contract.test.ts.
|
||||
*/
|
||||
export const FILTER_OPERATORS = [
|
||||
'eq',
|
||||
'ne',
|
||||
'in',
|
||||
'not_in',
|
||||
'lt',
|
||||
'lte',
|
||||
'gt',
|
||||
'gte',
|
||||
'contains',
|
||||
'is_null',
|
||||
'is_not_null',
|
||||
] as const;
|
||||
export type FilterOperator = (typeof FILTER_OPERATORS)[number];
|
||||
|
||||
/** Operators that take exactly one value. */
|
||||
const SINGLE_VALUE_OPERATORS: readonly FilterOperator[] = ['eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'contains'];
|
||||
/** Operators that take one-or-more values. */
|
||||
const MULTI_VALUE_OPERATORS: readonly FilterOperator[] = ['in', 'not_in'];
|
||||
/** Operators that take NO value. Supplying one is a rejection, not a shrug. */
|
||||
const NO_VALUE_OPERATORS: readonly FilterOperator[] = ['is_null', 'is_not_null'];
|
||||
|
||||
export const TIME_GRAINS = ['hour', 'day', 'week', 'month', 'quarter', 'year'] as const;
|
||||
export type TimeGrain = (typeof TIME_GRAINS)[number];
|
||||
|
||||
export const ORDER_DIRECTIONS = ['asc', 'desc'] as const;
|
||||
export type OrderDirection = (typeof ORDER_DIRECTIONS)[number];
|
||||
|
||||
/** Result encodings, matching `QueryResult.encoding` (BI-P002/BI-P005). */
|
||||
export const REQUEST_FORMATS = ['json', 'arrow_ipc'] as const;
|
||||
export type RequestFormat = (typeof REQUEST_FORMATS)[number];
|
||||
|
||||
// ── the request ────────────────────────────────────────────────────────────
|
||||
|
||||
export interface AnalyticsFilter {
|
||||
/** Caller-facing dimension NAME. Resolved through the catalog, never spliced. */
|
||||
dimension: string;
|
||||
operator: FilterOperator;
|
||||
/** Literal values. They travel as bound parameters; never as SQL text. */
|
||||
values?: string[];
|
||||
}
|
||||
|
||||
export interface AnalyticsTimeRange {
|
||||
/**
|
||||
* Caller-facing time dimension name. Omitted means "the dataset's declared
|
||||
* primary time dimension", which the resolution supplies — a consumer never
|
||||
* has to know the physical column.
|
||||
*/
|
||||
dimension?: string;
|
||||
/** ISO-8601 inclusive lower bound. */
|
||||
from?: string;
|
||||
/** ISO-8601 EXCLUSIVE upper bound (half-open, so adjacent ranges never double count). */
|
||||
to?: string;
|
||||
grain?: TimeGrain;
|
||||
}
|
||||
|
||||
export interface AnalyticsOrderBy {
|
||||
/**
|
||||
* A metric name or a dimension name that ALREADY APPEARS in this request.
|
||||
* Ordering by something not selected is refused: it is both a correctness
|
||||
* trap and the obvious place to smuggle an expression.
|
||||
*/
|
||||
ref: string;
|
||||
direction: OrderDirection;
|
||||
}
|
||||
|
||||
/** BI-P016's canonical request, field-for-field. */
|
||||
export interface AnalyticsQueryRequest {
|
||||
dataset: string;
|
||||
metrics: string[];
|
||||
dimensions: string[];
|
||||
filters: AnalyticsFilter[];
|
||||
timeRange?: AnalyticsTimeRange;
|
||||
orderBy?: AnalyticsOrderBy[];
|
||||
limit?: number;
|
||||
format?: RequestFormat;
|
||||
}
|
||||
|
||||
/** The exact keys the contract accepts at the top level. */
|
||||
export const ANALYTICS_QUERY_REQUEST_KEYS = [
|
||||
'dataset',
|
||||
'metrics',
|
||||
'dimensions',
|
||||
'filters',
|
||||
'timeRange',
|
||||
'orderBy',
|
||||
'limit',
|
||||
'format',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Keys a caller might reach for when they want to smuggle SQL. Listed
|
||||
* explicitly so the rejection MESSAGE names the rule rather than saying
|
||||
* "unknown key" — an integrator who tries `sql` should learn why, not guess.
|
||||
* The unknown-key rejection below covers everything, listed or not.
|
||||
*/
|
||||
const SQL_SMELLING_KEYS = new Set([
|
||||
'sql',
|
||||
'query',
|
||||
'rawSql',
|
||||
'raw_sql',
|
||||
'raw',
|
||||
'where',
|
||||
'having',
|
||||
'expression',
|
||||
'expr',
|
||||
'select',
|
||||
'from',
|
||||
'groupBy',
|
||||
'group_by',
|
||||
'sqlFragment',
|
||||
'unsafe',
|
||||
'tenant',
|
||||
'tenant_id',
|
||||
'tenantId',
|
||||
'project_id',
|
||||
'database',
|
||||
'catalog',
|
||||
'table',
|
||||
]);
|
||||
|
||||
const MAX_METRICS = 16;
|
||||
const MAX_DIMENSIONS = 8;
|
||||
const MAX_FILTERS = 32;
|
||||
const MAX_FILTER_VALUES = 500;
|
||||
const MAX_ORDER_BY = 8;
|
||||
/** A name is a catalog key, not free text; keep it short and boring. */
|
||||
const MAX_NAME_LENGTH = 128;
|
||||
/** A filter VALUE is data (it is bound, never spliced) but still bounded. */
|
||||
const MAX_VALUE_LENGTH = 4096;
|
||||
export const DEFAULT_REQUEST_LIMIT = 1000;
|
||||
|
||||
function invalid(detail: string): never {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', detail);
|
||||
}
|
||||
|
||||
function asObject(raw: unknown, what: string): Record<string, unknown> {
|
||||
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`);
|
||||
return raw as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function asName(raw: unknown, what: string): string {
|
||||
if (typeof raw !== 'string') invalid(`${what} must be a string`);
|
||||
const v = raw.trim();
|
||||
if (!v) invalid(`${what} must not be empty`);
|
||||
if (v.length > MAX_NAME_LENGTH) invalid(`${what} exceeds ${MAX_NAME_LENGTH} characters`);
|
||||
return v;
|
||||
}
|
||||
|
||||
function rejectUnknownKeys(obj: Record<string, unknown>, allowed: readonly string[], what: string): void {
|
||||
for (const key of Object.keys(obj)) {
|
||||
if (allowed.includes(key)) continue;
|
||||
if (SQL_SMELLING_KEYS.has(key)) {
|
||||
invalid(
|
||||
`"${key}" is not part of the analytics query contract — the product API does not accept SQL, ` +
|
||||
`catalogs, or tenant selection (BI-P016)`,
|
||||
);
|
||||
}
|
||||
invalid(`unknown field "${key}" in ${what}`);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse and VALIDATE an untrusted request body into the canonical contract.
|
||||
*
|
||||
* Fail-closed at every branch: an unparseable field is a rejection, never a
|
||||
* default. The only defaulting this function does is `filters: []`,
|
||||
* `dimensions: []` and `limit`, and each of those defaults is strictly
|
||||
* narrower than any value a caller could have supplied.
|
||||
*/
|
||||
export function parseAnalyticsQueryRequest(raw: unknown): AnalyticsQueryRequest {
|
||||
const obj = asObject(raw, 'analytics query request');
|
||||
rejectUnknownKeys(obj, ANALYTICS_QUERY_REQUEST_KEYS, 'analytics query request');
|
||||
|
||||
const dataset = asName(obj.dataset, 'dataset');
|
||||
|
||||
if (!Array.isArray(obj.metrics)) invalid('metrics must be an array');
|
||||
if (obj.metrics.length === 0) invalid('at least one metric is required');
|
||||
if (obj.metrics.length > MAX_METRICS) invalid(`at most ${MAX_METRICS} metrics per request`);
|
||||
const metrics = obj.metrics.map((m, i) => asName(m, `metrics[${i}]`));
|
||||
if (new Set(metrics).size !== metrics.length) invalid('metrics contains duplicates');
|
||||
|
||||
const rawDimensions = obj.dimensions ?? [];
|
||||
if (!Array.isArray(rawDimensions)) invalid('dimensions must be an array');
|
||||
if (rawDimensions.length > MAX_DIMENSIONS) invalid(`at most ${MAX_DIMENSIONS} dimensions per request`);
|
||||
const dimensions = rawDimensions.map((d, i) => asName(d, `dimensions[${i}]`));
|
||||
if (new Set(dimensions).size !== dimensions.length) invalid('dimensions contains duplicates');
|
||||
|
||||
const rawFilters = obj.filters ?? [];
|
||||
if (!Array.isArray(rawFilters)) invalid('filters must be an array');
|
||||
if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters per request`);
|
||||
const filters = rawFilters.map((f, i) => parseFilter(f, i));
|
||||
|
||||
const timeRange = obj.timeRange === undefined ? undefined : parseTimeRange(obj.timeRange);
|
||||
|
||||
const rawOrderBy = obj.orderBy ?? [];
|
||||
if (!Array.isArray(rawOrderBy)) invalid('orderBy must be an array');
|
||||
if (rawOrderBy.length > MAX_ORDER_BY) invalid(`at most ${MAX_ORDER_BY} orderBy entries`);
|
||||
const orderBy = rawOrderBy.map((o, i) => parseOrderBy(o, i));
|
||||
const selectable = new Set<string>([...metrics, ...dimensions]);
|
||||
for (const o of orderBy) {
|
||||
if (!selectable.has(o.ref)) {
|
||||
invalid(`orderBy "${o.ref}" is neither a requested metric nor a requested dimension`);
|
||||
}
|
||||
}
|
||||
|
||||
let limit = DEFAULT_REQUEST_LIMIT;
|
||||
if (obj.limit !== undefined) {
|
||||
if (typeof obj.limit !== 'number' || !Number.isFinite(obj.limit))
|
||||
invalid('limit must be a finite number');
|
||||
if (!Number.isInteger(obj.limit)) invalid('limit must be an integer');
|
||||
if (obj.limit < 1) invalid('limit must be >= 1');
|
||||
limit = obj.limit;
|
||||
}
|
||||
|
||||
let format: RequestFormat = 'json';
|
||||
if (obj.format !== undefined) {
|
||||
if (typeof obj.format !== 'string' || !(REQUEST_FORMATS as readonly string[]).includes(obj.format)) {
|
||||
invalid(`format must be one of ${REQUEST_FORMATS.join(', ')}`);
|
||||
}
|
||||
format = obj.format as RequestFormat;
|
||||
}
|
||||
|
||||
return { dataset, metrics, dimensions, filters, timeRange, orderBy, limit, format };
|
||||
}
|
||||
|
||||
function parseFilter(raw: unknown, index: number): AnalyticsFilter {
|
||||
const obj = asObject(raw, `filters[${index}]`);
|
||||
rejectUnknownKeys(obj, ['dimension', 'operator', 'values'], `filters[${index}]`);
|
||||
const dimension = asName(obj.dimension, `filters[${index}].dimension`);
|
||||
if (typeof obj.operator !== 'string' || !(FILTER_OPERATORS as readonly string[]).includes(obj.operator)) {
|
||||
invalid(`filters[${index}].operator must be one of ${FILTER_OPERATORS.join(', ')}`);
|
||||
}
|
||||
const operator = obj.operator as FilterOperator;
|
||||
|
||||
if (NO_VALUE_OPERATORS.includes(operator)) {
|
||||
if (obj.values !== undefined) invalid(`filters[${index}].operator "${operator}" takes no values`);
|
||||
return { dimension, operator };
|
||||
}
|
||||
|
||||
if (!Array.isArray(obj.values)) invalid(`filters[${index}].values must be an array`);
|
||||
if (obj.values.length === 0) invalid(`filters[${index}].values must not be empty`);
|
||||
if (obj.values.length > MAX_FILTER_VALUES) {
|
||||
invalid(`filters[${index}].values exceeds ${MAX_FILTER_VALUES} entries`);
|
||||
}
|
||||
if (SINGLE_VALUE_OPERATORS.includes(operator) && obj.values.length !== 1) {
|
||||
invalid(`filters[${index}].operator "${operator}" takes exactly one value`);
|
||||
}
|
||||
if (!MULTI_VALUE_OPERATORS.includes(operator) && !SINGLE_VALUE_OPERATORS.includes(operator)) {
|
||||
invalid(`filters[${index}].operator "${operator}" is not a value operator`);
|
||||
}
|
||||
const values = obj.values.map((v, i) => {
|
||||
if (typeof v !== 'string') invalid(`filters[${index}].values[${i}] must be a string`);
|
||||
if (v.length > MAX_VALUE_LENGTH)
|
||||
invalid(`filters[${index}].values[${i}] exceeds ${MAX_VALUE_LENGTH} characters`);
|
||||
return v;
|
||||
});
|
||||
return { dimension, operator, values };
|
||||
}
|
||||
|
||||
function parseTimeRange(raw: unknown): AnalyticsTimeRange {
|
||||
const obj = asObject(raw, 'timeRange');
|
||||
rejectUnknownKeys(obj, ['dimension', 'from', 'to', 'grain'], 'timeRange');
|
||||
const out: AnalyticsTimeRange = {};
|
||||
if (obj.dimension !== undefined) out.dimension = asName(obj.dimension, 'timeRange.dimension');
|
||||
if (obj.from !== undefined) out.from = asIsoInstant(obj.from, 'timeRange.from');
|
||||
if (obj.to !== undefined) out.to = asIsoInstant(obj.to, 'timeRange.to');
|
||||
if (obj.grain !== undefined) {
|
||||
if (typeof obj.grain !== 'string' || !(TIME_GRAINS as readonly string[]).includes(obj.grain)) {
|
||||
invalid(`timeRange.grain must be one of ${TIME_GRAINS.join(', ')}`);
|
||||
}
|
||||
out.grain = obj.grain as TimeGrain;
|
||||
}
|
||||
if (out.from && out.to && Date.parse(out.from) >= Date.parse(out.to)) {
|
||||
invalid('timeRange.from must be strictly before timeRange.to');
|
||||
}
|
||||
return out;
|
||||
}
|
||||
|
||||
/**
|
||||
* A timestamp is validated as a real instant, not merely as "a string that
|
||||
* gets bound". It is bound as a parameter either way, so this is not the
|
||||
* injection defence — it is the defence against a nonsense range silently
|
||||
* producing an empty answer that a dashboard renders as "zero sales".
|
||||
*/
|
||||
function asIsoInstant(raw: unknown, what: string): string {
|
||||
if (typeof raw !== 'string') invalid(`${what} must be an ISO-8601 string`);
|
||||
const v = raw.trim();
|
||||
if (!/^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})?)?$/.test(v)) {
|
||||
invalid(`${what} is not an ISO-8601 timestamp`);
|
||||
}
|
||||
if (Number.isNaN(Date.parse(v))) invalid(`${what} is not a valid timestamp`);
|
||||
return v;
|
||||
}
|
||||
|
||||
function parseOrderBy(raw: unknown, index: number): AnalyticsOrderBy {
|
||||
const obj = asObject(raw, `orderBy[${index}]`);
|
||||
rejectUnknownKeys(obj, ['ref', 'direction'], `orderBy[${index}]`);
|
||||
const ref = asName(obj.ref, `orderBy[${index}].ref`);
|
||||
if (typeof obj.direction !== 'string' || !(ORDER_DIRECTIONS as readonly string[]).includes(obj.direction)) {
|
||||
invalid(`orderBy[${index}].direction must be one of ${ORDER_DIRECTIONS.join(', ')}`);
|
||||
}
|
||||
return { ref, direction: obj.direction as OrderDirection };
|
||||
}
|
||||
|
||||
// ── resolution: names -> declared catalog entries ──────────────────────────
|
||||
|
||||
/**
|
||||
* What a metric NAME resolves to. Supplied from `MetricDefinition` rows
|
||||
* (BI-P002/BI-P028); the compilers never invent one.
|
||||
*/
|
||||
/**
|
||||
* A filter that belongs to the METRIC DEFINITION rather than to the request
|
||||
* (BI-P028 `MetricSpec.filters`) — "net revenue excludes internal transfers"
|
||||
* is part of what the metric MEANS, not something a caller asks for.
|
||||
*
|
||||
* It is carried in ALREADY-RESOLVED form: `field_name` is the physical key
|
||||
* inside `dim_names`, mapped from the definition's dimension name by whoever
|
||||
* builds the resolution. The compilers stay dumb — they bind and render, they
|
||||
* never look a dimension up — and `assertSafeIdentifier` still applies.
|
||||
*
|
||||
* These filters are NOT optional decoration. A compile path that drops them
|
||||
* answers a different question under the same canonical metric id, which is
|
||||
* exactly the divergence BI-P029's DoD exists to catch; see
|
||||
* tests/integration/semantic-parity.integration.test.ts.
|
||||
*/
|
||||
export interface ResolvedMetricFilter {
|
||||
field_name: string;
|
||||
operator: FilterOperator;
|
||||
values: readonly string[];
|
||||
}
|
||||
|
||||
export interface ResolvedMetric {
|
||||
name: string;
|
||||
metric_id: string;
|
||||
/** The `metric` key stored in the analytics event table. */
|
||||
event_metric_key: string;
|
||||
aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count' | 'count_distinct';
|
||||
/** Dimension names this metric may legitimately be sliced by. */
|
||||
allowed_dimensions: readonly string[];
|
||||
unit?: string;
|
||||
/**
|
||||
* The metric definition's own filters. Applied PER METRIC — folded into that
|
||||
* metric's conditional aggregate — never into the shared WHERE clause: a
|
||||
* request may ask for two metrics whose definitions exclude different rows,
|
||||
* and a shared predicate would silently apply one metric's meaning to the
|
||||
* other.
|
||||
*/
|
||||
definition_filters?: readonly ResolvedMetricFilter[];
|
||||
}
|
||||
|
||||
/** What a dimension NAME resolves to. Supplied from `DimensionDefinition` rows. */
|
||||
export interface ResolvedDimension {
|
||||
name: string;
|
||||
dimension_id: string;
|
||||
/** Physical key inside `dim_names`. Validated as an identifier downstream. */
|
||||
field_name: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The declared surface for ONE dataset, for ONE tenant. A compiler is handed
|
||||
* this; it cannot reach past it. An unresolvable name is a rejection.
|
||||
*/
|
||||
export interface QueryResolution {
|
||||
dataset_id: string;
|
||||
/** Semantic model version this resolution came from — part of the cache key (BI-P020). */
|
||||
semantic_model_version?: number;
|
||||
metric(name: string): ResolvedMetric | undefined;
|
||||
dimension(name: string): ResolvedDimension | undefined;
|
||||
/** Physical time dimension used when `timeRange.dimension` is omitted. */
|
||||
defaultTimeDimension?: string;
|
||||
}
|
||||
|
||||
export function queryResolutionFrom(args: {
|
||||
dataset_id: string;
|
||||
metrics: readonly ResolvedMetric[];
|
||||
dimensions: readonly ResolvedDimension[];
|
||||
semantic_model_version?: number;
|
||||
defaultTimeDimension?: string;
|
||||
}): QueryResolution {
|
||||
const metricMap = new Map(args.metrics.map((m) => [m.name, m]));
|
||||
const dimMap = new Map(args.dimensions.map((d) => [d.name, d]));
|
||||
return {
|
||||
dataset_id: args.dataset_id,
|
||||
semantic_model_version: args.semantic_model_version,
|
||||
defaultTimeDimension: args.defaultTimeDimension,
|
||||
// Map lookups: "__proto__" / "constructor" resolve to undefined, as they must.
|
||||
metric: (name) => metricMap.get(name),
|
||||
dimension: (name) => dimMap.get(name),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve every name in a request, or refuse the request.
|
||||
*
|
||||
* This is the single place where "a caller said X" becomes "the catalog
|
||||
* declares Y". Both compile targets (ClickHouse and semantic) consume the
|
||||
* RESOLVED form, which is why BI-P016's DoD holds: the consumer writes one
|
||||
* request and neither target sees consumer text.
|
||||
*/
|
||||
export interface ResolvedQuery {
|
||||
request: AnalyticsQueryRequest;
|
||||
dataset_id: string;
|
||||
metrics: ResolvedMetric[];
|
||||
dimensions: ResolvedDimension[];
|
||||
filters: Array<{ filter: AnalyticsFilter; dimension: ResolvedDimension }>;
|
||||
timeDimension?: string;
|
||||
semantic_model_version?: number;
|
||||
}
|
||||
|
||||
export function resolveAnalyticsQuery(
|
||||
request: AnalyticsQueryRequest,
|
||||
resolution: QueryResolution,
|
||||
): ResolvedQuery {
|
||||
if (request.dataset !== resolution.dataset_id) {
|
||||
// The resolution is built FOR a dataset the caller was authorised to read
|
||||
// (BI-P006 grants). A mismatch means the caller is asking about something
|
||||
// else entirely, and answering from this resolution would be a leak.
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics',
|
||||
'requested dataset does not match the authorised resolution',
|
||||
);
|
||||
}
|
||||
const metrics = request.metrics.map((name) => {
|
||||
const m = resolution.metric(name);
|
||||
if (!m) invalid(`unknown metric "${name}" for this dataset`);
|
||||
return m;
|
||||
});
|
||||
const dimensions = request.dimensions.map((name) => {
|
||||
const d = resolution.dimension(name);
|
||||
if (!d) invalid(`unknown dimension "${name}" for this dataset`);
|
||||
return d;
|
||||
});
|
||||
// Every requested dimension must be legitimate for EVERY requested metric.
|
||||
// Slicing a metric by a dimension its definition does not allow is how a
|
||||
// "harmless" grouping turns into an unintended disclosure.
|
||||
for (const m of metrics) {
|
||||
for (const d of dimensions) {
|
||||
if (!m.allowed_dimensions.includes(d.name)) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics',
|
||||
`metric "${m.name}" may not be sliced by dimension "${d.name}"`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const filters = request.filters.map((f) => {
|
||||
const d = resolution.dimension(f.dimension);
|
||||
if (!d) invalid(`unknown filter dimension "${f.dimension}"`);
|
||||
return { filter: f, dimension: d };
|
||||
});
|
||||
let timeDimension: string | undefined;
|
||||
if (request.timeRange) {
|
||||
if (request.timeRange.dimension) {
|
||||
const d = resolution.dimension(request.timeRange.dimension);
|
||||
if (!d) invalid(`unknown time dimension "${request.timeRange.dimension}"`);
|
||||
timeDimension = d.field_name;
|
||||
} else {
|
||||
timeDimension = resolution.defaultTimeDimension;
|
||||
if ((request.timeRange.from || request.timeRange.to) && !timeDimension) {
|
||||
invalid('dataset declares no default time dimension; timeRange.dimension is required');
|
||||
}
|
||||
}
|
||||
}
|
||||
return {
|
||||
request,
|
||||
dataset_id: resolution.dataset_id,
|
||||
metrics,
|
||||
dimensions,
|
||||
filters,
|
||||
timeDimension,
|
||||
semantic_model_version: resolution.semantic_model_version,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,415 @@
|
||||
/**
|
||||
* src/analytics/cost/query-budget.ts
|
||||
* ===================================
|
||||
* BI-P021 — QUERY COST / BUDGET.
|
||||
*
|
||||
* ── SAY THE HONEST THING FIRST ────────────────────────────────────────────
|
||||
* A budget checked once before execution and never again bounds NOTHING. The
|
||||
* estimate is a planner's guess; a query that scans ten times its estimate
|
||||
* runs to completion regardless, and the budget discovers this after the money
|
||||
* is spent. Every "cost control" that consists only of a pre-flight check is
|
||||
* an accounting feature wearing a safety feature's name.
|
||||
*
|
||||
* ClickHouse's own `max_rows_to_read` (BI-P012) DOES stop a query mid-flight —
|
||||
* but it bounds ONE query. It cannot express "this tenant may scan 10 billion
|
||||
* rows per hour across all its queries", because it has no idea what the
|
||||
* tenant's other queries did. That window is what a budget is for, and it is
|
||||
* exactly the thing a per-query setting cannot enforce.
|
||||
*
|
||||
* So this module has three parts, and only the second one is a control:
|
||||
*
|
||||
* 1. ADMISSION — estimate vs. remaining window. Rejects the obviously
|
||||
* unaffordable before it starts. Cheap, and insufficient.
|
||||
* 2. IN-FLIGHT — polls the engine's live `read_rows`/`read_bytes` counters
|
||||
* for the running query and CANCELS it (via BI-P019, with
|
||||
* verification) the moment window spend would be exceeded.
|
||||
* THIS is what makes the budget real.
|
||||
* 3. ACCOUNTING — settles the actual cost into a ledger the control plane
|
||||
* reads: CL0UD quotas, AUD0 usage events, B00K accounting.
|
||||
*
|
||||
* If part 2 is disabled (`enforceInFlight: false`), `describeEnforcement()`
|
||||
* says so in plain words, and the admission-only posture is reported as
|
||||
* "advisory". A control that does not control should say which one it is.
|
||||
*
|
||||
* ── THE CAVEAT THAT THE LIVE RUN MEASURED ────────────────────────────────
|
||||
* In-flight enforcement is granular to the poll interval, so a query can
|
||||
* OVERSHOOT its budget between polls — and on a columnar engine the overshoot
|
||||
* is large. Measured against ClickHouse 24.8.14.39 with a 1,000,000-row window
|
||||
* and a 100ms poll: the query was cancelled at 153,318,696 rows read, 55ms in.
|
||||
* A 153x overshoot. See evidence/bi-p019-p021/cancellation-and-budget.json,
|
||||
* which is regenerated by the live suite — the figure is machine-dependent and
|
||||
* moves between runs (an earlier run on the same box recorded 131M / 130x).
|
||||
* Do not quote a specific multiple; quote the SHAPE.
|
||||
*
|
||||
* That shape is still a bound — the same query, unwatched, read 4,227,318,261
|
||||
* rows before it was stopped in the BI-P019 case — but it is a bound of the
|
||||
* form "within roughly one poll interval of engine throughput", not "at
|
||||
* exactly the budget". Anyone sizing a budget must size it for that, and
|
||||
* anyone quoting this as a hard cap is wrong. Shortening the interval narrows
|
||||
* the overshoot and costs a `system.processes` read per poll per running
|
||||
* query.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import {
|
||||
cancelAndVerify,
|
||||
type CancellationOutcome,
|
||||
type EngineProcessControl,
|
||||
} from '../execution/cancellation';
|
||||
|
||||
/** The cost axes BI-P021 names. */
|
||||
export interface QueryCost {
|
||||
rows_scanned: number;
|
||||
bytes_scanned: number;
|
||||
execution_ms: number;
|
||||
engine: string;
|
||||
/** Bytes read from object storage, when the engine distinguishes it. */
|
||||
storage_read_bytes?: number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Weights turning raw counters into the single `cost_units` figure BI-P018
|
||||
* records and B00K bills.
|
||||
*
|
||||
* The defaults are deliberately dominated by BYTES, not rows: on a columnar
|
||||
* engine a billion-row scan of one narrow column is cheap and a
|
||||
* hundred-million-row scan of a wide one is not, so a rows-weighted unit would
|
||||
* bill the wrong query.
|
||||
*/
|
||||
export interface CostModel {
|
||||
unitsPerBillionRows: number;
|
||||
unitsPerGigabyteScanned: number;
|
||||
unitsPerSecond: number;
|
||||
unitsPerGigabyteStorageRead: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_COST_MODEL: CostModel = {
|
||||
unitsPerBillionRows: 1,
|
||||
unitsPerGigabyteScanned: 10,
|
||||
unitsPerSecond: 2,
|
||||
unitsPerGigabyteStorageRead: 15,
|
||||
};
|
||||
|
||||
export function costUnits(cost: QueryCost, model: CostModel = DEFAULT_COST_MODEL): number {
|
||||
const units =
|
||||
(cost.rows_scanned / 1e9) * model.unitsPerBillionRows +
|
||||
(cost.bytes_scanned / 2 ** 30) * model.unitsPerGigabyteScanned +
|
||||
(cost.execution_ms / 1000) * model.unitsPerSecond +
|
||||
((cost.storage_read_bytes ?? 0) / 2 ** 30) * model.unitsPerGigabyteStorageRead;
|
||||
// Rounded to six places so the same cost always serialises identically;
|
||||
// float noise in a billing figure produces support tickets.
|
||||
return Math.round(units * 1e6) / 1e6;
|
||||
}
|
||||
|
||||
/** A tenant's allowance over a rolling window. Issued by CL0UD entitlements. */
|
||||
export interface QueryBudget {
|
||||
windowMs: number;
|
||||
maxRowsScanned: number;
|
||||
maxBytesScanned: number;
|
||||
maxCostUnits: number;
|
||||
/** What to do when the window is exhausted. */
|
||||
onExhausted: 'reject' | 'throttle';
|
||||
/** For `throttle`: queries are delayed by this much instead of refused. */
|
||||
throttleDelayMs?: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_QUERY_BUDGET: QueryBudget = {
|
||||
windowMs: 60 * 60 * 1000,
|
||||
maxRowsScanned: 50_000_000_000,
|
||||
maxBytesScanned: 2 * 2 ** 40,
|
||||
maxCostUnits: 10_000,
|
||||
onExhausted: 'reject',
|
||||
};
|
||||
|
||||
interface LedgerEntry {
|
||||
at: number;
|
||||
rows: number;
|
||||
bytes: number;
|
||||
units: number;
|
||||
}
|
||||
|
||||
/** Where settled cost goes. AUD0/B00K/CL0UD sit behind this. */
|
||||
export interface UsageLedgerSink {
|
||||
record(entry: {
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
principal_id: string;
|
||||
query_id: string;
|
||||
cost: QueryCost;
|
||||
cost_units: number;
|
||||
/** True when the query was stopped by the budget rather than by the user. */
|
||||
terminated_by_budget: boolean;
|
||||
at: string;
|
||||
}): Promise<void> | void;
|
||||
}
|
||||
|
||||
export interface WindowSpend {
|
||||
rows_scanned: number;
|
||||
bytes_scanned: number;
|
||||
cost_units: number;
|
||||
queries: number;
|
||||
window_started_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory rolling window per tenant.
|
||||
*
|
||||
* Explicitly NOT the durable ledger — this is one process's view, and a
|
||||
* multi-replica deployment needs the shared counter (Valkey or the CL0UD quota
|
||||
* service) behind `SharedWindowStore` for the budget to bind across replicas.
|
||||
* Stated rather than implied: with N replicas and no shared store, a tenant
|
||||
* can spend N times its budget. `describeEnforcement()` reports this.
|
||||
*/
|
||||
export interface SharedWindowStore {
|
||||
/** Atomically add to the tenant's window and return the new totals. */
|
||||
add(
|
||||
tenantId: string,
|
||||
entry: { rows: number; bytes: number; units: number; windowMs: number },
|
||||
): Promise<WindowSpend>;
|
||||
get(tenantId: string): Promise<WindowSpend>;
|
||||
}
|
||||
|
||||
export class InMemoryWindowStore implements SharedWindowStore {
|
||||
private readonly entries = new Map<string, LedgerEntry[]>();
|
||||
constructor(private readonly now: () => number = Date.now) {}
|
||||
|
||||
private prune(tenantId: string, windowMs: number): LedgerEntry[] {
|
||||
const cutoff = this.now() - windowMs;
|
||||
const kept = (this.entries.get(tenantId) ?? []).filter((e) => e.at >= cutoff);
|
||||
this.entries.set(tenantId, kept);
|
||||
return kept;
|
||||
}
|
||||
|
||||
async add(
|
||||
tenantId: string,
|
||||
entry: { rows: number; bytes: number; units: number; windowMs: number },
|
||||
): Promise<WindowSpend> {
|
||||
const kept = this.prune(tenantId, entry.windowMs);
|
||||
kept.push({ at: this.now(), rows: entry.rows, bytes: entry.bytes, units: entry.units });
|
||||
this.entries.set(tenantId, kept);
|
||||
return this.totals(kept, entry.windowMs);
|
||||
}
|
||||
|
||||
async get(tenantId: string, windowMs = DEFAULT_QUERY_BUDGET.windowMs): Promise<WindowSpend> {
|
||||
return this.totals(this.prune(tenantId, windowMs), windowMs);
|
||||
}
|
||||
|
||||
private totals(entries: LedgerEntry[], windowMs: number): WindowSpend {
|
||||
return {
|
||||
rows_scanned: entries.reduce((a, e) => a + e.rows, 0),
|
||||
bytes_scanned: entries.reduce((a, e) => a + e.bytes, 0),
|
||||
cost_units: entries.reduce((a, e) => a + e.units, 0),
|
||||
queries: entries.length,
|
||||
window_started_at: new Date(this.now() - windowMs).toISOString(),
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
export interface BudgetEnforcerOptions {
|
||||
budget: QueryBudget;
|
||||
store: SharedWindowStore;
|
||||
ledger: UsageLedgerSink;
|
||||
control?: EngineProcessControl;
|
||||
costModel?: CostModel;
|
||||
/** The control. Off means admission-only, and `describeEnforcement` says so. */
|
||||
enforceInFlight?: boolean;
|
||||
pollIntervalMs?: number;
|
||||
/** True when `store` is shared across replicas. Affects the honest description. */
|
||||
storeIsShared?: boolean;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export interface AdmissionDecision {
|
||||
admitted: boolean;
|
||||
reason?: string;
|
||||
/** For `throttle`: how long the caller must wait first. */
|
||||
delay_ms?: number;
|
||||
remaining: { rows: number; bytes: number; units: number };
|
||||
}
|
||||
|
||||
export interface InFlightOutcome {
|
||||
/** True when the enforcer stopped the query. */
|
||||
terminated: boolean;
|
||||
reason?: 'rows' | 'bytes' | 'units';
|
||||
/** Present when it terminated: the verified cancellation. */
|
||||
cancellation?: CancellationOutcome;
|
||||
observed: { rows_read: number; bytes_read: number; elapsed_ms: number };
|
||||
polls: number;
|
||||
}
|
||||
|
||||
export class QueryBudgetEnforcer {
|
||||
private readonly opts: Required<
|
||||
Pick<BudgetEnforcerOptions, 'enforceInFlight' | 'pollIntervalMs' | 'storeIsShared'>
|
||||
> &
|
||||
BudgetEnforcerOptions;
|
||||
private readonly now: () => number;
|
||||
private readonly sleep: (ms: number) => Promise<void>;
|
||||
|
||||
constructor(options: BudgetEnforcerOptions) {
|
||||
this.opts = {
|
||||
enforceInFlight: options.enforceInFlight ?? true,
|
||||
pollIntervalMs: options.pollIntervalMs ?? 200,
|
||||
storeIsShared: options.storeIsShared ?? false,
|
||||
...options,
|
||||
};
|
||||
this.now = options.now ?? Date.now;
|
||||
this.sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
|
||||
if (this.opts.enforceInFlight && !this.opts.control) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.cost',
|
||||
'in-flight budget enforcement requires an EngineProcessControl; ' +
|
||||
'construct with enforceInFlight:false to run admission-only and be honest about it',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Exactly what this enforcer does and does not bound, in words an operator
|
||||
* can paste into an incident review. Returned by the analytics health
|
||||
* endpoint so the posture is visible without reading this file.
|
||||
*/
|
||||
describeEnforcement(): {
|
||||
admission: string;
|
||||
in_flight: string;
|
||||
across_replicas: string;
|
||||
binding: boolean;
|
||||
} {
|
||||
return {
|
||||
admission: 'estimate is compared to the remaining window before the query starts',
|
||||
in_flight: this.opts.enforceInFlight
|
||||
? `live engine counters are polled every ${this.opts.pollIntervalMs}ms and the query is cancelled ` +
|
||||
'(with verification) when window spend would be exceeded'
|
||||
: 'NOT ENFORCED — this budget is advisory. A query that exceeds its estimate runs to completion.',
|
||||
across_replicas: this.opts.storeIsShared
|
||||
? 'window spend is held in a shared store, so the budget binds across replicas'
|
||||
: 'window spend is per-process — with N replicas a tenant can spend up to N times this budget',
|
||||
binding: this.opts.enforceInFlight && this.opts.storeIsShared,
|
||||
};
|
||||
}
|
||||
|
||||
async admit(
|
||||
tenantId: string,
|
||||
estimate: { rows_to_read: number; bytes_to_read: number },
|
||||
): Promise<AdmissionDecision> {
|
||||
const spend = await this.opts.store.get(tenantId);
|
||||
const b = this.opts.budget;
|
||||
const remaining = {
|
||||
rows: Math.max(0, b.maxRowsScanned - spend.rows_scanned),
|
||||
bytes: Math.max(0, b.maxBytesScanned - spend.bytes_scanned),
|
||||
units: Math.max(0, b.maxCostUnits - spend.cost_units),
|
||||
};
|
||||
const over =
|
||||
estimate.rows_to_read > remaining.rows
|
||||
? 'rows'
|
||||
: estimate.bytes_to_read > remaining.bytes
|
||||
? 'bytes'
|
||||
: undefined;
|
||||
if (!over) return { admitted: true, remaining };
|
||||
if (b.onExhausted === 'throttle') {
|
||||
return {
|
||||
admitted: true,
|
||||
delay_ms: b.throttleDelayMs ?? 1000,
|
||||
reason: `tenant window ${over} budget exhausted; throttled`,
|
||||
remaining,
|
||||
};
|
||||
}
|
||||
return { admitted: false, reason: `tenant window ${over} budget exhausted`, remaining };
|
||||
}
|
||||
|
||||
/**
|
||||
* Watch a RUNNING query and stop it if it breaches the window.
|
||||
*
|
||||
* Runs concurrently with the query itself; the caller awaits both. Returns
|
||||
* as soon as the query leaves the engine's process list, so a well-behaved
|
||||
* query costs one extra poll.
|
||||
*/
|
||||
async enforceInFlight(tenantId: string, queryId: string, signal?: AbortSignal): Promise<InFlightOutcome> {
|
||||
const observed = { rows_read: 0, bytes_read: 0, elapsed_ms: 0 };
|
||||
if (!this.opts.enforceInFlight || !this.opts.control?.progress) {
|
||||
return { terminated: false, observed, polls: 0 };
|
||||
}
|
||||
const control = this.opts.control;
|
||||
const spendBefore = await this.opts.store.get(tenantId);
|
||||
const b = this.opts.budget;
|
||||
const remainingRows = Math.max(0, b.maxRowsScanned - spendBefore.rows_scanned);
|
||||
const remainingBytes = Math.max(0, b.maxBytesScanned - spendBefore.bytes_scanned);
|
||||
const remainingUnits = Math.max(0, b.maxCostUnits - spendBefore.cost_units);
|
||||
|
||||
let polls = 0;
|
||||
for (;;) {
|
||||
if (signal?.aborted) return { terminated: false, observed, polls };
|
||||
let progress;
|
||||
try {
|
||||
progress = await control.progress?.(queryId);
|
||||
} catch {
|
||||
progress = undefined;
|
||||
}
|
||||
polls += 1;
|
||||
if (!progress) {
|
||||
// Gone from the process list: finished, or never started. Either way
|
||||
// there is nothing left to bound.
|
||||
return { terminated: false, observed, polls };
|
||||
}
|
||||
observed.rows_read = progress.rows_read;
|
||||
observed.bytes_read = progress.bytes_read;
|
||||
observed.elapsed_ms = progress.elapsed_ms;
|
||||
|
||||
const liveUnits = costUnits(
|
||||
{
|
||||
rows_scanned: progress.rows_read,
|
||||
bytes_scanned: progress.bytes_read,
|
||||
execution_ms: progress.elapsed_ms,
|
||||
engine: control.engine,
|
||||
},
|
||||
this.opts.costModel,
|
||||
);
|
||||
const breach =
|
||||
progress.rows_read > remainingRows
|
||||
? 'rows'
|
||||
: progress.bytes_read > remainingBytes
|
||||
? 'bytes'
|
||||
: liveUnits > remainingUnits
|
||||
? 'units'
|
||||
: undefined;
|
||||
|
||||
if (breach) {
|
||||
const cancellation = await cancelAndVerify(control, queryId, { timeoutMs: 5_000 });
|
||||
return { terminated: true, reason: breach, cancellation, observed, polls };
|
||||
}
|
||||
await this.sleep(this.opts.pollIntervalMs);
|
||||
}
|
||||
}
|
||||
|
||||
/** Settle the actual cost. Called for EVERY outcome, including failures. */
|
||||
async settle(args: {
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
principal_id: string;
|
||||
query_id: string;
|
||||
cost: QueryCost;
|
||||
terminated_by_budget?: boolean;
|
||||
}): Promise<{ cost_units: number; window: WindowSpend }> {
|
||||
const units = costUnits(args.cost, this.opts.costModel);
|
||||
const window = await this.opts.store.add(args.tenant_id, {
|
||||
rows: args.cost.rows_scanned,
|
||||
bytes: args.cost.bytes_scanned,
|
||||
units,
|
||||
windowMs: this.opts.budget.windowMs,
|
||||
});
|
||||
await this.opts.ledger.record({
|
||||
tenant_id: args.tenant_id,
|
||||
project_id: args.project_id,
|
||||
principal_id: args.principal_id,
|
||||
query_id: args.query_id,
|
||||
cost: args.cost,
|
||||
cost_units: units,
|
||||
// A query the budget killed still COST what it burned before it died.
|
||||
// Not charging for it is how a tenant discovers that the cheapest query
|
||||
// is one that gets cancelled.
|
||||
terminated_by_budget: args.terminated_by_budget ?? false,
|
||||
at: new Date(this.now()).toISOString(),
|
||||
});
|
||||
return { cost_units: units, window };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,837 @@
|
||||
/**
|
||||
* src/analytics/dashboards/dashboard-document.ts
|
||||
* ===============================================
|
||||
* BI-P060 — ANALYTICS DASHBOARD MODEL.
|
||||
*
|
||||
* BI-P002 already named `DashboardDefinition` and `DashboardVersion` and gave
|
||||
* them storage (sql/008). What it did not give them is a BODY: `layout_json`
|
||||
* is a string, and a string can hold anything — including a Superset export
|
||||
* that only Superset can read back. This module is that body, and its whole
|
||||
* point is the DoD:
|
||||
*
|
||||
* "A native SIAX dashboard can be reconstructed without Superset."
|
||||
*
|
||||
* Reconstruction is not a claim you can make by writing a type. It is provable
|
||||
* exactly once you can take a document and produce, for every widget, a
|
||||
* runnable `AnalyticsQueryRequest` (BI-P016) plus a rendering spec, with no
|
||||
* vendor identifier anywhere in the plan. `planDashboard()` is that function
|
||||
* and `tests/analytics/dashboard-model.test.ts` runs it over a document
|
||||
* carrying Superset ids, asserting the plan is complete and vendor-free.
|
||||
*
|
||||
* THE VENDOR RULE, stated once: a Superset/Metabase/Cube identifier may be
|
||||
* carried as `vendor_ref` for round-tripping, and it is
|
||||
* - never a widget's source of truth (a widget with only a vendor_ref is
|
||||
* REFUSED — it is precisely the document that cannot be reconstructed),
|
||||
* - never part of the canonical hash (so re-exporting to a new Superset
|
||||
* instance does not fabricate a new dashboard version).
|
||||
*
|
||||
* NO SQL LIVES HERE either. Widget sources are metric/dimension NAMES that
|
||||
* resolve through the catalog, and query widgets are validated by
|
||||
* `parseAnalyticsQueryRequest`, which rejects `sql`, `where`, `tenant_id` and
|
||||
* every other smuggling key structurally. A dashboard is therefore not a new
|
||||
* way into the engine; it is a saved set of contract requests.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import {
|
||||
parseAnalyticsQueryRequest,
|
||||
TIME_GRAINS,
|
||||
type AnalyticsFilter,
|
||||
type AnalyticsQueryRequest,
|
||||
type FilterOperator,
|
||||
type TimeGrain,
|
||||
} from '../contract/query-request';
|
||||
|
||||
// ── vocabulary ─────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Visualization types. A CLOSED list, deliberately engine-neutral: these are
|
||||
* SIAX chart kinds (DES1GN BI-P051..P057 renders them), not Superset viz_type
|
||||
* strings. A document naming `echarts_timeseries` is a Superset document.
|
||||
*/
|
||||
export const VISUALIZATION_TYPES = [
|
||||
'big_number',
|
||||
'line',
|
||||
'area',
|
||||
'bar',
|
||||
'stacked_bar',
|
||||
'pie',
|
||||
'table',
|
||||
'pivot_table',
|
||||
'scatter',
|
||||
'heatmap',
|
||||
'funnel',
|
||||
'map',
|
||||
] as const;
|
||||
export type VisualizationType = (typeof VISUALIZATION_TYPES)[number];
|
||||
|
||||
export const REFRESH_MODES = ['manual', 'interval', 'scheduled'] as const;
|
||||
export type RefreshMode = (typeof REFRESH_MODES)[number];
|
||||
|
||||
/**
|
||||
* Who may open the dashboard. `roles` are cl0ud role names (BI-P033 decides,
|
||||
* this only declares). There is deliberately NO tenant field: a dashboard is
|
||||
* stored inside a tenant, and a document that could name one would be a way to
|
||||
* ask for another tenant's dashboard — the INF0 Wave-2 shape.
|
||||
*/
|
||||
export const DASHBOARD_VISIBILITIES = ['private', 'tenant', 'roles'] as const;
|
||||
export type DashboardVisibility = (typeof DASHBOARD_VISIBILITIES)[number];
|
||||
|
||||
export const PARAMETER_TYPES = ['string', 'number', 'date', 'enum'] as const;
|
||||
export type ParameterType = (typeof PARAMETER_TYPES)[number];
|
||||
|
||||
// ── the document ───────────────────────────────────────────────────────────
|
||||
|
||||
/** Grid placement. A 12-column grid, matching the portal layout (BI-P062). */
|
||||
export interface WidgetLayout {
|
||||
x: number;
|
||||
y: number;
|
||||
w: number;
|
||||
h: number;
|
||||
}
|
||||
|
||||
/** A saved dashboard filter. Its VALUES are data; its dimension is a name. */
|
||||
export interface DashboardFilter {
|
||||
filter_id: string;
|
||||
label: string;
|
||||
/** Caller-facing dimension name, resolved through the catalog downstream. */
|
||||
dimension: string;
|
||||
operator: FilterOperator;
|
||||
/** Default values, or `{{param}}` references to a declared parameter. */
|
||||
default_values?: string[];
|
||||
/** May a viewer change it? A locked filter is how row scoping is saved. */
|
||||
locked?: boolean;
|
||||
}
|
||||
|
||||
/** A declared parameter. Widgets and filters may reference it as `{{name}}`. */
|
||||
export interface DashboardParameter {
|
||||
name: string;
|
||||
type: ParameterType;
|
||||
required: boolean;
|
||||
default_value?: string;
|
||||
/** Required for `enum`; a closed list is the only way a value can be trusted. */
|
||||
allowed_values?: string[];
|
||||
}
|
||||
|
||||
/** A widget backed by ONE metric — the common case, and the cheapest to plan. */
|
||||
export interface MetricWidgetSource {
|
||||
kind: 'metric';
|
||||
dataset: string;
|
||||
metric: string;
|
||||
dimensions?: string[];
|
||||
grain?: TimeGrain;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
/** A widget backed by a saved contract request (still no SQL, by construction). */
|
||||
export interface QueryWidgetSource {
|
||||
kind: 'query';
|
||||
/** Parsed with `parseAnalyticsQueryRequest` at validation time, not at render time. */
|
||||
request: AnalyticsQueryRequest;
|
||||
}
|
||||
|
||||
export type WidgetSource = MetricWidgetSource | QueryWidgetSource;
|
||||
|
||||
export interface VisualizationSpec {
|
||||
type: VisualizationType;
|
||||
/** Field roles by NAME. `x`/`y` name metrics or dimensions in the source. */
|
||||
encoding: {
|
||||
x?: string;
|
||||
y?: string[];
|
||||
series?: string;
|
||||
value?: string;
|
||||
};
|
||||
/** Presentation-only knobs. Bounded scalars; never code, never a template. */
|
||||
options?: Record<string, string | number | boolean>;
|
||||
}
|
||||
|
||||
export interface DashboardWidget {
|
||||
widget_id: string;
|
||||
title: string;
|
||||
source: WidgetSource;
|
||||
visualization: VisualizationSpec;
|
||||
layout: WidgetLayout;
|
||||
/** filter_ids this widget honours. Empty means "all dashboard filters". */
|
||||
filter_bindings?: string[];
|
||||
/** Advisory only: excluded from the canonical hash, never a source of truth. */
|
||||
vendor_ref?: VendorRef;
|
||||
}
|
||||
|
||||
/** A foreign system's id for the same object. Round-tripping aid, nothing more. */
|
||||
export interface VendorRef {
|
||||
system: string;
|
||||
id: string;
|
||||
}
|
||||
|
||||
export interface DashboardRefresh {
|
||||
mode: RefreshMode;
|
||||
/** Required for `interval`. */
|
||||
interval_seconds?: number;
|
||||
/** Required for `scheduled`. */
|
||||
cron?: string;
|
||||
/**
|
||||
* The staleness the OWNER accepts. BI-P026 computes actual freshness; this
|
||||
* is the number that makes a freshness verdict actionable rather than
|
||||
* decorative, and BI-P105 reads it when deciding whether a pre-aggregate may
|
||||
* serve this dashboard.
|
||||
*/
|
||||
max_staleness_seconds: number;
|
||||
}
|
||||
|
||||
export interface DashboardPermissions {
|
||||
visibility: DashboardVisibility;
|
||||
/** Required (non-empty) when visibility is `roles`. */
|
||||
roles?: string[];
|
||||
/** Optional cl0ud entitlement key (BI-P079). */
|
||||
required_entitlement?: string;
|
||||
}
|
||||
|
||||
/** BI-P060's canonical document, field-for-field with the program's list. */
|
||||
export interface DashboardDocument {
|
||||
id: string;
|
||||
title: string;
|
||||
description?: string;
|
||||
layout: { grid_columns: number; row_height_px: number };
|
||||
filters: DashboardFilter[];
|
||||
widgets: DashboardWidget[];
|
||||
parameters: DashboardParameter[];
|
||||
refresh: DashboardRefresh;
|
||||
permissions: DashboardPermissions;
|
||||
/** Document schema version — bumped when THIS shape changes, not per edit. */
|
||||
version: number;
|
||||
vendor_ref?: VendorRef;
|
||||
}
|
||||
|
||||
export const DASHBOARD_DOCUMENT_KEYS = [
|
||||
'id',
|
||||
'title',
|
||||
'description',
|
||||
'layout',
|
||||
'filters',
|
||||
'widgets',
|
||||
'parameters',
|
||||
'refresh',
|
||||
'permissions',
|
||||
'version',
|
||||
'vendor_ref',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Keys that mean "this document is really a Superset export". Named
|
||||
* explicitly so the refusal explains the architecture instead of saying
|
||||
* "unknown key"; the unknown-key rule below catches everything else anyway.
|
||||
*/
|
||||
const VENDOR_NATIVE_KEYS = new Set([
|
||||
'slice_id',
|
||||
'slice_name',
|
||||
'viz_type',
|
||||
'datasource',
|
||||
'datasource_id',
|
||||
'datasource_type',
|
||||
'form_data',
|
||||
'query_context',
|
||||
'chart_id',
|
||||
'dashboard_id_superset',
|
||||
'position_json',
|
||||
'json_metadata',
|
||||
'card_id',
|
||||
'cube',
|
||||
]);
|
||||
|
||||
/** Keys that would turn a dashboard into an SQL surface. */
|
||||
const SQL_SMELLING_KEYS = new Set([
|
||||
'sql',
|
||||
'query_text',
|
||||
'statement',
|
||||
'expression',
|
||||
'where',
|
||||
'having',
|
||||
'raw',
|
||||
'tenant',
|
||||
'tenant_id',
|
||||
'project_id',
|
||||
'database',
|
||||
'table',
|
||||
]);
|
||||
|
||||
const MAX_WIDGETS = 60;
|
||||
const MAX_FILTERS = 32;
|
||||
const MAX_PARAMETERS = 24;
|
||||
const MAX_NAME_LENGTH = 128;
|
||||
const MAX_TITLE_LENGTH = 200;
|
||||
const MAX_GRID_COLUMNS = 24;
|
||||
const MIN_REFRESH_INTERVAL_SECONDS = 10;
|
||||
const MAX_STALENESS_SECONDS = 30 * 24 * 3600;
|
||||
/** `{{param}}` — the only templating the document has, and it binds a VALUE. */
|
||||
const PARAM_REFERENCE = /^\{\{\s*([A-Za-z][A-Za-z0-9_]{0,63})\s*\}\}$/;
|
||||
const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/;
|
||||
|
||||
function invalid(detail: string): never {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dashboard', detail);
|
||||
}
|
||||
|
||||
function asObject(raw: unknown, what: string): Record<string, unknown> {
|
||||
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`);
|
||||
return raw as Record<string, unknown>;
|
||||
}
|
||||
|
||||
function rejectUnknownKeys(obj: Record<string, unknown>, allowed: readonly string[], what: string): void {
|
||||
for (const key of Object.keys(obj)) {
|
||||
if (allowed.includes(key)) continue;
|
||||
if (VENDOR_NATIVE_KEYS.has(key)) {
|
||||
invalid(
|
||||
`"${key}" in ${what} is a vendor-native field — a SIAX dashboard is defined by metrics, ` +
|
||||
`queries and a visualization spec, not by a BI tool's internal ids (BI-P060). ` +
|
||||
`Carry the id in "vendor_ref" if you need to round-trip it.`,
|
||||
);
|
||||
}
|
||||
if (SQL_SMELLING_KEYS.has(key)) {
|
||||
invalid(`"${key}" in ${what} is not part of the dashboard model — dashboards do not carry SQL`);
|
||||
}
|
||||
invalid(`unknown field "${key}" in ${what}`);
|
||||
}
|
||||
}
|
||||
|
||||
function asString(raw: unknown, what: string, max = MAX_NAME_LENGTH): string {
|
||||
if (typeof raw !== 'string') invalid(`${what} must be a string`);
|
||||
const v = raw.trim();
|
||||
if (!v) invalid(`${what} must not be empty`);
|
||||
if (v.length > max) invalid(`${what} exceeds ${max} characters`);
|
||||
return v;
|
||||
}
|
||||
|
||||
function asIdentifier(raw: unknown, what: string): string {
|
||||
const v = asString(raw, what);
|
||||
if (!IDENTIFIER.test(v)) invalid(`${what} is not a valid identifier`);
|
||||
return v;
|
||||
}
|
||||
|
||||
function asInteger(raw: unknown, what: string, min: number, max: number): number {
|
||||
if (typeof raw !== 'number' || !Number.isFinite(raw)) invalid(`${what} must be a finite number`);
|
||||
if (!Number.isInteger(raw)) invalid(`${what} must be an integer`);
|
||||
if (raw < min || raw > max) invalid(`${what} must be between ${min} and ${max}`);
|
||||
return raw;
|
||||
}
|
||||
|
||||
function parseVendorRef(raw: unknown, what: string): VendorRef {
|
||||
const obj = asObject(raw, what);
|
||||
rejectUnknownKeys(obj, ['system', 'id'], what);
|
||||
return { system: asString(obj.system, `${what}.system`), id: asString(obj.id, `${what}.id`, 256) };
|
||||
}
|
||||
|
||||
// ── parsing ────────────────────────────────────────────────────────────────
|
||||
|
||||
function parseParameter(raw: unknown, i: number): DashboardParameter {
|
||||
const what = `parameters[${i}]`;
|
||||
const obj = asObject(raw, what);
|
||||
rejectUnknownKeys(obj, ['name', 'type', 'required', 'default_value', 'allowed_values'], what);
|
||||
const name = asIdentifier(obj.name, `${what}.name`);
|
||||
const type = asString(obj.type, `${what}.type`);
|
||||
if (!(PARAMETER_TYPES as readonly string[]).includes(type)) invalid(`${what}.type is not a parameter type`);
|
||||
if (typeof obj.required !== 'boolean') invalid(`${what}.required must be a boolean`);
|
||||
const p: DashboardParameter = { name, type: type as ParameterType, required: obj.required };
|
||||
if (obj.default_value !== undefined)
|
||||
p.default_value = asString(obj.default_value, `${what}.default_value`, 4096);
|
||||
if (obj.allowed_values !== undefined) {
|
||||
if (!Array.isArray(obj.allowed_values) || obj.allowed_values.length === 0)
|
||||
invalid(`${what}.allowed_values must be a non-empty array`);
|
||||
p.allowed_values = obj.allowed_values.map((v, j) => asString(v, `${what}.allowed_values[${j}]`, 4096));
|
||||
}
|
||||
if (p.type === 'enum' && !p.allowed_values) {
|
||||
invalid(`${what} is an enum parameter with no allowed_values — an unbounded enum is a free-text field`);
|
||||
}
|
||||
if (p.required && p.default_value !== undefined) {
|
||||
invalid(`${what} is required AND has a default — a default makes "required" unenforceable`);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
function parseFilter(raw: unknown, i: number, parameterNames: Set<string>): DashboardFilter {
|
||||
const what = `filters[${i}]`;
|
||||
const obj = asObject(raw, what);
|
||||
rejectUnknownKeys(obj, ['filter_id', 'label', 'dimension', 'operator', 'default_values', 'locked'], what);
|
||||
const filter_id = asIdentifier(obj.filter_id, `${what}.filter_id`);
|
||||
const label = asString(obj.label, `${what}.label`, MAX_TITLE_LENGTH);
|
||||
const dimension = asString(obj.dimension, `${what}.dimension`);
|
||||
const operator = asString(obj.operator, `${what}.operator`) as FilterOperator;
|
||||
const f: DashboardFilter = { filter_id, label, dimension, operator };
|
||||
if (obj.default_values !== undefined) {
|
||||
if (!Array.isArray(obj.default_values)) invalid(`${what}.default_values must be an array`);
|
||||
f.default_values = obj.default_values.map((v, j) => {
|
||||
const s = asString(v, `${what}.default_values[${j}]`, 4096);
|
||||
const m = PARAM_REFERENCE.exec(s);
|
||||
if (m && !parameterNames.has(m[1])) {
|
||||
invalid(`${what}.default_values[${j}] references undeclared parameter "${m[1]}"`);
|
||||
}
|
||||
return s;
|
||||
});
|
||||
}
|
||||
if (obj.locked !== undefined) {
|
||||
if (typeof obj.locked !== 'boolean') invalid(`${what}.locked must be a boolean`);
|
||||
f.locked = obj.locked;
|
||||
}
|
||||
return f;
|
||||
}
|
||||
|
||||
function parseVisualization(raw: unknown, what: string): VisualizationSpec {
|
||||
const obj = asObject(raw, what);
|
||||
rejectUnknownKeys(obj, ['type', 'encoding', 'options'], what);
|
||||
const type = asString(obj.type, `${what}.type`);
|
||||
if (!(VISUALIZATION_TYPES as readonly string[]).includes(type)) {
|
||||
invalid(
|
||||
`${what}.type "${type}" is not a SIAX visualization type — the canonical list is engine-neutral ` +
|
||||
`so a chart survives a BI engine swap (BI-P060)`,
|
||||
);
|
||||
}
|
||||
const encodingRaw = asObject(obj.encoding ?? {}, `${what}.encoding`);
|
||||
rejectUnknownKeys(encodingRaw, ['x', 'y', 'series', 'value'], `${what}.encoding`);
|
||||
const encoding: VisualizationSpec['encoding'] = {};
|
||||
if (encodingRaw.x !== undefined) encoding.x = asString(encodingRaw.x, `${what}.encoding.x`);
|
||||
if (encodingRaw.series !== undefined)
|
||||
encoding.series = asString(encodingRaw.series, `${what}.encoding.series`);
|
||||
if (encodingRaw.value !== undefined) encoding.value = asString(encodingRaw.value, `${what}.encoding.value`);
|
||||
if (encodingRaw.y !== undefined) {
|
||||
if (!Array.isArray(encodingRaw.y)) invalid(`${what}.encoding.y must be an array`);
|
||||
encoding.y = encodingRaw.y.map((v, j) => asString(v, `${what}.encoding.y[${j}]`));
|
||||
}
|
||||
const spec: VisualizationSpec = { type: type as VisualizationType, encoding };
|
||||
if (obj.options !== undefined) {
|
||||
const options = asObject(obj.options, `${what}.options`);
|
||||
const out: Record<string, string | number | boolean> = {};
|
||||
for (const [k, v] of Object.entries(options)) {
|
||||
if (!IDENTIFIER.test(k)) invalid(`${what}.options key "${k}" is not an identifier`);
|
||||
if (typeof v === 'string') {
|
||||
if (v.length > 256) invalid(`${what}.options.${k} exceeds 256 characters`);
|
||||
out[k] = v;
|
||||
} else if (typeof v === 'number') {
|
||||
if (!Number.isFinite(v)) invalid(`${what}.options.${k} must be finite`);
|
||||
out[k] = v;
|
||||
} else if (typeof v === 'boolean') {
|
||||
out[k] = v;
|
||||
} else {
|
||||
invalid(`${what}.options.${k} must be a string, number or boolean — options are not a payload`);
|
||||
}
|
||||
}
|
||||
spec.options = out;
|
||||
}
|
||||
return spec;
|
||||
}
|
||||
|
||||
function parseSource(raw: unknown, what: string): WidgetSource {
|
||||
const obj = asObject(raw, what);
|
||||
const kind = asString(obj.kind, `${what}.kind`);
|
||||
if (kind === 'metric') {
|
||||
rejectUnknownKeys(obj, ['kind', 'dataset', 'metric', 'dimensions', 'grain', 'limit'], what);
|
||||
const source: MetricWidgetSource = {
|
||||
kind: 'metric',
|
||||
dataset: asString(obj.dataset, `${what}.dataset`),
|
||||
metric: asString(obj.metric, `${what}.metric`),
|
||||
};
|
||||
if (obj.dimensions !== undefined) {
|
||||
if (!Array.isArray(obj.dimensions)) invalid(`${what}.dimensions must be an array`);
|
||||
source.dimensions = obj.dimensions.map((d, j) => asString(d, `${what}.dimensions[${j}]`));
|
||||
}
|
||||
if (obj.grain !== undefined) {
|
||||
const grain = asString(obj.grain, `${what}.grain`);
|
||||
if (!(TIME_GRAINS as readonly string[]).includes(grain)) invalid(`${what}.grain is not a time grain`);
|
||||
source.grain = grain as TimeGrain;
|
||||
}
|
||||
if (obj.limit !== undefined) source.limit = asInteger(obj.limit, `${what}.limit`, 1, 100_000);
|
||||
return source;
|
||||
}
|
||||
if (kind === 'query') {
|
||||
rejectUnknownKeys(obj, ['kind', 'request'], what);
|
||||
// parseAnalyticsQueryRequest is the SQL boundary: it is why a query widget
|
||||
// cannot become an SQL widget by adding a field.
|
||||
return { kind: 'query', request: parseAnalyticsQueryRequest(obj.request) };
|
||||
}
|
||||
invalid(
|
||||
`${what}.kind must be "metric" or "query" — a widget with no reconstructable source cannot be ` +
|
||||
`rendered without the tool that exported it (BI-P060 DoD)`,
|
||||
);
|
||||
}
|
||||
|
||||
function parseWidget(raw: unknown, i: number, gridColumns: number): DashboardWidget {
|
||||
const what = `widgets[${i}]`;
|
||||
const obj = asObject(raw, what);
|
||||
rejectUnknownKeys(
|
||||
obj,
|
||||
['widget_id', 'title', 'source', 'visualization', 'layout', 'filter_bindings', 'vendor_ref'],
|
||||
what,
|
||||
);
|
||||
const widget_id = asIdentifier(obj.widget_id, `${what}.widget_id`);
|
||||
const title = asString(obj.title, `${what}.title`, MAX_TITLE_LENGTH);
|
||||
const source = parseSource(obj.source, `${what}.source`);
|
||||
const visualization = parseVisualization(obj.visualization, `${what}.visualization`);
|
||||
|
||||
const layoutObj = asObject(obj.layout, `${what}.layout`);
|
||||
rejectUnknownKeys(layoutObj, ['x', 'y', 'w', 'h'], `${what}.layout`);
|
||||
const layout: WidgetLayout = {
|
||||
x: asInteger(layoutObj.x, `${what}.layout.x`, 0, gridColumns - 1),
|
||||
y: asInteger(layoutObj.y, `${what}.layout.y`, 0, 10_000),
|
||||
w: asInteger(layoutObj.w, `${what}.layout.w`, 1, gridColumns),
|
||||
h: asInteger(layoutObj.h, `${what}.layout.h`, 1, 200),
|
||||
};
|
||||
if (layout.x + layout.w > gridColumns) {
|
||||
invalid(`${what}.layout overflows the ${gridColumns}-column grid (x=${layout.x}, w=${layout.w})`);
|
||||
}
|
||||
|
||||
const widget: DashboardWidget = { widget_id, title, source, visualization, layout };
|
||||
if (obj.filter_bindings !== undefined) {
|
||||
if (!Array.isArray(obj.filter_bindings)) invalid(`${what}.filter_bindings must be an array`);
|
||||
widget.filter_bindings = obj.filter_bindings.map((b, j) =>
|
||||
asIdentifier(b, `${what}.filter_bindings[${j}]`),
|
||||
);
|
||||
}
|
||||
if (obj.vendor_ref !== undefined) widget.vendor_ref = parseVendorRef(obj.vendor_ref, `${what}.vendor_ref`);
|
||||
return widget;
|
||||
}
|
||||
|
||||
/**
|
||||
* Parse an untrusted dashboard document. Fail-closed everywhere: a field this
|
||||
* function does not understand is a refusal, because "ignored for now" is how
|
||||
* a Superset-only document ends up stored as canonical truth.
|
||||
*/
|
||||
export function parseDashboardDocument(raw: unknown): DashboardDocument {
|
||||
const obj = asObject(raw, 'dashboard document');
|
||||
rejectUnknownKeys(obj, DASHBOARD_DOCUMENT_KEYS, 'dashboard document');
|
||||
|
||||
const id = asIdentifier(obj.id, 'id');
|
||||
const title = asString(obj.title, 'title', MAX_TITLE_LENGTH);
|
||||
const version = asInteger(obj.version, 'version', 1, 1_000_000);
|
||||
|
||||
const layoutObj = asObject(obj.layout, 'layout');
|
||||
rejectUnknownKeys(layoutObj, ['grid_columns', 'row_height_px'], 'layout');
|
||||
const layout = {
|
||||
grid_columns: asInteger(layoutObj.grid_columns, 'layout.grid_columns', 1, MAX_GRID_COLUMNS),
|
||||
row_height_px: asInteger(layoutObj.row_height_px, 'layout.row_height_px', 8, 400),
|
||||
};
|
||||
|
||||
const rawParameters = obj.parameters ?? [];
|
||||
if (!Array.isArray(rawParameters)) invalid('parameters must be an array');
|
||||
if (rawParameters.length > MAX_PARAMETERS) invalid(`at most ${MAX_PARAMETERS} parameters`);
|
||||
const parameters = rawParameters.map((p, i) => parseParameter(p, i));
|
||||
const parameterNames = new Set(parameters.map((p) => p.name));
|
||||
if (parameterNames.size !== parameters.length) invalid('parameters contains duplicate names');
|
||||
|
||||
const rawFilters = obj.filters ?? [];
|
||||
if (!Array.isArray(rawFilters)) invalid('filters must be an array');
|
||||
if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters`);
|
||||
const filters = rawFilters.map((f, i) => parseFilter(f, i, parameterNames));
|
||||
const filterIds = new Set(filters.map((f) => f.filter_id));
|
||||
if (filterIds.size !== filters.length) invalid('filters contains duplicate filter_id');
|
||||
|
||||
if (!Array.isArray(obj.widgets)) invalid('widgets must be an array');
|
||||
if (obj.widgets.length === 0) invalid('a dashboard with no widgets has nothing to reconstruct');
|
||||
if (obj.widgets.length > MAX_WIDGETS) invalid(`at most ${MAX_WIDGETS} widgets`);
|
||||
const widgets = obj.widgets.map((w, i) => parseWidget(w, i, layout.grid_columns));
|
||||
if (new Set(widgets.map((w) => w.widget_id)).size !== widgets.length) {
|
||||
invalid('widgets contains duplicate widget_id');
|
||||
}
|
||||
for (const w of widgets) {
|
||||
for (const b of w.filter_bindings ?? []) {
|
||||
if (!filterIds.has(b)) invalid(`widget "${w.widget_id}" binds undeclared filter "${b}"`);
|
||||
}
|
||||
}
|
||||
|
||||
const refreshObj = asObject(obj.refresh, 'refresh');
|
||||
rejectUnknownKeys(refreshObj, ['mode', 'interval_seconds', 'cron', 'max_staleness_seconds'], 'refresh');
|
||||
const mode = asString(refreshObj.mode, 'refresh.mode');
|
||||
if (!(REFRESH_MODES as readonly string[]).includes(mode)) invalid('refresh.mode is not a refresh mode');
|
||||
const refresh: DashboardRefresh = {
|
||||
mode: mode as RefreshMode,
|
||||
max_staleness_seconds: asInteger(
|
||||
refreshObj.max_staleness_seconds,
|
||||
'refresh.max_staleness_seconds',
|
||||
1,
|
||||
MAX_STALENESS_SECONDS,
|
||||
),
|
||||
};
|
||||
if (refresh.mode === 'interval') {
|
||||
refresh.interval_seconds = asInteger(
|
||||
refreshObj.interval_seconds,
|
||||
'refresh.interval_seconds',
|
||||
MIN_REFRESH_INTERVAL_SECONDS,
|
||||
86_400,
|
||||
);
|
||||
if (refreshObj.cron !== undefined) invalid('refresh.cron is meaningless when mode is "interval"');
|
||||
} else if (refresh.mode === 'scheduled') {
|
||||
refresh.cron = asString(refreshObj.cron, 'refresh.cron', 128);
|
||||
if (refreshObj.interval_seconds !== undefined) {
|
||||
invalid('refresh.interval_seconds is meaningless when mode is "scheduled"');
|
||||
}
|
||||
} else if (refreshObj.interval_seconds !== undefined || refreshObj.cron !== undefined) {
|
||||
invalid('refresh.mode is "manual" but a schedule was supplied');
|
||||
}
|
||||
|
||||
const permObj = asObject(obj.permissions, 'permissions');
|
||||
rejectUnknownKeys(permObj, ['visibility', 'roles', 'required_entitlement'], 'permissions');
|
||||
const visibility = asString(permObj.visibility, 'permissions.visibility');
|
||||
if (!(DASHBOARD_VISIBILITIES as readonly string[]).includes(visibility)) {
|
||||
invalid('permissions.visibility is not a visibility');
|
||||
}
|
||||
const permissions: DashboardPermissions = { visibility: visibility as DashboardVisibility };
|
||||
if (permObj.roles !== undefined) {
|
||||
if (!Array.isArray(permObj.roles)) invalid('permissions.roles must be an array');
|
||||
permissions.roles = permObj.roles.map((r, i) => asIdentifier(r, `permissions.roles[${i}]`));
|
||||
}
|
||||
if (permissions.visibility === 'roles' && (permissions.roles ?? []).length === 0) {
|
||||
invalid(
|
||||
'permissions.visibility is "roles" with no roles — an empty role list reads as "deny all" to one ' +
|
||||
'implementation and "allow all" to the next; say which',
|
||||
);
|
||||
}
|
||||
if (permObj.required_entitlement !== undefined) {
|
||||
permissions.required_entitlement = asIdentifier(
|
||||
permObj.required_entitlement,
|
||||
'permissions.required_entitlement',
|
||||
);
|
||||
}
|
||||
|
||||
const doc: DashboardDocument = {
|
||||
id,
|
||||
title,
|
||||
layout,
|
||||
filters,
|
||||
widgets,
|
||||
parameters,
|
||||
refresh,
|
||||
permissions,
|
||||
version,
|
||||
};
|
||||
if (obj.description !== undefined) doc.description = asString(obj.description, 'description', 2000);
|
||||
if (obj.vendor_ref !== undefined) doc.vendor_ref = parseVendorRef(obj.vendor_ref, 'vendor_ref');
|
||||
return doc;
|
||||
}
|
||||
|
||||
// ── canonical hash ─────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Stable ordering of every field that MEANS something, with `vendor_ref`
|
||||
* omitted at both levels. Two documents that render identically hash
|
||||
* identically even if their JSON key order differs or one of them has been
|
||||
* round-tripped through a BI tool; a document whose meaning changed hashes
|
||||
* differently. That is what makes `DashboardVersion.layout_hash` a usable
|
||||
* identity for BI-P061 rollback.
|
||||
*/
|
||||
function canonicalize(doc: DashboardDocument): unknown {
|
||||
return [
|
||||
doc.id,
|
||||
doc.title,
|
||||
doc.description ?? '',
|
||||
doc.version,
|
||||
[doc.layout.grid_columns, doc.layout.row_height_px],
|
||||
doc.parameters
|
||||
.map((p) => [p.name, p.type, p.required, p.default_value ?? '', [...(p.allowed_values ?? [])].sort()])
|
||||
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
|
||||
doc.filters
|
||||
.map((f) => [f.filter_id, f.label, f.dimension, f.operator, f.default_values ?? [], f.locked ?? false])
|
||||
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
|
||||
doc.widgets
|
||||
.map((w) => [
|
||||
w.widget_id,
|
||||
w.title,
|
||||
w.source.kind === 'metric'
|
||||
? [
|
||||
'metric',
|
||||
w.source.dataset,
|
||||
w.source.metric,
|
||||
w.source.dimensions ?? [],
|
||||
w.source.grain ?? '',
|
||||
w.source.limit ?? 0,
|
||||
]
|
||||
: ['query', canonicalRequest(w.source.request)],
|
||||
[
|
||||
w.visualization.type,
|
||||
w.visualization.encoding.x ?? '',
|
||||
w.visualization.encoding.y ?? [],
|
||||
w.visualization.encoding.series ?? '',
|
||||
w.visualization.encoding.value ?? '',
|
||||
Object.keys(w.visualization.options ?? {})
|
||||
.sort()
|
||||
.map((k) => [k, (w.visualization.options ?? {})[k]]),
|
||||
],
|
||||
[w.layout.x, w.layout.y, w.layout.w, w.layout.h],
|
||||
[...(w.filter_bindings ?? [])].sort(),
|
||||
])
|
||||
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
|
||||
[
|
||||
doc.refresh.mode,
|
||||
doc.refresh.interval_seconds ?? 0,
|
||||
doc.refresh.cron ?? '',
|
||||
doc.refresh.max_staleness_seconds,
|
||||
],
|
||||
[
|
||||
doc.permissions.visibility,
|
||||
[...(doc.permissions.roles ?? [])].sort(),
|
||||
doc.permissions.required_entitlement ?? '',
|
||||
],
|
||||
];
|
||||
}
|
||||
|
||||
function canonicalRequest(r: AnalyticsQueryRequest): unknown {
|
||||
return [
|
||||
r.dataset,
|
||||
[...r.metrics],
|
||||
[...r.dimensions],
|
||||
r.filters.map((f) => [f.dimension, f.operator, f.values ?? []]),
|
||||
r.timeRange
|
||||
? [r.timeRange.dimension ?? '', r.timeRange.from ?? '', r.timeRange.to ?? '', r.timeRange.grain ?? '']
|
||||
: [],
|
||||
(r.orderBy ?? []).map((o) => [o.ref, o.direction]),
|
||||
r.limit ?? 0,
|
||||
r.format ?? 'json',
|
||||
];
|
||||
}
|
||||
|
||||
/** sha256 of the canonical form. Fills `DashboardVersion.layout_hash`. */
|
||||
export function dashboardDocumentHash(doc: DashboardDocument): string {
|
||||
return crypto
|
||||
.createHash('sha256')
|
||||
.update(JSON.stringify(canonicalize(doc)))
|
||||
.digest('hex');
|
||||
}
|
||||
|
||||
// ── reconstruction (the DoD) ───────────────────────────────────────────────
|
||||
|
||||
export interface PlannedWidget {
|
||||
widget_id: string;
|
||||
title: string;
|
||||
/** Runnable, contract-valid, engine-neutral. This is the DoD in one field. */
|
||||
request: AnalyticsQueryRequest;
|
||||
visualization: VisualizationSpec;
|
||||
layout: WidgetLayout;
|
||||
}
|
||||
|
||||
export interface DashboardPlan {
|
||||
dashboard_id: string;
|
||||
title: string;
|
||||
document_hash: string;
|
||||
grid_columns: number;
|
||||
widgets: PlannedWidget[];
|
||||
/** Parameter values actually bound, after defaults. Reported, not implied. */
|
||||
bound_parameters: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface PlanOptions {
|
||||
/** Caller-supplied parameter values. Validated against the declarations. */
|
||||
parameters?: Record<string, string>;
|
||||
/** Filter ids a viewer switched off. `locked` filters cannot be switched off. */
|
||||
disabled_filter_ids?: string[];
|
||||
}
|
||||
|
||||
function bindParameters(doc: DashboardDocument, supplied: Record<string, string>): Record<string, string> {
|
||||
const bound: Record<string, string> = {};
|
||||
const declared = new Set(doc.parameters.map((p) => p.name));
|
||||
for (const name of Object.keys(supplied)) {
|
||||
if (!declared.has(name)) invalid(`parameter "${name}" is not declared by this dashboard`);
|
||||
}
|
||||
for (const p of doc.parameters) {
|
||||
const value = supplied[p.name] ?? p.default_value;
|
||||
if (value === undefined) {
|
||||
if (p.required) invalid(`required parameter "${p.name}" was not supplied`);
|
||||
continue;
|
||||
}
|
||||
if (p.allowed_values && !p.allowed_values.includes(value)) {
|
||||
invalid(`parameter "${p.name}" value is not in its allowed_values`);
|
||||
}
|
||||
if (p.type === 'number' && !Number.isFinite(Number(value))) {
|
||||
invalid(`parameter "${p.name}" is declared number but "${value}" is not numeric`);
|
||||
}
|
||||
if (p.type === 'date' && Number.isNaN(Date.parse(value))) {
|
||||
invalid(`parameter "${p.name}" is declared date but "${value}" is not an ISO date`);
|
||||
}
|
||||
bound[p.name] = value;
|
||||
}
|
||||
return bound;
|
||||
}
|
||||
|
||||
function resolveFilterValues(f: DashboardFilter, bound: Record<string, string>): string[] | undefined {
|
||||
if (!f.default_values) return undefined;
|
||||
return f.default_values.map((v) => {
|
||||
const m = PARAM_REFERENCE.exec(v);
|
||||
if (!m) return v;
|
||||
const value = bound[m[1]];
|
||||
if (value === undefined) {
|
||||
invalid(`filter "${f.filter_id}" references parameter "${m[1]}", which has no value`);
|
||||
}
|
||||
return value;
|
||||
});
|
||||
}
|
||||
|
||||
/**
|
||||
* Turn a dashboard document into runnable analytics requests.
|
||||
*
|
||||
* This IS BI-P060's DoD. If this function returns a plan, the dashboard can be
|
||||
* rendered by anything that can execute `AnalyticsQueryRequest` and draw the
|
||||
* twelve visualization types — Superset not required, not consulted, and not
|
||||
* present anywhere in the output.
|
||||
*/
|
||||
export function planDashboard(doc: DashboardDocument, options: PlanOptions = {}): DashboardPlan {
|
||||
const bound = bindParameters(doc, options.parameters ?? {});
|
||||
const disabled = new Set(options.disabled_filter_ids ?? []);
|
||||
for (const f of doc.filters) {
|
||||
if (disabled.has(f.filter_id) && f.locked) {
|
||||
invalid(
|
||||
`filter "${f.filter_id}" is locked and cannot be disabled — locked filters are how a saved ` +
|
||||
`dashboard pins its scope, and a viewer removing one silently widens the result set`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const widgets = doc.widgets.map((w) => {
|
||||
const applicable = doc.filters.filter(
|
||||
(f) =>
|
||||
!disabled.has(f.filter_id) &&
|
||||
(w.filter_bindings === undefined || w.filter_bindings.length === 0
|
||||
? true
|
||||
: w.filter_bindings.includes(f.filter_id)),
|
||||
);
|
||||
const dashboardFilters: AnalyticsFilter[] = applicable.map((f) => {
|
||||
const values = resolveFilterValues(f, bound);
|
||||
const af: AnalyticsFilter = { dimension: f.dimension, operator: f.operator };
|
||||
if (values !== undefined) af.values = values;
|
||||
return af;
|
||||
});
|
||||
|
||||
const base =
|
||||
w.source.kind === 'metric'
|
||||
? {
|
||||
dataset: w.source.dataset,
|
||||
metrics: [w.source.metric],
|
||||
dimensions: w.source.dimensions ?? [],
|
||||
filters: [] as AnalyticsFilter[],
|
||||
...(w.source.grain ? { timeRange: { grain: w.source.grain } } : {}),
|
||||
...(w.source.limit ? { limit: w.source.limit } : {}),
|
||||
}
|
||||
: {
|
||||
...w.source.request,
|
||||
timeRange: w.source.request.timeRange,
|
||||
};
|
||||
|
||||
// Re-parse the merged request rather than trusting the merge: the filters
|
||||
// we just added come from the document, and a document is untrusted input
|
||||
// even after it has been stored.
|
||||
const request = parseAnalyticsQueryRequest({
|
||||
...base,
|
||||
filters: [...(w.source.kind === 'query' ? w.source.request.filters : []), ...dashboardFilters],
|
||||
});
|
||||
|
||||
return {
|
||||
widget_id: w.widget_id,
|
||||
title: w.title,
|
||||
request,
|
||||
visualization: w.visualization,
|
||||
layout: w.layout,
|
||||
};
|
||||
});
|
||||
|
||||
return {
|
||||
dashboard_id: doc.id,
|
||||
title: doc.title,
|
||||
document_hash: dashboardDocumentHash(doc),
|
||||
grid_columns: doc.layout.grid_columns,
|
||||
widgets,
|
||||
bound_parameters: bound,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,449 @@
|
||||
/**
|
||||
* src/analytics/dashboards/dashboard-release.ts
|
||||
* ==============================================
|
||||
* BI-P061 — DASHBOARD VERSIONING.
|
||||
*
|
||||
* DRAFT -> PUBLISHED -> SUPERSEDED, and the DoD: "rollback to previous
|
||||
* dashboard definition".
|
||||
*
|
||||
* Three decisions worth stating, because each one is a bug someone would
|
||||
* otherwise write:
|
||||
*
|
||||
* 1. ROLLBACK MINTS A NEW VERSION. It does not resurrect the old release row
|
||||
* and it does not edit the published one. "We went back to v3" is then a
|
||||
* fact with a timestamp (v5 = rollback_of_version 3) instead of a hole in
|
||||
* the history where v4 used to be. `verifyRollback()` proves the restore is
|
||||
* faithful by comparing BI-P060 canonical hashes, so a rollback that
|
||||
* silently restored a different document fails loudly.
|
||||
*
|
||||
* 2. THE DOCUMENT IS FROZEN, THE RELEASE MOVES. sql/008 makes
|
||||
* DashboardVersion rows immutable at the database. The release ledger
|
||||
* (sql/010) carries the state, and its trigger allows exactly two
|
||||
* transitions and freezes every other column. Un-publishing is refused:
|
||||
* it is how a rollback gets faked without leaving a trace.
|
||||
*
|
||||
* 3. THE APPLICATION CHECK IS NOT THE GUARANTEE. `publish()` checks for an
|
||||
* existing published release, and that check is a read-then-write race.
|
||||
* The partial unique index in sql/010 is what actually holds under
|
||||
* concurrency; the realinfra test drops it, runs the race and shows two
|
||||
* published releases for one dashboard, so nobody has to take that on
|
||||
* faith. This module maps the resulting constraint violation onto a
|
||||
* CONFLICT rather than a 500.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import { dashboardDocumentHash, parseDashboardDocument, type DashboardDocument } from './dashboard-document';
|
||||
|
||||
export const DASHBOARD_RELEASE_STATES = ['draft', 'published', 'superseded'] as const;
|
||||
export type DashboardReleaseState = (typeof DASHBOARD_RELEASE_STATES)[number];
|
||||
|
||||
/** The only transitions that exist. Everything else is a refusal, not a no-op. */
|
||||
const LEGAL_TRANSITIONS: Readonly<Record<DashboardReleaseState, readonly DashboardReleaseState[]>> = {
|
||||
draft: ['published'],
|
||||
published: ['superseded'],
|
||||
superseded: [],
|
||||
};
|
||||
|
||||
export function isLegalTransition(from: DashboardReleaseState, to: DashboardReleaseState): boolean {
|
||||
return LEGAL_TRANSITIONS[from].includes(to);
|
||||
}
|
||||
|
||||
export interface DashboardRelease {
|
||||
release_id: string;
|
||||
dashboard_id: string;
|
||||
dashboard_version_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
version: number;
|
||||
state: DashboardReleaseState;
|
||||
/** BI-P060 canonical hash of the released document. */
|
||||
document_hash: string;
|
||||
/** Set when this release restores an earlier version. */
|
||||
rollback_of_version?: number;
|
||||
created_by?: string;
|
||||
created_at: string;
|
||||
published_at?: string;
|
||||
superseded_at?: string;
|
||||
superseded_by_release_id?: string;
|
||||
}
|
||||
|
||||
/** The immutable frozen document behind a release (sql/008's DashboardVersion). */
|
||||
export interface DashboardVersionRecord {
|
||||
dashboard_version_id: string;
|
||||
dashboard_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
version: number;
|
||||
layout_json: string;
|
||||
layout_hash: string;
|
||||
created_at: string;
|
||||
created_by?: string;
|
||||
}
|
||||
|
||||
export interface TenantScope {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The persistence port. Deliberately narrow, and every method is tenant-scoped
|
||||
* — there is no signature that lets a caller forget the tenant.
|
||||
*/
|
||||
export interface DashboardReleaseStore {
|
||||
/** Highest version number ever minted for this dashboard, or 0. */
|
||||
maxVersion(scope: TenantScope, dashboardId: string): Promise<number>;
|
||||
insertVersionAndRelease(version: DashboardVersionRecord, release: DashboardRelease): Promise<void>;
|
||||
getRelease(scope: TenantScope, releaseId: string): Promise<DashboardRelease | undefined>;
|
||||
getVersion(scope: TenantScope, dashboardVersionId: string): Promise<DashboardVersionRecord | undefined>;
|
||||
findVersionByNumber(
|
||||
scope: TenantScope,
|
||||
dashboardId: string,
|
||||
version: number,
|
||||
): Promise<DashboardVersionRecord | undefined>;
|
||||
findByState(
|
||||
scope: TenantScope,
|
||||
dashboardId: string,
|
||||
state: DashboardReleaseState,
|
||||
): Promise<DashboardRelease | undefined>;
|
||||
listReleases(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]>;
|
||||
/**
|
||||
* Publish `releaseId` and supersede `supersedes` in ONE transaction. Both or
|
||||
* neither: a supersede that lands without its publish leaves a dashboard
|
||||
* with no live version, and a publish that lands without its supersede
|
||||
* leaves two.
|
||||
*/
|
||||
publishAtomically(args: {
|
||||
scope: TenantScope;
|
||||
releaseId: string;
|
||||
publishedAt: string;
|
||||
supersedes?: { release_id: string; superseded_at: string };
|
||||
}): Promise<void>;
|
||||
}
|
||||
|
||||
function conflict(detail: string): never {
|
||||
throw new SiaxError('CONFLICT', '0pt.analytics.dashboard', detail);
|
||||
}
|
||||
|
||||
function notFound(detail: string): never {
|
||||
throw new SiaxError('NOT_FOUND', '0pt.analytics.dashboard', detail);
|
||||
}
|
||||
|
||||
function assertScope(scope: TenantScope): void {
|
||||
if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.dashboard',
|
||||
'dashboard releases require a verified tenant_id + project_id',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
export interface DraftResult {
|
||||
release: DashboardRelease;
|
||||
version: DashboardVersionRecord;
|
||||
}
|
||||
|
||||
/**
|
||||
* The lifecycle. Stateless apart from its store and its clock; the clock is
|
||||
* injectable so a test can pin timestamps instead of sleeping.
|
||||
*/
|
||||
export class DashboardReleaseService {
|
||||
constructor(
|
||||
private readonly store: DashboardReleaseStore,
|
||||
private readonly now: () => Date = () => new Date(),
|
||||
private readonly newId: () => string = () => crypto.randomUUID(),
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Freeze a document as the next version and open a DRAFT release for it.
|
||||
* The document is re-parsed here even if the caller already parsed it: what
|
||||
* gets frozen must be what the model accepts, not what a caller assembled.
|
||||
*/
|
||||
async draft(args: {
|
||||
scope: TenantScope;
|
||||
dashboardId: string;
|
||||
document: unknown;
|
||||
createdBy?: string;
|
||||
rollbackOfVersion?: number;
|
||||
}): Promise<DraftResult> {
|
||||
assertScope(args.scope);
|
||||
const document = parseDashboardDocument(args.document);
|
||||
const existingDraft = await this.store.findByState(args.scope, args.dashboardId, 'draft');
|
||||
if (existingDraft) {
|
||||
conflict(
|
||||
`dashboard "${args.dashboardId}" already has an open draft (version ${existingDraft.version}); ` +
|
||||
`publish or abandon it before opening another — two drafts race to publish and the loser's ` +
|
||||
`edits vanish`,
|
||||
);
|
||||
}
|
||||
const next = (await this.store.maxVersion(args.scope, args.dashboardId)) + 1;
|
||||
const at = this.now().toISOString();
|
||||
const hash = dashboardDocumentHash(document);
|
||||
|
||||
const version: DashboardVersionRecord = {
|
||||
dashboard_version_id: this.newId(),
|
||||
dashboard_id: args.dashboardId,
|
||||
tenant_id: args.scope.tenant_id,
|
||||
project_id: args.scope.project_id,
|
||||
version: next,
|
||||
layout_json: JSON.stringify(document),
|
||||
layout_hash: hash,
|
||||
created_at: at,
|
||||
...(args.createdBy ? { created_by: args.createdBy } : {}),
|
||||
};
|
||||
const release: DashboardRelease = {
|
||||
release_id: this.newId(),
|
||||
dashboard_id: args.dashboardId,
|
||||
dashboard_version_id: version.dashboard_version_id,
|
||||
tenant_id: args.scope.tenant_id,
|
||||
project_id: args.scope.project_id,
|
||||
version: next,
|
||||
state: 'draft',
|
||||
document_hash: hash,
|
||||
created_at: at,
|
||||
...(args.createdBy ? { created_by: args.createdBy } : {}),
|
||||
...(args.rollbackOfVersion ? { rollback_of_version: args.rollbackOfVersion } : {}),
|
||||
};
|
||||
await this.store.insertVersionAndRelease(version, release);
|
||||
return { release, version };
|
||||
}
|
||||
|
||||
/**
|
||||
* DRAFT -> PUBLISHED, superseding whatever was live, in one transaction.
|
||||
* The pre-check exists to return a friendly CONFLICT; the guarantee is the
|
||||
* partial unique index (sql/010).
|
||||
*/
|
||||
async publish(args: {
|
||||
scope: TenantScope;
|
||||
releaseId: string;
|
||||
}): Promise<{ published: DashboardRelease; superseded?: DashboardRelease }> {
|
||||
assertScope(args.scope);
|
||||
const release = await this.store.getRelease(args.scope, args.releaseId);
|
||||
if (!release) notFound(`dashboard release "${args.releaseId}" does not exist in this tenant`);
|
||||
if (release.state !== 'draft') {
|
||||
conflict(
|
||||
`release "${args.releaseId}" is ${release.state}, not draft — ${release.state} -> published is ` +
|
||||
`not a legal transition (BI-P061)`,
|
||||
);
|
||||
}
|
||||
const live = await this.store.findByState(args.scope, release.dashboard_id, 'published');
|
||||
const at = this.now().toISOString();
|
||||
await this.store.publishAtomically({
|
||||
scope: args.scope,
|
||||
releaseId: release.release_id,
|
||||
publishedAt: at,
|
||||
...(live ? { supersedes: { release_id: live.release_id, superseded_at: at } } : {}),
|
||||
});
|
||||
|
||||
return {
|
||||
published: { ...release, state: 'published', published_at: at },
|
||||
...(live
|
||||
? {
|
||||
superseded: {
|
||||
...live,
|
||||
state: 'superseded',
|
||||
superseded_at: at,
|
||||
superseded_by_release_id: release.release_id,
|
||||
},
|
||||
}
|
||||
: {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** The release currently serving, or undefined if the dashboard has never been published. */
|
||||
async live(scope: TenantScope, dashboardId: string): Promise<DashboardRelease | undefined> {
|
||||
assertScope(scope);
|
||||
return this.store.findByState(scope, dashboardId, 'published');
|
||||
}
|
||||
|
||||
/** The frozen document behind a release, re-parsed on the way out. */
|
||||
async documentOf(scope: TenantScope, release: DashboardRelease): Promise<DashboardDocument> {
|
||||
assertScope(scope);
|
||||
const version = await this.store.getVersion(scope, release.dashboard_version_id);
|
||||
if (!version) notFound(`dashboard version "${release.dashboard_version_id}" is missing`);
|
||||
const document = parseDashboardDocument(JSON.parse(version.layout_json));
|
||||
const hash = dashboardDocumentHash(document);
|
||||
if (hash !== release.document_hash) {
|
||||
// Storage returned something other than what was released. Refusing is
|
||||
// the only safe answer: a dashboard that renders a document nobody
|
||||
// approved is worse than a dashboard that will not open.
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dashboard',
|
||||
`stored document for release "${release.release_id}" hashes to ${hash}, not the released ` +
|
||||
`${release.document_hash} — refusing to serve an unapproved definition`,
|
||||
);
|
||||
}
|
||||
return document;
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P061's DoD. Restores version `toVersion` by minting a NEW version with
|
||||
* the SAME document and publishing it. Returns the new release plus the
|
||||
* verification that the restore is faithful.
|
||||
*/
|
||||
async rollback(args: {
|
||||
scope: TenantScope;
|
||||
dashboardId: string;
|
||||
toVersion: number;
|
||||
createdBy?: string;
|
||||
}): Promise<{ release: DashboardRelease; restored_from: number; document_hash: string }> {
|
||||
assertScope(args.scope);
|
||||
const target = await this.store.findVersionByNumber(args.scope, args.dashboardId, args.toVersion);
|
||||
if (!target) {
|
||||
notFound(`dashboard "${args.dashboardId}" has no version ${args.toVersion} in this tenant`);
|
||||
}
|
||||
const live = await this.store.findByState(args.scope, args.dashboardId, 'published');
|
||||
if (live && live.version === args.toVersion) {
|
||||
conflict(`version ${args.toVersion} is already live; a rollback to the live version is a no-op`);
|
||||
}
|
||||
const document = parseDashboardDocument(JSON.parse(target.layout_json));
|
||||
const hash = dashboardDocumentHash(document);
|
||||
if (hash !== target.layout_hash) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dashboard',
|
||||
`version ${args.toVersion} hashes to ${hash} but was stored as ${target.layout_hash} — ` +
|
||||
`refusing to roll back to a definition that has changed underneath us`,
|
||||
);
|
||||
}
|
||||
|
||||
const { release } = await this.draft({
|
||||
scope: args.scope,
|
||||
dashboardId: args.dashboardId,
|
||||
document,
|
||||
...(args.createdBy ? { createdBy: args.createdBy } : {}),
|
||||
rollbackOfVersion: args.toVersion,
|
||||
});
|
||||
const { published } = await this.publish({ scope: args.scope, releaseId: release.release_id });
|
||||
|
||||
if (published.document_hash !== target.layout_hash) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dashboard',
|
||||
`rollback published ${published.document_hash}, not the requested ${target.layout_hash}`,
|
||||
);
|
||||
}
|
||||
return { release: published, restored_from: args.toVersion, document_hash: published.document_hash };
|
||||
}
|
||||
|
||||
/** Full ledger, oldest first. A rollback is an entry here, never a deletion. */
|
||||
async history(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]> {
|
||||
assertScope(scope);
|
||||
const rows = await this.store.listReleases(scope, dashboardId);
|
||||
return [...rows].sort((a, b) => a.version - b.version);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* In-memory store. Used by the unit tests, and it enforces the same two
|
||||
* uniqueness rules the SQL does — a store that is laxer than the database
|
||||
* makes the unit tests pass on states production can never reach.
|
||||
*/
|
||||
export class InMemoryDashboardReleaseStore implements DashboardReleaseStore {
|
||||
private readonly versions: DashboardVersionRecord[] = [];
|
||||
private readonly releases: DashboardRelease[] = [];
|
||||
|
||||
private scoped<T extends { tenant_id: string; project_id: string }>(scope: TenantScope, rows: T[]): T[] {
|
||||
return rows.filter((r) => r.tenant_id === scope.tenant_id && r.project_id === scope.project_id);
|
||||
}
|
||||
|
||||
async maxVersion(scope: TenantScope, dashboardId: string): Promise<number> {
|
||||
return this.scoped(scope, this.versions)
|
||||
.filter((v) => v.dashboard_id === dashboardId)
|
||||
.reduce((max, v) => Math.max(max, v.version), 0);
|
||||
}
|
||||
|
||||
async insertVersionAndRelease(version: DashboardVersionRecord, release: DashboardRelease): Promise<void> {
|
||||
const scope = { tenant_id: release.tenant_id, project_id: release.project_id };
|
||||
if (
|
||||
release.state === 'draft' &&
|
||||
this.scoped(scope, this.releases).some(
|
||||
(r) => r.dashboard_id === release.dashboard_id && r.state === 'draft',
|
||||
)
|
||||
) {
|
||||
conflict('one open draft per dashboard');
|
||||
}
|
||||
if (this.versions.some((v) => v.dashboard_id === version.dashboard_id && v.version === version.version)) {
|
||||
conflict(`version ${version.version} already exists for "${version.dashboard_id}"`);
|
||||
}
|
||||
this.versions.push({ ...version });
|
||||
this.releases.push({ ...release });
|
||||
}
|
||||
|
||||
async getRelease(scope: TenantScope, releaseId: string): Promise<DashboardRelease | undefined> {
|
||||
const r = this.scoped(scope, this.releases).find((x) => x.release_id === releaseId);
|
||||
return r ? { ...r } : undefined;
|
||||
}
|
||||
|
||||
async getVersion(scope: TenantScope, id: string): Promise<DashboardVersionRecord | undefined> {
|
||||
const v = this.scoped(scope, this.versions).find((x) => x.dashboard_version_id === id);
|
||||
return v ? { ...v } : undefined;
|
||||
}
|
||||
|
||||
async findVersionByNumber(
|
||||
scope: TenantScope,
|
||||
dashboardId: string,
|
||||
version: number,
|
||||
): Promise<DashboardVersionRecord | undefined> {
|
||||
const v = this.scoped(scope, this.versions).find(
|
||||
(x) => x.dashboard_id === dashboardId && x.version === version,
|
||||
);
|
||||
return v ? { ...v } : undefined;
|
||||
}
|
||||
|
||||
async findByState(
|
||||
scope: TenantScope,
|
||||
dashboardId: string,
|
||||
state: DashboardReleaseState,
|
||||
): Promise<DashboardRelease | undefined> {
|
||||
const r = this.scoped(scope, this.releases).find(
|
||||
(x) => x.dashboard_id === dashboardId && x.state === state,
|
||||
);
|
||||
return r ? { ...r } : undefined;
|
||||
}
|
||||
|
||||
async listReleases(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]> {
|
||||
return this.scoped(scope, this.releases)
|
||||
.filter((r) => r.dashboard_id === dashboardId)
|
||||
.map((r) => ({ ...r }));
|
||||
}
|
||||
|
||||
async publishAtomically(args: {
|
||||
scope: TenantScope;
|
||||
releaseId: string;
|
||||
publishedAt: string;
|
||||
supersedes?: { release_id: string; superseded_at: string };
|
||||
}): Promise<void> {
|
||||
const target = this.releases.find(
|
||||
(r) =>
|
||||
r.release_id === args.releaseId &&
|
||||
r.tenant_id === args.scope.tenant_id &&
|
||||
r.project_id === args.scope.project_id,
|
||||
);
|
||||
if (!target) notFound(`release "${args.releaseId}" not found`);
|
||||
if (!isLegalTransition(target.state, 'published')) {
|
||||
conflict(`illegal transition ${target.state} -> published`);
|
||||
}
|
||||
const alreadyLive = this.releases.find(
|
||||
(r) =>
|
||||
r.dashboard_id === target.dashboard_id &&
|
||||
r.state === 'published' &&
|
||||
r.release_id !== args.supersedes?.release_id,
|
||||
);
|
||||
if (alreadyLive) conflict('one published release per dashboard');
|
||||
|
||||
if (args.supersedes) {
|
||||
const live = this.releases.find((r) => r.release_id === args.supersedes?.release_id);
|
||||
if (!live) notFound(`release "${args.supersedes.release_id}" not found`);
|
||||
if (!isLegalTransition(live.state, 'superseded')) {
|
||||
conflict(`illegal transition ${live.state} -> superseded`);
|
||||
}
|
||||
live.state = 'superseded';
|
||||
live.superseded_at = args.supersedes.superseded_at;
|
||||
live.superseded_by_release_id = target.release_id;
|
||||
}
|
||||
target.state = 'published';
|
||||
target.published_at = args.publishedAt;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,474 @@
|
||||
/**
|
||||
* src/analytics/domain/contract.ts
|
||||
* =================================
|
||||
* BI-P004 — ANALYTICS DATA CONTRACT.
|
||||
*
|
||||
* `DatasetContract` is the machine-checkable promise a dataset makes to every
|
||||
* consumer downstream of it. BI-P004's DoD is a GATE, not a document:
|
||||
*
|
||||
* "Breaking dataset schema changes fail compatibility gate unless
|
||||
* explicitly versioned."
|
||||
*
|
||||
* so `checkContractCompatibility` is the deliverable and the prose is the
|
||||
* commentary. The gate is fail-closed: an unrecognised change is BREAKING,
|
||||
* never "probably fine".
|
||||
*
|
||||
* Versioning is semver-shaped (`major.minor.patch`) with one rule that is not
|
||||
* negotiable: a BREAKING change requires a MAJOR bump. A breaking change
|
||||
* shipped under a minor bump is exactly the failure this gate exists to
|
||||
* catch, so it is reported as a violation rather than tolerated.
|
||||
*/
|
||||
import crypto from "node:crypto";
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities";
|
||||
import type { AnalyticsTier } from "./tiers";
|
||||
|
||||
export interface ContractField {
|
||||
name: string;
|
||||
type: AnalyticsFieldType;
|
||||
nullable: boolean;
|
||||
role: FieldRole;
|
||||
/** Part of the dataset's primary identifier. */
|
||||
primary_identifier?: boolean;
|
||||
classification?: PiiClassification;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface ContractQualityRule {
|
||||
rule_id: string;
|
||||
kind: string;
|
||||
field_name?: string;
|
||||
severity: "warn" | "block";
|
||||
}
|
||||
|
||||
/** BI-P004's named contents, one field each. */
|
||||
export interface DatasetContract {
|
||||
dataset_id: string;
|
||||
/** semver `major.minor.patch`. */
|
||||
version: string;
|
||||
tier: AnalyticsTier;
|
||||
fields: ContractField[];
|
||||
/** Field names forming the primary identifier, in order. */
|
||||
primary_identifiers: string[];
|
||||
/** Field names usable as dimensions (subset of `fields`). */
|
||||
dimensions: string[];
|
||||
/** Field names usable as measures (subset of `fields`). */
|
||||
measures: string[];
|
||||
/** Freshness SLO in seconds. */
|
||||
freshness_slo_seconds: number;
|
||||
retention_days: number;
|
||||
/** Dataset-level PII classification (the max of its fields, at least). */
|
||||
pii_classification: PiiClassification;
|
||||
/** Upstream refs: source ids, dataset ids, external system refs. */
|
||||
source_lineage: string[];
|
||||
quality_rules: ContractQualityRule[];
|
||||
/** GDPR purpose limitation — what this data may be USED for. */
|
||||
allowed_purposes: string[];
|
||||
}
|
||||
|
||||
export interface SemVer {
|
||||
major: number;
|
||||
minor: number;
|
||||
patch: number;
|
||||
}
|
||||
|
||||
const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/;
|
||||
|
||||
export function parseSemVer(v: string): SemVer {
|
||||
const m = SEMVER_RE.exec(v);
|
||||
if (!m) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`);
|
||||
}
|
||||
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) };
|
||||
}
|
||||
|
||||
// ── change classification ──────────────────────────────────────────────────
|
||||
|
||||
export type ChangeSeverity = "compatible" | "breaking";
|
||||
|
||||
export interface ContractChange {
|
||||
severity: ChangeSeverity;
|
||||
/** Stable machine code so callers can allow-list specific changes. */
|
||||
code: string;
|
||||
field?: string;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Type widening that does NOT break a reader. Deliberately conservative:
|
||||
* int64 -> float64 loses exactness for large integers, so it is NOT here.
|
||||
* Everything absent from this table is breaking.
|
||||
*/
|
||||
const SAFE_TYPE_WIDENING: Partial<Record<AnalyticsFieldType, readonly AnalyticsFieldType[]>> = {
|
||||
date: ["timestamp"],
|
||||
};
|
||||
|
||||
function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean {
|
||||
if (from === to) return true;
|
||||
return (SAFE_TYPE_WIDENING[from] ?? []).includes(to);
|
||||
}
|
||||
|
||||
/**
|
||||
* Diff two contracts for the SAME dataset and classify every change.
|
||||
*
|
||||
* Breaking (a reader written against `previous` can now fail or silently
|
||||
* change meaning):
|
||||
* - field removed
|
||||
* - field type changed other than a listed safe widening
|
||||
* - nullable false -> true (readers may not handle nulls)
|
||||
* - a field's role changes (a measure becoming a dimension changes meaning)
|
||||
* - primary identifier set changes
|
||||
* - a field leaves `measures` or `dimensions`
|
||||
* - freshness SLO loosened (a consumer's own SLO may depend on it)
|
||||
* - retention shortened (history a consumer relied on disappears)
|
||||
* - an allowed purpose is removed (a legitimate consumer becomes illegitimate)
|
||||
* - PII classification raised (consumers may lack clearance)
|
||||
* - a blocking quality rule is removed or downgraded to warn
|
||||
*
|
||||
* Compatible: additive nullable fields, new dimensions/measures, new quality
|
||||
* rules, loosened PII classification, added purposes, longer retention,
|
||||
* tightened freshness.
|
||||
*/
|
||||
export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] {
|
||||
if (previous.dataset_id !== next.dataset_id) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
"contract compatibility is only defined between versions of the SAME dataset",
|
||||
);
|
||||
}
|
||||
const changes: ContractChange[] = [];
|
||||
const prevFields = new Map(previous.fields.map((f) => [f.name, f]));
|
||||
const nextFields = new Map(next.fields.map((f) => [f.name, f]));
|
||||
|
||||
for (const [name, pf] of prevFields) {
|
||||
const nf = nextFields.get(name);
|
||||
if (!nf) {
|
||||
changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` });
|
||||
continue;
|
||||
}
|
||||
if (!typeChangeIsCompatible(pf.type, nf.type)) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "field_type_changed",
|
||||
field: name,
|
||||
detail: `field "${name}" type ${pf.type} -> ${nf.type}`,
|
||||
});
|
||||
} else if (pf.type !== nf.type) {
|
||||
changes.push({
|
||||
severity: "compatible",
|
||||
code: "field_type_widened",
|
||||
field: name,
|
||||
detail: `field "${name}" widened ${pf.type} -> ${nf.type}`,
|
||||
});
|
||||
}
|
||||
if (!pf.nullable && nf.nullable) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "field_became_nullable",
|
||||
field: name,
|
||||
detail: `field "${name}" became nullable`,
|
||||
});
|
||||
}
|
||||
if (pf.nullable && !nf.nullable) {
|
||||
changes.push({
|
||||
severity: "compatible",
|
||||
code: "field_became_required",
|
||||
field: name,
|
||||
detail: `field "${name}" is no longer nullable`,
|
||||
});
|
||||
}
|
||||
if (pf.role !== nf.role) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "field_role_changed",
|
||||
field: name,
|
||||
detail: `field "${name}" role ${pf.role} -> ${nf.role}`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const [name, nf] of nextFields) {
|
||||
if (prevFields.has(name)) continue;
|
||||
if (nf.nullable) {
|
||||
changes.push({
|
||||
severity: "compatible",
|
||||
code: "nullable_field_added",
|
||||
field: name,
|
||||
detail: `nullable field "${name}" added`,
|
||||
});
|
||||
} else {
|
||||
// A required new field breaks producers, and breaks readers that
|
||||
// reconstruct rows (they now have no value to supply).
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "required_field_added",
|
||||
field: name,
|
||||
detail: `required (non-nullable) field "${name}" added`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
const idsEqual =
|
||||
previous.primary_identifiers.length === next.primary_identifiers.length &&
|
||||
previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v);
|
||||
if (!idsEqual) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "primary_identifier_changed",
|
||||
detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const d of previous.dimensions) {
|
||||
if (!next.dimensions.includes(d)) {
|
||||
changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` });
|
||||
}
|
||||
}
|
||||
for (const d of next.dimensions) {
|
||||
if (!previous.dimensions.includes(d)) {
|
||||
changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` });
|
||||
}
|
||||
}
|
||||
for (const m of previous.measures) {
|
||||
if (!next.measures.includes(m)) {
|
||||
changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` });
|
||||
}
|
||||
}
|
||||
for (const m of next.measures) {
|
||||
if (!previous.measures.includes(m)) {
|
||||
changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` });
|
||||
}
|
||||
}
|
||||
|
||||
if (next.freshness_slo_seconds > previous.freshness_slo_seconds) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "freshness_slo_loosened",
|
||||
detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`,
|
||||
});
|
||||
}
|
||||
if (next.retention_days < previous.retention_days) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "retention_shortened",
|
||||
detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`,
|
||||
});
|
||||
}
|
||||
|
||||
for (const p of previous.allowed_purposes) {
|
||||
if (!next.allowed_purposes.includes(p)) {
|
||||
changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` });
|
||||
}
|
||||
}
|
||||
|
||||
const PII_RANK: Record<PiiClassification, number> = {
|
||||
none: 0,
|
||||
pseudonymous: 1,
|
||||
personal: 2,
|
||||
sensitive: 3,
|
||||
};
|
||||
if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "pii_classification_raised",
|
||||
detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`,
|
||||
});
|
||||
}
|
||||
|
||||
const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r]));
|
||||
for (const [id, pr] of prevRules) {
|
||||
const nr = next.quality_rules.find((r) => r.rule_id === id);
|
||||
if (!nr && pr.severity === "block") {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "blocking_quality_rule_removed",
|
||||
detail: `blocking quality rule "${id}" removed`,
|
||||
});
|
||||
} else if (nr && pr.severity === "block" && nr.severity === "warn") {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "blocking_quality_rule_downgraded",
|
||||
detail: `quality rule "${id}" downgraded block -> warn`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
if (previous.tier !== next.tier) {
|
||||
changes.push({
|
||||
severity: "breaking",
|
||||
code: "tier_changed",
|
||||
detail: `tier ${previous.tier} -> ${next.tier}`,
|
||||
});
|
||||
}
|
||||
|
||||
return changes;
|
||||
}
|
||||
|
||||
export interface CompatibilityVerdict {
|
||||
compatible: boolean;
|
||||
/** True when a MAJOR bump legitimises the breaking changes. */
|
||||
explicitly_versioned: boolean;
|
||||
changes: ContractChange[];
|
||||
breaking: ContractChange[];
|
||||
previous_version: string;
|
||||
next_version: string;
|
||||
/** Human-readable reason when `compatible` is false. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P004's gate. Returns a verdict; `assertContractCompatible` throws.
|
||||
*
|
||||
* Rules, in order:
|
||||
* 1. the version must move FORWARD (equal or lower is rejected outright);
|
||||
* 2. breaking changes REQUIRE a major bump — that is what "explicitly
|
||||
* versioned" means;
|
||||
* 3. a major bump with no breaking change is allowed (over-declaring is
|
||||
* safe; under-declaring is what we refuse).
|
||||
*/
|
||||
export function checkContractCompatibility(
|
||||
previous: DatasetContract,
|
||||
next: DatasetContract,
|
||||
): CompatibilityVerdict {
|
||||
const pv = parseSemVer(previous.version);
|
||||
const nv = parseSemVer(next.version);
|
||||
const changes = diffContracts(previous, next);
|
||||
const breaking = changes.filter((c) => c.severity === "breaking");
|
||||
|
||||
const forward =
|
||||
nv.major > pv.major ||
|
||||
(nv.major === pv.major && nv.minor > pv.minor) ||
|
||||
(nv.major === pv.major && nv.minor === pv.minor && nv.patch > pv.patch);
|
||||
if (!forward) {
|
||||
return {
|
||||
compatible: false,
|
||||
explicitly_versioned: false,
|
||||
changes,
|
||||
breaking,
|
||||
previous_version: previous.version,
|
||||
next_version: next.version,
|
||||
reason: `version must move forward: ${previous.version} -> ${next.version}`,
|
||||
};
|
||||
}
|
||||
|
||||
const majorBumped = nv.major > pv.major;
|
||||
if (breaking.length > 0 && !majorBumped) {
|
||||
return {
|
||||
compatible: false,
|
||||
explicitly_versioned: false,
|
||||
changes,
|
||||
breaking,
|
||||
previous_version: previous.version,
|
||||
next_version: next.version,
|
||||
reason:
|
||||
`${breaking.length} breaking change(s) require a MAJOR version bump ` +
|
||||
`(${previous.version} -> ${next.version} is not one): ` +
|
||||
breaking.map((b) => b.code).join(", "),
|
||||
};
|
||||
}
|
||||
|
||||
return {
|
||||
compatible: true,
|
||||
explicitly_versioned: majorBumped && breaking.length > 0,
|
||||
changes,
|
||||
breaking,
|
||||
previous_version: previous.version,
|
||||
next_version: next.version,
|
||||
};
|
||||
}
|
||||
|
||||
/** Throwing form for call sites that treat incompatibility as a failure. */
|
||||
export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict {
|
||||
const verdict = checkContractCompatibility(previous, next);
|
||||
if (!verdict.compatible) {
|
||||
throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed");
|
||||
}
|
||||
return verdict;
|
||||
}
|
||||
|
||||
/**
|
||||
* Structural validation of a single contract (independent of history).
|
||||
* Catches the contract that is internally inconsistent before it is ever
|
||||
* compared to anything.
|
||||
*/
|
||||
export function validateContract(contract: DatasetContract): void {
|
||||
parseSemVer(contract.version);
|
||||
if (!contract.dataset_id.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id");
|
||||
}
|
||||
if (contract.fields.length === 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`);
|
||||
}
|
||||
const names = new Set<string>();
|
||||
for (const f of contract.fields) {
|
||||
if (names.has(f.name)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`);
|
||||
}
|
||||
names.add(f.name);
|
||||
}
|
||||
for (const label of ["primary_identifiers", "dimensions", "measures"] as const) {
|
||||
for (const n of contract[label]) {
|
||||
if (!names.has(n)) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`${label} references unknown field "${n}" in ${contract.dataset_id}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
if (contract.primary_identifiers.length === 0) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`contract ${contract.dataset_id} declares no primary identifier`,
|
||||
);
|
||||
}
|
||||
if (contract.allowed_purposes.length === 0) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`,
|
||||
);
|
||||
}
|
||||
if (contract.freshness_slo_seconds <= 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive");
|
||||
}
|
||||
if (contract.retention_days < 0) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative");
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Stable hash of a contract's MEANING (field order-insensitive), used as
|
||||
* `DatasetVersion.contract_hash` so two versions that differ only in comment
|
||||
* text do not look like schema drift.
|
||||
*/
|
||||
export function contractHash(contract: DatasetContract): string {
|
||||
const canonical = {
|
||||
dataset_id: contract.dataset_id,
|
||||
version: contract.version,
|
||||
tier: contract.tier,
|
||||
fields: [...contract.fields]
|
||||
.map((f) => ({
|
||||
name: f.name,
|
||||
type: f.type,
|
||||
nullable: f.nullable,
|
||||
role: f.role,
|
||||
primary_identifier: f.primary_identifier ?? false,
|
||||
classification: f.classification ?? "none",
|
||||
}))
|
||||
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
|
||||
primary_identifiers: contract.primary_identifiers,
|
||||
dimensions: [...contract.dimensions].sort(),
|
||||
measures: [...contract.measures].sort(),
|
||||
freshness_slo_seconds: contract.freshness_slo_seconds,
|
||||
retention_days: contract.retention_days,
|
||||
pii_classification: contract.pii_classification,
|
||||
source_lineage: [...contract.source_lineage].sort(),
|
||||
quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)),
|
||||
allowed_purposes: [...contract.allowed_purposes].sort(),
|
||||
};
|
||||
return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
|
||||
}
|
||||
@@ -0,0 +1,536 @@
|
||||
/**
|
||||
* src/analytics/domain/entities.ts
|
||||
* =================================
|
||||
* BI-P002 — ANALYTICS DOMAIN MODEL.
|
||||
*
|
||||
* The nineteen canonical analytics entities, named EXACTLY as the program
|
||||
* lists them (no renaming, no merging, no "we already have something like
|
||||
* that"):
|
||||
*
|
||||
* AnalyticsDataset DatasetVersion DatasetField
|
||||
* DatasetGrant MetricDefinition MetricVersion
|
||||
* DimensionDefinition SemanticModel AnalyticsQuery
|
||||
* QueryExecution QueryResult DashboardDefinition
|
||||
* DashboardVersion ReportDefinition ReportRun
|
||||
* DataQualityRule DataQualityRun LineageReference
|
||||
* BIApplicationBinding
|
||||
*
|
||||
* Every tenant-owned record carries `tenant_id` + `project_id` where
|
||||
* applicable. The four version entities the program names as immutable
|
||||
* (MetricVersion, SemanticModel version, DashboardVersion, DatasetVersion)
|
||||
* are `readonly` here AND immutable in the database — sql/008 installs BEFORE
|
||||
* UPDATE/DELETE triggers, because a TypeScript `readonly` is a compiler
|
||||
* opinion, not a storage guarantee.
|
||||
*
|
||||
* NAME COLLISION, deliberate: src/services/dataset-version-service.ts already
|
||||
* exports an interface called `DatasetVersion` for the ST0RE-artifact-backed
|
||||
* dataset versioning that predates this program. That one is about *dataset
|
||||
* content artifacts*; this one is about *analytics dataset schema versions*
|
||||
* and lives in its own module namespace (`AnalyticsDatasetVersion` is NOT
|
||||
* used as the name — BI-P002 lists `DatasetVersion` verbatim and that is what
|
||||
* ships). Import them qualified when both are needed in one file.
|
||||
*
|
||||
* ARCHITECTURE BOUNDARY: these entities are owned by 0pt. A product repo
|
||||
* (VehIQ) supplies domain MEANING through `BIApplicationBinding`; it never
|
||||
* defines its own copies of these tables and never receives engine
|
||||
* credentials.
|
||||
*/
|
||||
import type { AnalyticsTier } from "./tiers";
|
||||
|
||||
/** Every tenant-owned analytics record. */
|
||||
export interface TenantScoped {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
}
|
||||
|
||||
// ── field/typing vocabulary ────────────────────────────────────────────────
|
||||
|
||||
export const ANALYTICS_FIELD_TYPES = [
|
||||
"string",
|
||||
"int64",
|
||||
"float64",
|
||||
"bool",
|
||||
"date",
|
||||
"timestamp",
|
||||
"decimal",
|
||||
"json",
|
||||
] as const;
|
||||
export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number];
|
||||
|
||||
/** A field's analytical role. Measures aggregate; dimensions group. */
|
||||
export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const;
|
||||
export type FieldRole = (typeof FIELD_ROLES)[number];
|
||||
|
||||
/** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */
|
||||
export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const;
|
||||
export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number];
|
||||
|
||||
// ── 1. AnalyticsDataset ────────────────────────────────────────────────────
|
||||
|
||||
export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const;
|
||||
export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number];
|
||||
|
||||
/**
|
||||
* BI-P003 DoD: "Every dataset declares tier, schema, owner, source, freshness,
|
||||
* quality, retention, classification." All eight are REQUIRED fields here —
|
||||
* an optional declaration is not a declaration.
|
||||
*/
|
||||
export interface AnalyticsDataset extends TenantScoped {
|
||||
dataset_id: string;
|
||||
name: string;
|
||||
/** BI-P003 tier. */
|
||||
tier: AnalyticsTier;
|
||||
/** Contract id + version this dataset's schema conforms to (BI-P004). */
|
||||
schema_contract_id: string;
|
||||
schema_contract_version: string;
|
||||
/** Accountable human/team, not the person who typed the create call. */
|
||||
owner_id: string;
|
||||
/** Upstream: a 0pt source_id, another dataset_id, or an external ref. */
|
||||
source_ref: string;
|
||||
/** Freshness SLO in seconds — how stale the data may be before it is late. */
|
||||
freshness_slo_seconds: number;
|
||||
/** DataQualityRule ids that gate this dataset. */
|
||||
quality_rule_ids: string[];
|
||||
/** Retention in days. 0 = keep indefinitely (must be justified in policy). */
|
||||
retention_days: number;
|
||||
classification: PiiClassification;
|
||||
status: AnalyticsDatasetStatus;
|
||||
/** BI-P003 escape hatch for RAW/NORMALIZED exposure to a BI surface. */
|
||||
raw_access_exception?: string;
|
||||
description?: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
created_by?: string;
|
||||
}
|
||||
|
||||
// ── 2. DatasetVersion (IMMUTABLE) ──────────────────────────────────────────
|
||||
|
||||
export interface DatasetVersion extends TenantScoped {
|
||||
readonly dataset_version_id: string;
|
||||
readonly dataset_id: string;
|
||||
/** Monotonic per dataset, starting at 1. */
|
||||
readonly version: number;
|
||||
/** The frozen DatasetContract JSON at this version (BI-P004). */
|
||||
readonly contract_hash: string;
|
||||
readonly schema_contract_version: string;
|
||||
readonly record_count?: number;
|
||||
readonly created_at: string;
|
||||
readonly created_by?: string;
|
||||
}
|
||||
|
||||
// ── 3. DatasetField ────────────────────────────────────────────────────────
|
||||
|
||||
export interface DatasetField extends TenantScoped {
|
||||
dataset_field_id: string;
|
||||
dataset_id: string;
|
||||
/** The dataset version this field description belongs to. */
|
||||
dataset_version: number;
|
||||
name: string;
|
||||
field_type: AnalyticsFieldType;
|
||||
nullable: boolean;
|
||||
role: FieldRole;
|
||||
/** True for the field(s) forming the dataset's primary identifier. */
|
||||
primary_identifier: boolean;
|
||||
classification: PiiClassification;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
// ── 4. DatasetGrant ────────────────────────────────────────────────────────
|
||||
|
||||
export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const;
|
||||
export type GrantLevel = (typeof GRANT_LEVELS)[number];
|
||||
|
||||
/**
|
||||
* Who may use a dataset, and FOR WHAT. `allowed_purposes` is the GDPR purpose
|
||||
* limitation carried into analytics — a grant without a purpose is refused
|
||||
* by the repository layer.
|
||||
*/
|
||||
export interface DatasetGrant extends TenantScoped {
|
||||
grant_id: string;
|
||||
dataset_id: string;
|
||||
/** Subject: a CL0UD principal, role, or BI application binding id. */
|
||||
subject_id: string;
|
||||
subject_type: "user" | "role" | "service" | "bi_application";
|
||||
level: GrantLevel;
|
||||
allowed_purposes: string[];
|
||||
expires_at?: string;
|
||||
created_at: string;
|
||||
created_by?: string;
|
||||
}
|
||||
|
||||
// ── 5. MetricDefinition ────────────────────────────────────────────────────
|
||||
|
||||
export const METRIC_AGGREGATIONS = [
|
||||
"sum",
|
||||
"avg",
|
||||
"min",
|
||||
"max",
|
||||
"count",
|
||||
"count_distinct",
|
||||
"ratio",
|
||||
"derived",
|
||||
] as const;
|
||||
export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number];
|
||||
|
||||
export interface MetricDefinition extends TenantScoped {
|
||||
metric_id: string;
|
||||
/** Stable business name; unique per tenant. */
|
||||
name: string;
|
||||
dataset_id: string;
|
||||
aggregation: MetricAggregation;
|
||||
/** Measure field aggregated, or the numerator for ratio metrics. */
|
||||
measure_field: string;
|
||||
/** Denominator for ratio metrics. */
|
||||
denominator_field?: string;
|
||||
/** Dimensions this metric may legitimately be sliced by. */
|
||||
allowed_dimension_ids: string[];
|
||||
/** Current live version number (points at a MetricVersion). */
|
||||
current_version: number;
|
||||
owner_id: string;
|
||||
description?: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
// ── 6. MetricVersion (IMMUTABLE) ───────────────────────────────────────────
|
||||
|
||||
export interface MetricVersion extends TenantScoped {
|
||||
readonly metric_version_id: string;
|
||||
readonly metric_id: string;
|
||||
readonly version: number;
|
||||
/** Frozen definition JSON — what this metric MEANT at this version. */
|
||||
readonly definition_json: string;
|
||||
readonly definition_hash: string;
|
||||
/** Free-text reason the metric changed; required for version > 1. */
|
||||
readonly change_reason?: string;
|
||||
readonly created_at: string;
|
||||
readonly created_by?: string;
|
||||
}
|
||||
|
||||
// ── 7. DimensionDefinition ─────────────────────────────────────────────────
|
||||
|
||||
export interface DimensionDefinition extends TenantScoped {
|
||||
dimension_id: string;
|
||||
name: string;
|
||||
dataset_id: string;
|
||||
/**
|
||||
* Physical column/key this dimension resolves to. This is the ONLY place a
|
||||
* caller-supplied dimension name becomes a physical identifier — the query
|
||||
* compiler resolves through here and never splices caller input (BI-P011
|
||||
* "dimension identifier injection").
|
||||
*/
|
||||
field_name: string;
|
||||
field_type: AnalyticsFieldType;
|
||||
classification: PiiClassification;
|
||||
description?: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// ── 8. SemanticModel (versioned; versions IMMUTABLE) ───────────────────────
|
||||
|
||||
export interface SemanticModel extends TenantScoped {
|
||||
semantic_model_id: string;
|
||||
name: string;
|
||||
/** Immutable version number — a change mints a new row, never an UPDATE. */
|
||||
readonly version: number;
|
||||
/** MetricFlow/Cube-compatible model document (BI-P029/BI-P032 consume it). */
|
||||
readonly model_json: string;
|
||||
readonly model_hash: string;
|
||||
dataset_ids: string[];
|
||||
owner_id: string;
|
||||
readonly created_at: string;
|
||||
readonly created_by?: string;
|
||||
}
|
||||
|
||||
// ── 9. AnalyticsQuery ──────────────────────────────────────────────────────
|
||||
|
||||
export const QUERY_SURFACES = ["metric", "sql", "projection"] as const;
|
||||
export type QuerySurface = (typeof QUERY_SURFACES)[number];
|
||||
|
||||
/**
|
||||
* A SAVED, reusable query. The request that actually runs is
|
||||
* `src/analytics/engine/query-compiler.ts`'s `MetricQueryRequest`; this entity
|
||||
* is its durable, governed form.
|
||||
*/
|
||||
export interface AnalyticsQuery extends TenantScoped {
|
||||
query_id: string;
|
||||
name: string;
|
||||
surface: QuerySurface;
|
||||
dataset_id: string;
|
||||
metric_id?: string;
|
||||
/** Serialized closed-DSL request. Never raw SQL for surface="metric". */
|
||||
request_json: string;
|
||||
owner_id: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
// ── 10. QueryExecution ─────────────────────────────────────────────────────
|
||||
|
||||
export const EXECUTION_STATES = [
|
||||
"queued",
|
||||
"running",
|
||||
"succeeded",
|
||||
"failed",
|
||||
"cancelled",
|
||||
"rejected",
|
||||
] as const;
|
||||
export type ExecutionState = (typeof EXECUTION_STATES)[number];
|
||||
|
||||
export interface QueryExecution extends TenantScoped {
|
||||
execution_id: string;
|
||||
query_id?: string;
|
||||
/** Engine-side identifier used to CANCEL this execution (BI-P009). */
|
||||
engine_query_id: string;
|
||||
state: ExecutionState;
|
||||
/** Which provider ran it: "clickhouse" | "duckdb" | "trino". */
|
||||
provider: string;
|
||||
submitted_at: string;
|
||||
started_at?: string;
|
||||
finished_at?: string;
|
||||
duration_ms?: number;
|
||||
rows_read?: number;
|
||||
bytes_read?: number;
|
||||
result_rows?: number;
|
||||
/** SiaxErrorCode when state="failed"/"rejected". Never a raw engine body. */
|
||||
error_code?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
// ── 11. QueryResult ────────────────────────────────────────────────────────
|
||||
|
||||
export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const;
|
||||
export type ResultEncoding = (typeof RESULT_ENCODINGS)[number];
|
||||
|
||||
export interface QueryResult extends TenantScoped {
|
||||
result_id: string;
|
||||
execution_id: string;
|
||||
encoding: ResultEncoding;
|
||||
row_count: number;
|
||||
/** Canonical ST0RE ref when materialised; inline results carry no bytes here. */
|
||||
artifact_ref?: string;
|
||||
/** True when the engine truncated at max_result_rows (BI-P012). */
|
||||
truncated: boolean;
|
||||
expires_at?: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
// ── 12. DashboardDefinition ────────────────────────────────────────────────
|
||||
|
||||
export interface DashboardDefinition extends TenantScoped {
|
||||
dashboard_id: string;
|
||||
name: string;
|
||||
/** Live version pointer. */
|
||||
current_version: number;
|
||||
owner_id: string;
|
||||
/** Datasets the dashboard reads — checked against BI-P003 tier rules. */
|
||||
dataset_ids: string[];
|
||||
description?: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
// ── 13. DashboardVersion (IMMUTABLE) ───────────────────────────────────────
|
||||
|
||||
export interface DashboardVersion extends TenantScoped {
|
||||
readonly dashboard_version_id: string;
|
||||
readonly dashboard_id: string;
|
||||
readonly version: number;
|
||||
/** Frozen layout/panel document. */
|
||||
readonly layout_json: string;
|
||||
readonly layout_hash: string;
|
||||
readonly created_at: string;
|
||||
readonly created_by?: string;
|
||||
}
|
||||
|
||||
// ── 14. ReportDefinition ───────────────────────────────────────────────────
|
||||
|
||||
export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const;
|
||||
export type ReportFormat = (typeof REPORT_FORMATS)[number];
|
||||
|
||||
export interface ReportDefinition extends TenantScoped {
|
||||
report_id: string;
|
||||
name: string;
|
||||
dashboard_id?: string;
|
||||
query_id?: string;
|
||||
format: ReportFormat;
|
||||
/** Cron expression; undefined = on-demand only. */
|
||||
schedule_cron?: string;
|
||||
owner_id: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
// ── 15. ReportRun ──────────────────────────────────────────────────────────
|
||||
|
||||
export interface ReportRun extends TenantScoped {
|
||||
report_run_id: string;
|
||||
report_id: string;
|
||||
state: ExecutionState;
|
||||
started_at: string;
|
||||
finished_at?: string;
|
||||
/** ST0RE artifact ref of the produced report. */
|
||||
artifact_ref?: string;
|
||||
error_code?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
// ── 16. DataQualityRule ────────────────────────────────────────────────────
|
||||
|
||||
export const QUALITY_RULE_KINDS = [
|
||||
"not_null",
|
||||
"unique",
|
||||
"accepted_values",
|
||||
"range",
|
||||
"freshness",
|
||||
"row_count",
|
||||
"referential",
|
||||
"custom",
|
||||
] as const;
|
||||
export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number];
|
||||
|
||||
export const QUALITY_SEVERITIES = ["warn", "block"] as const;
|
||||
export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number];
|
||||
|
||||
export interface DataQualityRule extends TenantScoped {
|
||||
rule_id: string;
|
||||
dataset_id: string;
|
||||
kind: QualityRuleKind;
|
||||
/** Field the rule applies to; omitted for dataset-level rules. */
|
||||
field_name?: string;
|
||||
/** Rule parameters (bounds, accepted values, max age). */
|
||||
params_json: string;
|
||||
severity: QualitySeverity;
|
||||
created_at: string;
|
||||
created_by?: string;
|
||||
}
|
||||
|
||||
// ── 17. DataQualityRun ─────────────────────────────────────────────────────
|
||||
|
||||
export interface DataQualityRun extends TenantScoped {
|
||||
quality_run_id: string;
|
||||
rule_id: string;
|
||||
dataset_id: string;
|
||||
dataset_version?: number;
|
||||
passed: boolean;
|
||||
/** Rows that violated the rule. */
|
||||
violation_count: number;
|
||||
evaluated_rows: number;
|
||||
started_at: string;
|
||||
finished_at?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
// ── 18. LineageReference ───────────────────────────────────────────────────
|
||||
|
||||
export const LINEAGE_DIRECTIONS = ["input", "output"] as const;
|
||||
export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number];
|
||||
|
||||
/**
|
||||
* The queryable projection of an OpenLineage edge (BI-P027). AUD0 stays the
|
||||
* system of record; this is 0pt's local index so "what fed this dashboard"
|
||||
* is answerable without leaving the analytics plane.
|
||||
*/
|
||||
export interface LineageReference extends TenantScoped {
|
||||
lineage_reference_id: string;
|
||||
/** OpenLineage run id this edge came from. */
|
||||
run_id: string;
|
||||
job_name: string;
|
||||
direction: LineageDirection;
|
||||
/** The analytics entity this edge touches. */
|
||||
entity_type: "dataset" | "metric" | "dashboard" | "report";
|
||||
entity_id: string;
|
||||
/** Namespaced dataset name on the other side of the edge. */
|
||||
counterpart_ref: string;
|
||||
observed_at: string;
|
||||
}
|
||||
|
||||
// ── 19. BIApplicationBinding ───────────────────────────────────────────────
|
||||
|
||||
export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const;
|
||||
export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number];
|
||||
|
||||
/**
|
||||
* How an external BI surface is attached to the analytics plane.
|
||||
*
|
||||
* NORMATIVE (BI program §0 + the SIAX/0pt ownership boundary): this binding
|
||||
* carries NO engine credential. A bound application authenticates to the 0pt
|
||||
* Analytics API with a CL0UD identity and receives compiled, tenant-forced
|
||||
* results — it never learns a ClickHouse DSN, user, or password. The absence
|
||||
* of a credential field here is the enforcement, not a comment.
|
||||
*/
|
||||
export interface BIApplicationBinding extends TenantScoped {
|
||||
binding_id: string;
|
||||
kind: BiApplicationKind;
|
||||
name: string;
|
||||
/** CL0UD principal the application authenticates as. */
|
||||
principal_id: string;
|
||||
/** Datasets/metrics this application may reach at all. */
|
||||
dataset_ids: string[];
|
||||
metric_ids: string[];
|
||||
allowed_purposes: string[];
|
||||
enabled: boolean;
|
||||
created_at: string;
|
||||
created_by?: string;
|
||||
}
|
||||
|
||||
// ── registry of the canonical nineteen ─────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The canonical entity names, verbatim from BI-P002. Persisted-schema tests
|
||||
* assert sql/008 defines a table for every one of these, so a future rename
|
||||
* or omission is a test failure rather than drift.
|
||||
*/
|
||||
export const ANALYTICS_DOMAIN_ENTITIES = [
|
||||
"AnalyticsDataset",
|
||||
"DatasetVersion",
|
||||
"DatasetField",
|
||||
"DatasetGrant",
|
||||
"MetricDefinition",
|
||||
"MetricVersion",
|
||||
"DimensionDefinition",
|
||||
"SemanticModel",
|
||||
"AnalyticsQuery",
|
||||
"QueryExecution",
|
||||
"QueryResult",
|
||||
"DashboardDefinition",
|
||||
"DashboardVersion",
|
||||
"ReportDefinition",
|
||||
"ReportRun",
|
||||
"DataQualityRule",
|
||||
"DataQualityRun",
|
||||
"LineageReference",
|
||||
"BIApplicationBinding",
|
||||
] as const;
|
||||
export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number];
|
||||
|
||||
/** Entity -> physical table in sql/008_analytics_domain_model.sql. */
|
||||
export const ANALYTICS_ENTITY_TABLES: Record<AnalyticsDomainEntity, string> = {
|
||||
AnalyticsDataset: "opt_analytics_dataset",
|
||||
DatasetVersion: "opt_analytics_dataset_version",
|
||||
DatasetField: "opt_analytics_dataset_field",
|
||||
DatasetGrant: "opt_analytics_dataset_grant",
|
||||
MetricDefinition: "opt_analytics_metric_definition",
|
||||
MetricVersion: "opt_analytics_metric_version",
|
||||
DimensionDefinition: "opt_analytics_dimension_definition",
|
||||
SemanticModel: "opt_analytics_semantic_model",
|
||||
AnalyticsQuery: "opt_analytics_query",
|
||||
QueryExecution: "opt_analytics_query_execution",
|
||||
QueryResult: "opt_analytics_query_result",
|
||||
DashboardDefinition: "opt_analytics_dashboard_definition",
|
||||
DashboardVersion: "opt_analytics_dashboard_version",
|
||||
ReportDefinition: "opt_analytics_report_definition",
|
||||
ReportRun: "opt_analytics_report_run",
|
||||
DataQualityRule: "opt_analytics_data_quality_rule",
|
||||
DataQualityRun: "opt_analytics_data_quality_run",
|
||||
LineageReference: "opt_analytics_lineage_reference",
|
||||
BIApplicationBinding: "opt_analytics_bi_application_binding",
|
||||
};
|
||||
|
||||
/** The four entities BI-P002 requires to be immutable once written. */
|
||||
export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [
|
||||
"DatasetVersion",
|
||||
"MetricVersion",
|
||||
"SemanticModel",
|
||||
"DashboardVersion",
|
||||
];
|
||||
@@ -0,0 +1,83 @@
|
||||
/**
|
||||
* src/analytics/domain/tiers.ts
|
||||
* ==============================
|
||||
* BI-P003 — ANALYTICS DATA TIERS.
|
||||
*
|
||||
* The four canonical tiers every analytics dataset in SIAX belongs to. The
|
||||
* tier is a CONTRACT about where a dataset may be read from and by whom, not
|
||||
* a naming convention:
|
||||
*
|
||||
* RAW -> ST0RE Parquet/Iceberg (as-ingested, no business meaning)
|
||||
* NORMALIZED -> ST0RE Iceberg (typed, deduplicated, still source-shaped)
|
||||
* CURATED -> ST0RE Iceberg (business entities, modelled)
|
||||
* SERVING -> ClickHouse / projection (query-optimised, dashboard-facing)
|
||||
*
|
||||
* BI-P003's normative sentence is "No business dashboard reads arbitrary raw
|
||||
* ingestion tables by default" — `assertReadableByBiSurface` is where that
|
||||
* sentence stops being prose. A BI surface (dashboard, report, embedded
|
||||
* chart) may read SERVING and CURATED; RAW and NORMALIZED require an
|
||||
* explicit, recorded exception on the dataset itself.
|
||||
*
|
||||
* ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product
|
||||
* repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers.
|
||||
*/
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
|
||||
export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const;
|
||||
export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number];
|
||||
|
||||
/** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */
|
||||
export const TIER_PHYSICAL_HOME: Record<AnalyticsTier, string> = {
|
||||
RAW: "st0re:iceberg|parquet",
|
||||
NORMALIZED: "st0re:iceberg",
|
||||
CURATED: "st0re:iceberg",
|
||||
SERVING: "clickhouse:projection",
|
||||
};
|
||||
|
||||
/** Tiers a BI presentation surface may read without an explicit exception. */
|
||||
export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"];
|
||||
|
||||
export function isAnalyticsTier(v: unknown): v is AnalyticsTier {
|
||||
return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v);
|
||||
}
|
||||
|
||||
/**
|
||||
* Rank used for promotion checks: a dataset may only be BUILT FROM a tier at
|
||||
* or below its own (SERVING can be built from CURATED; CURATED must never be
|
||||
* built from SERVING, which would make the projection its own source).
|
||||
*/
|
||||
export const TIER_RANK: Record<AnalyticsTier, number> = {
|
||||
RAW: 0,
|
||||
NORMALIZED: 1,
|
||||
CURATED: 2,
|
||||
SERVING: 3,
|
||||
};
|
||||
|
||||
/**
|
||||
* BI-P003 DoD gate. `raw_access_exception` is the recorded, auditable escape
|
||||
* hatch — absent it, a RAW/NORMALIZED dataset is invisible to dashboards.
|
||||
*/
|
||||
export function assertReadableByBiSurface(
|
||||
dataset: { dataset_id: string; tier: AnalyticsTier; raw_access_exception?: string },
|
||||
surface: string,
|
||||
): void {
|
||||
if (BI_READABLE_TIERS.includes(dataset.tier)) return;
|
||||
if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return;
|
||||
throw new SiaxError(
|
||||
"AUTHORIZATION_DENIED",
|
||||
"0pt.analytics",
|
||||
`BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` +
|
||||
`without a recorded raw_access_exception (BI-P003)`,
|
||||
);
|
||||
}
|
||||
|
||||
/** A dataset may not declare a source tier ABOVE its own tier. */
|
||||
export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void {
|
||||
if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,563 @@
|
||||
/**
|
||||
* src/analytics/engine/clickhouse-engine.ts
|
||||
* ==========================================
|
||||
* BI-P009 — ClickHouse as the CANONICAL `SiaxAnalyticsEngine` provider.
|
||||
*
|
||||
* PRESERVATION (BI-P009's first sentence): `ClickHouseAnalytics`
|
||||
* (src/analytics/clickhouse-analytics.ts) is untouched by this change — not
|
||||
* one line. This engine COMPOSES it:
|
||||
*
|
||||
* * `health()` delegates to `ClickHouseAnalytics.health()`.
|
||||
* * `executeMetric()` on the pre-existing `opt_metric_events` table
|
||||
* delegates to `ClickHouseAnalytics.queryMetric()`, i.e. keeps the
|
||||
* already-proven forced-tenant path exactly as it is.
|
||||
* * everything else (the BI-P010 tables) goes through
|
||||
* `src/analytics/engine/query-compiler.ts`, which forces the same
|
||||
* predicate for the same reason.
|
||||
*
|
||||
* "Do not regress the existing forced tenant predicate" is enforced by
|
||||
* tests/analytics/tenant-predicate-regression.test.ts, which fails if either
|
||||
* compiler stops emitting `tenant_id = {tenant:String}`.
|
||||
*
|
||||
* BI-P012 governance is applied on EVERY statement, including the delegated
|
||||
* ones — the settings ride on the HTTP request, and `readonly=1` means the
|
||||
* query text cannot re-set them.
|
||||
*
|
||||
* LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts
|
||||
* ClickHouse pipelines, never quarantined market data.
|
||||
*/
|
||||
import crypto from "node:crypto";
|
||||
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors";
|
||||
import { ClickHouseAnalytics } from "../clickhouse-analytics";
|
||||
import {
|
||||
compileMetricQuery,
|
||||
compileProjectionQuery,
|
||||
dimensionCatalogFrom,
|
||||
FORCED_TENANT_PREDICATE,
|
||||
type DimensionCatalog,
|
||||
} from "./query-compiler";
|
||||
import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema";
|
||||
import {
|
||||
applyQuota,
|
||||
clickhouseSettings,
|
||||
ConcurrencyGate,
|
||||
DEFAULT_BUDGETS,
|
||||
planningSettings,
|
||||
type ResourceBudget,
|
||||
} from "./resource-governance";
|
||||
import type {
|
||||
AnalystQueryRequest,
|
||||
AnalyticsIdentity,
|
||||
EngineResult,
|
||||
HealthReport,
|
||||
MetricRequest,
|
||||
ProjectionIngestRequest,
|
||||
ProjectionRefreshResult,
|
||||
QueryEstimate,
|
||||
QueryExplanation,
|
||||
SiaxAnalyticsEngine,
|
||||
} from "./siax-analytics-engine";
|
||||
|
||||
export interface ClickHouseEngineOptions {
|
||||
baseUrl: string;
|
||||
/** "user:password" — from an env/secret ref, never a literal in code. */
|
||||
basicAuth?: string;
|
||||
database?: string;
|
||||
timeoutMs?: number;
|
||||
/** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */
|
||||
dimensionCatalogs?: Record<string, Record<string, string>>;
|
||||
schema?: Omit<AnalyticsSchemaOptions, "database">;
|
||||
}
|
||||
|
||||
const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined };
|
||||
|
||||
export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
|
||||
readonly provider = "clickhouse";
|
||||
|
||||
/** PRESERVED pre-existing implementation, composed not replaced. */
|
||||
private readonly legacy: ClickHouseAnalytics;
|
||||
private readonly baseUrl: string;
|
||||
private readonly authHeader?: string;
|
||||
private readonly database: string;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly dimensionCatalogs: Record<string, Record<string, string>>;
|
||||
private readonly schemaOpts: Omit<AnalyticsSchemaOptions, "database">;
|
||||
private readonly gate: ConcurrencyGate;
|
||||
/** query_id -> AbortController, so cancelQuery can stop the client side too. */
|
||||
private readonly inFlight = new Map<string, AbortController>();
|
||||
|
||||
constructor(opts: ClickHouseEngineOptions) {
|
||||
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
||||
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
|
||||
this.database = opts.database ?? "opt_analytics";
|
||||
this.timeoutMs = opts.timeoutMs ?? 30_000;
|
||||
this.dimensionCatalogs = opts.dimensionCatalogs ?? {};
|
||||
this.schemaOpts = opts.schema ?? {};
|
||||
this.legacy = new ClickHouseAnalytics({
|
||||
baseUrl: opts.baseUrl,
|
||||
basicAuth: opts.basicAuth,
|
||||
database: this.database,
|
||||
timeoutMs: this.timeoutMs,
|
||||
});
|
||||
this.gate = new ConcurrencyGate((tenantId) => this.budgetForTenant(tenantId).maxConcurrentQueries);
|
||||
}
|
||||
|
||||
/** Access to the preserved implementation, for callers that still use it. */
|
||||
get legacyAnalytics(): ClickHouseAnalytics {
|
||||
return this.legacy;
|
||||
}
|
||||
|
||||
private budgetForTenant(_tenantId: string): ResourceBudget {
|
||||
return DEFAULT_BUDGETS.INTERACTIVE;
|
||||
}
|
||||
|
||||
private budgetFor(identity: AnalyticsIdentity): ResourceBudget {
|
||||
const tier = identity.budget_tier ?? "INTERACTIVE";
|
||||
return applyQuota(DEFAULT_BUDGETS[tier], identity.quota);
|
||||
}
|
||||
|
||||
private catalogFor(identity: AnalyticsIdentity): DimensionCatalog {
|
||||
const pairs = this.dimensionCatalogs[identity.tenant_id];
|
||||
return pairs ? dimensionCatalogFrom(pairs) : EMPTY_CATALOG;
|
||||
}
|
||||
|
||||
/** Fail closed: a missing/blank verified tenant is never a wildcard. */
|
||||
private assertIdentity(identity: AnalyticsIdentity): void {
|
||||
if (!identity?.tenant_id || !identity.tenant_id.trim()) {
|
||||
throw new SiaxError(
|
||||
"UNAUTHENTICATED",
|
||||
"0pt.analytics",
|
||||
"analytics requires a verified CL0UD identity carrying tenant_id",
|
||||
);
|
||||
}
|
||||
if (!identity.principal_id || !identity.principal_id.trim()) {
|
||||
throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id");
|
||||
}
|
||||
}
|
||||
|
||||
// ── schema ───────────────────────────────────────────────────────────────
|
||||
|
||||
/** Apply BI-P010 DDL (idempotent) plus the preserved legacy table. */
|
||||
async ensureSchema(): Promise<void> {
|
||||
await this.legacy.ensureSchema();
|
||||
for (const stmt of analyticsSchemaDdl({ database: this.database, ...this.schemaOpts })) {
|
||||
await this.ddl(stmt);
|
||||
}
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 1: health ─────────────────────────────────────────
|
||||
|
||||
async health(): Promise<HealthReport> {
|
||||
try {
|
||||
const legacyOk = await this.legacy.health();
|
||||
if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" };
|
||||
const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {});
|
||||
const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v;
|
||||
return { healthy: true, provider: this.provider, version };
|
||||
} catch (err) {
|
||||
return {
|
||||
healthy: false,
|
||||
provider: this.provider,
|
||||
detail: err instanceof SiaxError ? err.code : "unreachable",
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 2: executeMetric ──────────────────────────────────
|
||||
|
||||
async executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise<EngineResult> {
|
||||
this.assertIdentity(identity);
|
||||
const budget = this.budgetFor(identity);
|
||||
const release = this.gate.acquire(identity.tenant_id);
|
||||
const started = Date.now();
|
||||
const queryId = this.newQueryId(identity);
|
||||
try {
|
||||
const compiled = compileMetricQuery(
|
||||
{
|
||||
tenant_id: identity.tenant_id, // FORCED from the verified identity
|
||||
project_id: identity.project_id,
|
||||
dataset_id: request.dataset_id,
|
||||
metric: request.metric,
|
||||
aggregate: request.aggregate,
|
||||
group_by: request.group_by,
|
||||
filters: request.filters,
|
||||
from_time: request.from_time,
|
||||
to_time: request.to_time,
|
||||
limit: request.limit,
|
||||
table: request.table,
|
||||
},
|
||||
this.database,
|
||||
this.catalogFor(identity),
|
||||
budget.maxResultRows,
|
||||
);
|
||||
const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as
|
||||
| ClickHouseJsonResponse
|
||||
| undefined;
|
||||
const groupNames = request.group_by ?? [];
|
||||
const rows = (json?.data ?? []).map((r) => {
|
||||
const groups: Record<string, string> = {};
|
||||
groupNames.forEach((name, i) => {
|
||||
const v = (r as Record<string, unknown>)[`grp${i}`];
|
||||
groups[name] = v === undefined || v === null ? "" : String(v);
|
||||
});
|
||||
return {
|
||||
...(groupNames.length ? { groups } : {}),
|
||||
value: Number((r as { value: number | string }).value),
|
||||
};
|
||||
});
|
||||
const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), budget.maxResultRows));
|
||||
return {
|
||||
rows,
|
||||
query_id: queryId,
|
||||
truncated: rows.length >= limit,
|
||||
stats: {
|
||||
duration_ms: Date.now() - started,
|
||||
rows_read: json?.statistics?.rows_read,
|
||||
bytes_read: json?.statistics?.bytes_read,
|
||||
result_rows: rows.length,
|
||||
},
|
||||
};
|
||||
} finally {
|
||||
release();
|
||||
this.inFlight.delete(queryId);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Delegating path to the PRESERVED implementation for the pre-existing
|
||||
* `opt_metric_events` table. Kept so no existing caller has to migrate and
|
||||
* the already-proven forced-predicate path stays in service.
|
||||
*/
|
||||
async executeLegacyMetric(
|
||||
identity: AnalyticsIdentity,
|
||||
request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number },
|
||||
): Promise<EngineResult> {
|
||||
this.assertIdentity(identity);
|
||||
const started = Date.now();
|
||||
const rows = await this.legacy.queryMetric({
|
||||
tenant_id: identity.tenant_id, // FORCED from the verified identity
|
||||
metric: request.metric,
|
||||
aggregate: request.aggregate,
|
||||
dataset_id: request.dataset_id,
|
||||
group_by: request.group_by,
|
||||
from_time: request.from_time,
|
||||
to_time: request.to_time,
|
||||
limit: request.limit,
|
||||
});
|
||||
return {
|
||||
rows: rows.map((r) => ({
|
||||
...(r.group !== undefined && request.group_by ? { groups: { [request.group_by]: r.group } } : {}),
|
||||
value: r.value,
|
||||
})),
|
||||
query_id: this.newQueryId(identity),
|
||||
truncated: false,
|
||||
stats: { duration_ms: Date.now() - started, result_rows: rows.length },
|
||||
};
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 3: executeQuery (analyst SQL) ─────────────────────
|
||||
|
||||
async executeQuery(identity: AnalyticsIdentity, _request: AnalystQueryRequest): Promise<EngineResult> {
|
||||
this.assertIdentity(identity);
|
||||
// Deliberate, documented refusal — see AnalystQueryRequest's doc comment.
|
||||
// Opening a raw-SQL surface before BI-P017 (SQL dialect contract) and
|
||||
// BI-P033 (analytics authorization) exist would create precisely the
|
||||
// "raw SQL attempt" hole BI-P011's adversarial list enumerates.
|
||||
throw new SiaxError(
|
||||
"CAPABILITY_UNAVAILABLE",
|
||||
"0pt.analytics",
|
||||
"analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land",
|
||||
);
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 4: estimateQuery ──────────────────────────────────
|
||||
|
||||
async estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryEstimate> {
|
||||
this.assertIdentity(identity);
|
||||
const budget = this.budgetFor(identity);
|
||||
const compiled = compileMetricQuery(
|
||||
{
|
||||
tenant_id: identity.tenant_id,
|
||||
project_id: identity.project_id,
|
||||
dataset_id: request.dataset_id,
|
||||
metric: request.metric,
|
||||
aggregate: request.aggregate,
|
||||
group_by: request.group_by,
|
||||
filters: request.filters,
|
||||
from_time: request.from_time,
|
||||
to_time: request.to_time,
|
||||
limit: request.limit,
|
||||
table: request.table,
|
||||
},
|
||||
this.database,
|
||||
this.catalogFor(identity),
|
||||
budget.maxResultRows,
|
||||
);
|
||||
// `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read
|
||||
// AFTER primary-key pruning — which is why tenant_id leads every ORDER BY
|
||||
// in schema.ts: the estimate for one tenant does not include everyone.
|
||||
const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, "");
|
||||
let rowsToRead = 0;
|
||||
let partsRead = 0;
|
||||
try {
|
||||
const res = (await this.raw(
|
||||
`EXPLAIN ESTIMATE ${estimateSql} FORMAT JSON`,
|
||||
compiled.params,
|
||||
planningSettings(budget),
|
||||
)) as { data?: { rows?: number | string; parts?: number | string }[] } | undefined;
|
||||
for (const row of res?.data ?? []) {
|
||||
rowsToRead += Number(row.rows ?? 0);
|
||||
partsRead += Number(row.parts ?? 0);
|
||||
}
|
||||
} catch (err) {
|
||||
// An estimate that cannot be produced must not read as "cheap".
|
||||
throw err instanceof SiaxError
|
||||
? err
|
||||
: new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query");
|
||||
}
|
||||
// Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes.
|
||||
// 128 B/row is a deliberately PESSIMISTIC constant for these tables
|
||||
// (measured mean in the BI-P010 benchmark is lower), so the gate errs
|
||||
// toward refusing rather than admitting an oversized query.
|
||||
const bytesToRead = rowsToRead * 128;
|
||||
const within = rowsToRead <= budget.maxRowsToRead && bytesToRead <= budget.maxBytesToRead;
|
||||
return {
|
||||
estimated_rows_to_read: rowsToRead,
|
||||
estimated_bytes_to_read: bytesToRead,
|
||||
within_budget: within,
|
||||
budget,
|
||||
...(within
|
||||
? {}
|
||||
: {
|
||||
reason:
|
||||
`estimate ${rowsToRead} rows / ${bytesToRead} bytes over budget ` +
|
||||
`(${budget.maxRowsToRead} rows / ${budget.maxBytesToRead} bytes), parts=${partsRead}`,
|
||||
}),
|
||||
};
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 5: cancelQuery ────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Two-sided cancellation. The client-side abort stops us waiting; the
|
||||
* server-side KILL QUERY stops ClickHouse burning CPU. Cancelling is
|
||||
* tenant-scoped: the query id embeds the tenant and is checked, so tenant A
|
||||
* cannot cancel tenant B's work.
|
||||
*/
|
||||
async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise<boolean> {
|
||||
this.assertIdentity(identity);
|
||||
if (!queryId.startsWith(this.queryIdPrefix(identity))) {
|
||||
throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant");
|
||||
}
|
||||
this.inFlight.get(queryId)?.abort();
|
||||
this.inFlight.delete(queryId);
|
||||
try {
|
||||
// KILL QUERY is a mutation of server state, so it cannot ride the
|
||||
// readonly=1 settings the SELECT path uses.
|
||||
await this.raw(
|
||||
`KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`,
|
||||
{ param_killId: queryId },
|
||||
{ max_execution_time: "10" },
|
||||
);
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 6: explain ────────────────────────────────────────
|
||||
|
||||
async explain(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryExplanation> {
|
||||
this.assertIdentity(identity);
|
||||
const budget = this.budgetFor(identity);
|
||||
const compiled = compileMetricQuery(
|
||||
{
|
||||
tenant_id: identity.tenant_id,
|
||||
project_id: identity.project_id,
|
||||
dataset_id: request.dataset_id,
|
||||
metric: request.metric,
|
||||
aggregate: request.aggregate,
|
||||
group_by: request.group_by,
|
||||
filters: request.filters,
|
||||
from_time: request.from_time,
|
||||
to_time: request.to_time,
|
||||
limit: request.limit,
|
||||
table: request.table,
|
||||
},
|
||||
this.database,
|
||||
this.catalogFor(identity),
|
||||
budget.maxResultRows,
|
||||
);
|
||||
const sql = compiled.sql.replace(/ FORMAT JSON$/, "");
|
||||
const res = (await this.raw(
|
||||
`EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`,
|
||||
compiled.params,
|
||||
planningSettings(budget),
|
||||
)) as string | undefined;
|
||||
return {
|
||||
plan: typeof res === "string" ? res : JSON.stringify(res ?? ""),
|
||||
forced_tenant_predicate: FORCED_TENANT_PREDICATE,
|
||||
table: compiled.table,
|
||||
};
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 7: ingestProjection ───────────────────────────────
|
||||
|
||||
async ingestProjection(
|
||||
identity: AnalyticsIdentity,
|
||||
request: ProjectionIngestRequest,
|
||||
): Promise<{ ingested: number }> {
|
||||
this.assertIdentity(identity);
|
||||
if (request.rows.length === 0) return { ingested: 0 };
|
||||
const now = new Date();
|
||||
const version = Math.floor(now.getTime());
|
||||
const body = request.rows
|
||||
.map((r) => {
|
||||
if (!r.entity_key || !r.entity_key.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key");
|
||||
}
|
||||
return JSON.stringify({
|
||||
// tenant/project come from the VERIFIED identity, never the row —
|
||||
// a row-supplied tenant_id is the INF0 Wave-2 bug class.
|
||||
tenant_id: identity.tenant_id,
|
||||
project_id: identity.project_id ?? "",
|
||||
dataset_id: request.dataset_id,
|
||||
entity_key: r.entity_key,
|
||||
as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""),
|
||||
version: r.version ?? version,
|
||||
attributes: r.attributes ?? {},
|
||||
measures: r.measures ?? {},
|
||||
});
|
||||
})
|
||||
.join("\n");
|
||||
await this.raw(
|
||||
`INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`,
|
||||
{},
|
||||
{},
|
||||
undefined,
|
||||
body,
|
||||
);
|
||||
return { ingested: request.rows.length };
|
||||
}
|
||||
|
||||
// ── BI-P009 capability 8: refreshProjection ──────────────────────────────
|
||||
|
||||
/**
|
||||
* ReplacingMergeTree deduplicates on a background merge, so a read straight
|
||||
* after an ingest can still see superseded rows. `OPTIMIZE ... FINAL` forces
|
||||
* the merge for THIS tenant's partition only — the table is
|
||||
* `PARTITION BY (tenant_id)` precisely so one tenant's refresh does not
|
||||
* rewrite another tenant's parts (BI-P012's "without degrading unrelated
|
||||
* tenant" applies to maintenance too, not only to queries).
|
||||
*/
|
||||
async refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise<ProjectionRefreshResult> {
|
||||
this.assertIdentity(identity);
|
||||
await this.raw(
|
||||
`OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`,
|
||||
{ param_tenant: identity.tenant_id },
|
||||
{ max_execution_time: "120" },
|
||||
);
|
||||
const compiled = compileProjectionQuery(
|
||||
{ tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 },
|
||||
this.database,
|
||||
1,
|
||||
);
|
||||
const countSql =
|
||||
`SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` +
|
||||
`WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`;
|
||||
const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as
|
||||
| { data?: { value: number | string }[] }
|
||||
| undefined;
|
||||
return {
|
||||
dataset_id,
|
||||
row_count: Number(res?.data?.[0]?.value ?? 0),
|
||||
refreshed_at: new Date().toISOString(),
|
||||
};
|
||||
}
|
||||
|
||||
// ── transport ────────────────────────────────────────────────────────────
|
||||
|
||||
private queryIdPrefix(identity: AnalyticsIdentity): string {
|
||||
return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`;
|
||||
}
|
||||
|
||||
private newQueryId(identity: AnalyticsIdentity): string {
|
||||
return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`;
|
||||
}
|
||||
|
||||
/** DDL path: no readonly, no result parsing beyond errors. */
|
||||
private async ddl(sql: string): Promise<void> {
|
||||
await this.raw(sql, {}, { max_execution_time: "60" });
|
||||
}
|
||||
|
||||
private async raw(
|
||||
sql: string,
|
||||
params: Record<string, string>,
|
||||
settings: Record<string, string>,
|
||||
queryId?: string,
|
||||
body?: string,
|
||||
): Promise<unknown> {
|
||||
const url = new URL(this.baseUrl);
|
||||
url.searchParams.set("query", sql);
|
||||
for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
|
||||
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
||||
if (queryId) url.searchParams.set("query_id", queryId);
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.authHeader) headers["Authorization"] = this.authHeader;
|
||||
|
||||
const controller = new AbortController();
|
||||
if (queryId) this.inFlight.set(queryId, controller);
|
||||
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: body ?? "",
|
||||
signal: controller.signal,
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
|
||||
} finally {
|
||||
clearTimeout(timeout);
|
||||
}
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
// ClickHouse error text can echo the query; keep only the numeric code
|
||||
// so a caller never receives another tenant's identifiers back.
|
||||
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
|
||||
throw new SiaxError(
|
||||
siaxErrorCodeForUpstreamStatus(res.status),
|
||||
"clickhouse",
|
||||
`query failed (CH code ${code})`,
|
||||
);
|
||||
}
|
||||
if (!text) return undefined;
|
||||
let parsed: unknown;
|
||||
try {
|
||||
parsed = JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
// DEFENCE IN DEPTH against a partial result that looks complete.
|
||||
// ClickHouse normally answers a breached limit with an HTTP error status
|
||||
// (measured on 24.8: 408 for max_execution_time, 500 for
|
||||
// max_rows_to_read), which `res.ok` above already catches. But when rows
|
||||
// have already been flushed, the JSON body can carry an `exception` key
|
||||
// under a 200 — a truncated answer indistinguishable from a whole one.
|
||||
// Treat any body carrying an exception as a failure, whatever the status.
|
||||
if (parsed && typeof parsed === "object" && "exception" in parsed) {
|
||||
const ex = String((parsed as { exception?: unknown }).exception ?? "");
|
||||
const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown";
|
||||
throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`);
|
||||
}
|
||||
return parsed;
|
||||
}
|
||||
}
|
||||
|
||||
interface ClickHouseJsonResponse {
|
||||
data?: Record<string, unknown>[];
|
||||
statistics?: { rows_read?: number; bytes_read?: number; elapsed?: number };
|
||||
}
|
||||
@@ -0,0 +1,304 @@
|
||||
/**
|
||||
* src/analytics/engine/query-compiler.ts
|
||||
* =======================================
|
||||
* BI-P011 — CLICKHOUSE TENANT ISOLATION, layer 2:
|
||||
* "`0pt` query compiler forces tenant".
|
||||
*
|
||||
* The compiler is the ONLY thing in this repo that turns an analytics request
|
||||
* into SQL text. Its contract:
|
||||
*
|
||||
* * `tenant_id` is not an argument the caller may omit — it comes from the
|
||||
* VERIFIED CL0UD identity (layer 1) and is written into the WHERE clause
|
||||
* by this function, unconditionally, as a bound parameter.
|
||||
* * caller-supplied VALUES are never spliced into SQL text. They travel as
|
||||
* ClickHouse HTTP `param_*` query parameters.
|
||||
* * caller-supplied IDENTIFIERS (a dimension name, a group-by column) are
|
||||
* never spliced either. They are RESOLVED against a declared allow-list
|
||||
* of `DimensionDefinition`s and only the resolved, already-known physical
|
||||
* name reaches the SQL — so "dimension identifier injection" has no
|
||||
* surface, because caller text never becomes an identifier.
|
||||
* * there is no raw-SQL entry point. `compileMetricQuery` cannot express a
|
||||
* UNION, a subquery, or a second table, so a caller cannot reach a table
|
||||
* the compiler did not choose.
|
||||
*
|
||||
* WHY A SEPARATE MODULE from clickhouse-analytics.ts: that file's compiler
|
||||
* already forces the tenant predicate correctly and BI-P009 says PRESERVE IT.
|
||||
* This module is the generalised compiler for the BI-P010 tables; it repeats
|
||||
* the guarantee rather than replacing it, and
|
||||
* tests/analytics/tenant-predicate-regression.test.ts asserts BOTH.
|
||||
*/
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
import type { AnalyticsTable } from "./schema";
|
||||
|
||||
/** A physical identifier that has been VALIDATED, not merely received. */
|
||||
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
||||
|
||||
export function assertSafeIdentifier(value: string, what: string): string {
|
||||
if (!IDENT_RE.test(value)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`);
|
||||
}
|
||||
return value;
|
||||
}
|
||||
|
||||
export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const;
|
||||
export type MetricAggregate = (typeof METRIC_AGGREGATES)[number];
|
||||
|
||||
/**
|
||||
* The dimensions a caller is allowed to name, resolved from
|
||||
* `DimensionDefinition` rows. The compiler looks a caller's dimension name up
|
||||
* HERE; a miss is a rejection, never a pass-through.
|
||||
*/
|
||||
export interface DimensionCatalog {
|
||||
/** caller-facing dimension name -> physical dimension key in dim_names. */
|
||||
resolve(name: string): string | undefined;
|
||||
}
|
||||
|
||||
export function dimensionCatalogFrom(pairs: Record<string, string>): DimensionCatalog {
|
||||
const map = new Map(Object.entries(pairs));
|
||||
return {
|
||||
resolve(name: string): string | undefined {
|
||||
// Own-property lookup on a Map — no prototype-chain surprises
|
||||
// ("constructor", "__proto__" resolve to undefined, as they must).
|
||||
return map.get(name);
|
||||
},
|
||||
};
|
||||
}
|
||||
|
||||
export interface MetricQueryRequest {
|
||||
/** Supplied by the engine from the verified identity, NOT by the caller. */
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
dataset_id?: string;
|
||||
metric: string;
|
||||
aggregate: MetricAggregate;
|
||||
/** Caller-facing dimension names; resolved through the catalog. */
|
||||
group_by?: string[];
|
||||
/** Equality filters on dimensions: caller-facing name -> value. */
|
||||
filters?: Record<string, string>;
|
||||
from_time?: string;
|
||||
to_time?: string;
|
||||
limit?: number;
|
||||
/** Which BI-P010 table to read. Chosen by the engine, not the caller. */
|
||||
table?: AnalyticsTable;
|
||||
}
|
||||
|
||||
export interface CompiledQuery {
|
||||
sql: string;
|
||||
/** ClickHouse HTTP `param_*` entries. Values NEVER appear in `sql`. */
|
||||
params: Record<string, string>;
|
||||
/** The predicate the compiler forced, for assertion in tests. */
|
||||
forcedTenantPredicate: string;
|
||||
table: AnalyticsTable;
|
||||
}
|
||||
|
||||
/**
|
||||
* The exact predicate text every compiled query must contain. Exported so a
|
||||
* regression test can assert on the literal string — if someone deletes the
|
||||
* forcing, the test fails on a missing substring rather than on a subtle
|
||||
* behavioural difference that a mocked engine would happily hide.
|
||||
*/
|
||||
export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}";
|
||||
|
||||
const AGG_SQL: Record<MetricAggregate, string> = {
|
||||
sum: "sum(value)",
|
||||
avg: "avg(value)",
|
||||
min: "min(value)",
|
||||
max: "max(value)",
|
||||
count: "count()",
|
||||
count_distinct: "uniqExact(event_id)",
|
||||
};
|
||||
|
||||
const MAX_GROUP_BY = 8;
|
||||
|
||||
/**
|
||||
* Compile a closed-DSL metric request into tenant-forced SQL.
|
||||
*
|
||||
* @param req the request; `tenant_id` MUST already be the verified identity.
|
||||
* @param db the analytics database name (engine-controlled, validated).
|
||||
* @param catalog the tenant's declared dimensions.
|
||||
* @param maxResultRows the governance ceiling (BI-P012); `limit` is clamped.
|
||||
*/
|
||||
export function compileMetricQuery(
|
||||
req: MetricQueryRequest,
|
||||
db: string,
|
||||
catalog: DimensionCatalog,
|
||||
maxResultRows: number,
|
||||
): CompiledQuery {
|
||||
if (!req.tenant_id || !req.tenant_id.trim()) {
|
||||
// Fail closed. A blank tenant must never compile to "no predicate".
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
|
||||
}
|
||||
assertSafeIdentifier(db, "database");
|
||||
const table: AnalyticsTable = req.table ?? "opt_analytics_event";
|
||||
assertSafeIdentifier(table, "table");
|
||||
if (!METRIC_AGGREGATES.includes(req.aggregate)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate");
|
||||
}
|
||||
|
||||
const params: Record<string, string> = {
|
||||
param_tenant: req.tenant_id,
|
||||
param_metric: req.metric,
|
||||
};
|
||||
|
||||
// ── the forced predicate. First condition, always, no branch. ─────────
|
||||
const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"];
|
||||
|
||||
if (req.project_id) {
|
||||
conditions.push("project_id = {project:String}");
|
||||
params.param_project = req.project_id;
|
||||
}
|
||||
if (req.dataset_id) {
|
||||
conditions.push("dataset_id = {dataset:String}");
|
||||
params.param_dataset = req.dataset_id;
|
||||
}
|
||||
if (req.from_time) {
|
||||
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
|
||||
params.param_fromTime = req.from_time;
|
||||
}
|
||||
if (req.to_time) {
|
||||
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
|
||||
params.param_toTime = req.to_time;
|
||||
}
|
||||
|
||||
// ── filters: names resolved through the catalog, values parameterized ──
|
||||
let filterIdx = 0;
|
||||
for (const [name, value] of Object.entries(req.filters ?? {})) {
|
||||
const physical = catalog.resolve(name);
|
||||
if (physical === undefined) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`);
|
||||
}
|
||||
// `physical` came from the catalog, but validate anyway: a corrupted or
|
||||
// hostile catalog row must not become an injection vector either.
|
||||
assertSafeIdentifier(physical, "dimension");
|
||||
const nameKey = `f${filterIdx}n`;
|
||||
const valKey = `f${filterIdx}v`;
|
||||
conditions.push(
|
||||
`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`,
|
||||
);
|
||||
params[`param_${nameKey}`] = physical;
|
||||
params[`param_${valKey}`] = value;
|
||||
filterIdx += 1;
|
||||
}
|
||||
|
||||
// ── group by: same resolution, plus a hard cardinality ceiling ─────────
|
||||
const groupBy = req.group_by ?? [];
|
||||
if (groupBy.length > MAX_GROUP_BY) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.analytics",
|
||||
`at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`,
|
||||
);
|
||||
}
|
||||
const selectParts: string[] = [];
|
||||
const groupParts: string[] = [];
|
||||
groupBy.forEach((name, i) => {
|
||||
const physical = catalog.resolve(name);
|
||||
if (physical === undefined) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`);
|
||||
}
|
||||
assertSafeIdentifier(physical, "dimension");
|
||||
const key = `g${i}`;
|
||||
// The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text.
|
||||
selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`);
|
||||
groupParts.push(`grp${i}`);
|
||||
params[`param_${key}`] = physical;
|
||||
});
|
||||
|
||||
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
|
||||
|
||||
const sql =
|
||||
`SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` +
|
||||
`FROM ${db}.${table} ` +
|
||||
`WHERE ${conditions.join(" AND ")} ` +
|
||||
(groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") +
|
||||
`LIMIT ${limit} FORMAT JSON`;
|
||||
|
||||
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table };
|
||||
}
|
||||
|
||||
/**
|
||||
* Compile a projection lookup (SERVING tier, BI-P010's opt_dataset_projection).
|
||||
* Same forcing, different table and key.
|
||||
*/
|
||||
export function compileProjectionQuery(
|
||||
req: { tenant_id: string; project_id?: string; dataset_id: string; entity_keys?: string[]; limit?: number },
|
||||
db: string,
|
||||
maxResultRows: number,
|
||||
): CompiledQuery {
|
||||
if (!req.tenant_id || !req.tenant_id.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
|
||||
}
|
||||
assertSafeIdentifier(db, "database");
|
||||
const params: Record<string, string> = {
|
||||
param_tenant: req.tenant_id,
|
||||
param_dataset: req.dataset_id,
|
||||
};
|
||||
const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"];
|
||||
if (req.project_id) {
|
||||
conditions.push("project_id = {project:String}");
|
||||
params.param_project = req.project_id;
|
||||
}
|
||||
if (req.entity_keys && req.entity_keys.length > 0) {
|
||||
conditions.push("has({entityKeys:Array(String)}, entity_key)");
|
||||
params.param_entityKeys = JSON.stringify(req.entity_keys);
|
||||
}
|
||||
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
|
||||
const sql =
|
||||
`SELECT entity_key, as_of, version, attributes, measures ` +
|
||||
`FROM ${db}.opt_dataset_projection ` +
|
||||
`WHERE ${conditions.join(" AND ")} ` +
|
||||
`ORDER BY entity_key LIMIT ${limit} FORMAT JSON`;
|
||||
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" };
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P011 layer-3 bootstrap: the DDL for a RESTRICTED per-tenant ClickHouse
|
||||
* user with a row policy pinning `tenant_id`.
|
||||
*
|
||||
* This is the layer that survives a compiler bug. The compiler forcing the
|
||||
* predicate is layer 2; if a future refactor drops it, layer 3 still returns
|
||||
* zero foreign rows because the SERVER filters them. The BI-P011 proof runs
|
||||
* the adversarial suite twice — once through an unrestricted user (hole open)
|
||||
* and once through this restricted user (hole closed) — because a claim of
|
||||
* defence in depth that has never been tested with one layer removed is not
|
||||
* evidence.
|
||||
*
|
||||
* `tenantId` is validated as an identifier-safe token AND embedded as a
|
||||
* quoted literal; ClickHouse's CREATE ROW POLICY takes no bound parameters,
|
||||
* so validation is the only protection available and it is applied strictly.
|
||||
*/
|
||||
export function restrictedTenantUserDdl(args: {
|
||||
db: string;
|
||||
tenantId: string;
|
||||
userName: string;
|
||||
password: string;
|
||||
tables: readonly string[];
|
||||
settingsProfile?: string;
|
||||
}): string[] {
|
||||
assertSafeIdentifier(args.db, "database");
|
||||
assertSafeIdentifier(args.userName, "user");
|
||||
// A tenant id is `t_<slug>` in this estate; enforce that shape strictly
|
||||
// rather than trusting the caller.
|
||||
if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe");
|
||||
}
|
||||
if (args.password.includes("'") || args.password.includes("\\")) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded");
|
||||
}
|
||||
const profile = args.settingsProfile;
|
||||
if (profile) assertSafeIdentifier(profile, "settings profile");
|
||||
|
||||
const stmts: string[] = [
|
||||
`CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` +
|
||||
(profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`),
|
||||
];
|
||||
for (const table of args.tables) {
|
||||
assertSafeIdentifier(table, "table");
|
||||
stmts.push(
|
||||
`CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` +
|
||||
`FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`,
|
||||
);
|
||||
stmts.push(`GRANT SELECT ON ${args.db}.${table} TO ${args.userName}`);
|
||||
}
|
||||
return stmts;
|
||||
}
|
||||
@@ -0,0 +1,222 @@
|
||||
/**
|
||||
* src/analytics/engine/resource-governance.ts
|
||||
* ============================================
|
||||
* BI-P012 — CLICKHOUSE RESOURCE GOVERNANCE.
|
||||
*
|
||||
* Every setting BI-P012 names, in one place, with a value:
|
||||
*
|
||||
* max_execution_time max_result_rows max_rows_to_read
|
||||
* max_bytes_to_read max_memory_usage max_concurrent_queries
|
||||
*
|
||||
* Three layers, and the order matters:
|
||||
*
|
||||
* 1. SERVER-SIDE SETTINGS travel with every statement on the HTTP
|
||||
* interface. ClickHouse enforces them; the client cannot opt out
|
||||
* because the client is what writes them. `readonly=1` on top means a
|
||||
* SELECT cannot re-set them mid-query (readonly=1 forbids SETTINGS
|
||||
* changes as well as writes — that is the whole reason it is here and
|
||||
* not merely `readonly=2`).
|
||||
* 2. `max_concurrent_queries` is NOT a per-query setting in ClickHouse; it
|
||||
* is a per-USER quota. `concurrencySlotSql` renders the CREATE
|
||||
* SETTINGS PROFILE that carries it, applied to the tenant's restricted
|
||||
* user (BI-P011 layer 3) so one tenant's fan-out cannot consume the
|
||||
* whole server.
|
||||
* 3. A local admission gate (`ConcurrencyGate`) refuses the N+1st
|
||||
* in-flight query per tenant before it ever reaches the network. Two
|
||||
* layers because the ClickHouse quota is per-user and one 0pt process
|
||||
* can still queue work faster than the server rejects it.
|
||||
*
|
||||
* Budget tiers exist because "one limit for everyone" is how a small tenant
|
||||
* subsidises a query bomb. `INTERACTIVE` is the dashboard default and is
|
||||
* deliberately mean.
|
||||
*/
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
|
||||
export interface ResourceBudget {
|
||||
/** Seconds. ClickHouse `max_execution_time`. */
|
||||
maxExecutionTimeSec: number;
|
||||
/** Rows returned to the caller. ClickHouse `max_result_rows`. */
|
||||
maxResultRows: number;
|
||||
/** Rows SCANNED (not returned). ClickHouse `max_rows_to_read`. */
|
||||
maxRowsToRead: number;
|
||||
/** Bytes scanned. ClickHouse `max_bytes_to_read`. */
|
||||
maxBytesToRead: number;
|
||||
/** Bytes of RAM per query. ClickHouse `max_memory_usage`. */
|
||||
maxMemoryUsageBytes: number;
|
||||
/** In-flight queries per tenant. Enforced by quota + local gate. */
|
||||
maxConcurrentQueries: number;
|
||||
}
|
||||
|
||||
export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const;
|
||||
export type BudgetTier = (typeof BUDGET_TIERS)[number];
|
||||
|
||||
/**
|
||||
* Defaults. Chosen against the shape of the analytics tables in
|
||||
* `src/analytics/engine/schema.ts`, not plucked from the air:
|
||||
* INTERACTIVE must answer a dashboard panel inside a human's patience budget,
|
||||
* so it caps the SCAN, which is what actually costs time on a MergeTree.
|
||||
*/
|
||||
export const DEFAULT_BUDGETS: Record<BudgetTier, ResourceBudget> = {
|
||||
INTERACTIVE: {
|
||||
maxExecutionTimeSec: 10,
|
||||
maxResultRows: 10_000,
|
||||
maxRowsToRead: 50_000_000,
|
||||
maxBytesToRead: 2 * 1024 * 1024 * 1024,
|
||||
maxMemoryUsageBytes: 2 * 1024 * 1024 * 1024,
|
||||
maxConcurrentQueries: 8,
|
||||
},
|
||||
ANALYTICAL: {
|
||||
maxExecutionTimeSec: 60,
|
||||
maxResultRows: 500_000,
|
||||
maxRowsToRead: 2_000_000_000,
|
||||
maxBytesToRead: 32 * 1024 * 1024 * 1024,
|
||||
maxMemoryUsageBytes: 8 * 1024 * 1024 * 1024,
|
||||
maxConcurrentQueries: 4,
|
||||
},
|
||||
BATCH: {
|
||||
maxExecutionTimeSec: 900,
|
||||
maxResultRows: 10_000_000,
|
||||
maxRowsToRead: 50_000_000_000,
|
||||
maxBytesToRead: 512 * 1024 * 1024 * 1024,
|
||||
maxMemoryUsageBytes: 24 * 1024 * 1024 * 1024,
|
||||
maxConcurrentQueries: 2,
|
||||
},
|
||||
};
|
||||
|
||||
/**
|
||||
* A CL0UD-issued quota may only ever TIGHTEN a tier, never loosen it. A
|
||||
* control-plane bug that hands out a generous number must not be able to
|
||||
* disarm the engine's own ceiling.
|
||||
*/
|
||||
export function applyQuota(base: ResourceBudget, quota: Partial<ResourceBudget> | undefined): ResourceBudget {
|
||||
if (!quota) return { ...base };
|
||||
const pick = (b: number, q: number | undefined): number =>
|
||||
typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b;
|
||||
return {
|
||||
maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec),
|
||||
maxResultRows: pick(base.maxResultRows, quota.maxResultRows),
|
||||
maxRowsToRead: pick(base.maxRowsToRead, quota.maxRowsToRead),
|
||||
maxBytesToRead: pick(base.maxBytesToRead, quota.maxBytesToRead),
|
||||
maxMemoryUsageBytes: pick(base.maxMemoryUsageBytes, quota.maxMemoryUsageBytes),
|
||||
maxConcurrentQueries: pick(base.maxConcurrentQueries, quota.maxConcurrentQueries),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The settings that ride along on the HTTP request. `readonly: "1"` is what
|
||||
* makes them un-overridable from inside the query text.
|
||||
*
|
||||
* `timeout_overflow_mode` / `result_overflow_mode` / `read_overflow_mode` are
|
||||
* set to `throw` EXPLICITLY: ClickHouse's default for some of these is
|
||||
* `break`, which silently returns a PARTIAL result. A partial result that
|
||||
* looks complete is the analytics equivalent of a false green, so the engine
|
||||
* refuses it.
|
||||
*/
|
||||
export function clickhouseSettings(budget: ResourceBudget): Record<string, string> {
|
||||
return {
|
||||
readonly: "1",
|
||||
max_execution_time: String(budget.maxExecutionTimeSec),
|
||||
timeout_overflow_mode: "throw",
|
||||
max_result_rows: String(budget.maxResultRows),
|
||||
result_overflow_mode: "throw",
|
||||
max_rows_to_read: String(budget.maxRowsToRead),
|
||||
max_bytes_to_read: String(budget.maxBytesToRead),
|
||||
read_overflow_mode: "throw",
|
||||
max_memory_usage: String(budget.maxMemoryUsageBytes),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* Settings for a PLANNER call (`EXPLAIN ESTIMATE`, `EXPLAIN PLAN`).
|
||||
*
|
||||
* Measured, not assumed: running `EXPLAIN ESTIMATE` under the caller's own
|
||||
* `max_rows_to_read` makes ClickHouse refuse the ESTIMATE itself with code 158
|
||||
* whenever the estimate would be over budget — i.e. exactly the case the
|
||||
* estimate exists to detect. An estimator that cannot describe an expensive
|
||||
* query is useless, so the read ceilings are deliberately NOT applied here.
|
||||
* The planner still runs `readonly = 1` under a short deadline and a memory
|
||||
* cap, so this is a narrower envelope, never an unbounded one.
|
||||
*/
|
||||
export function planningSettings(budget: ResourceBudget): Record<string, string> {
|
||||
return {
|
||||
readonly: "1",
|
||||
max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)),
|
||||
timeout_overflow_mode: "throw",
|
||||
max_memory_usage: String(budget.maxMemoryUsageBytes),
|
||||
};
|
||||
}
|
||||
|
||||
/** Every setting name BI-P012 requires, for the DoD assertion. */
|
||||
export const REQUIRED_GOVERNANCE_SETTINGS = [
|
||||
"max_execution_time",
|
||||
"max_result_rows",
|
||||
"max_rows_to_read",
|
||||
"max_bytes_to_read",
|
||||
"max_memory_usage",
|
||||
"max_concurrent_queries",
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* `max_concurrent_queries` lives on a SETTINGS PROFILE, not on a statement.
|
||||
* Rendered here so BI-P011's restricted-user bootstrap and BI-P012's budget
|
||||
* come from one source.
|
||||
*/
|
||||
export function concurrencyProfileSql(profileName: string, budget: ResourceBudget): string[] {
|
||||
return [
|
||||
`CREATE SETTINGS PROFILE IF NOT EXISTS ${profileName} SETTINGS ` +
|
||||
`max_execution_time = ${budget.maxExecutionTimeSec}, ` +
|
||||
`max_result_rows = ${budget.maxResultRows}, ` +
|
||||
`max_rows_to_read = ${budget.maxRowsToRead}, ` +
|
||||
`max_bytes_to_read = ${budget.maxBytesToRead}, ` +
|
||||
`max_memory_usage = ${budget.maxMemoryUsageBytes}, ` +
|
||||
`readonly = 1`,
|
||||
`CREATE QUOTA IF NOT EXISTS ${profileName}_quota ` +
|
||||
`FOR INTERVAL 1 SECOND MAX queries = ${budget.maxConcurrentQueries * 20} ` +
|
||||
`TO ALL EXCEPT default`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Local admission control. One instance per process; keyed by tenant so a
|
||||
* noisy tenant is throttled without touching anyone else's slots — which is
|
||||
* exactly BI-P012's DoD wording ("without degrading unrelated tenant").
|
||||
*/
|
||||
export class ConcurrencyGate {
|
||||
private readonly inFlight = new Map<string, number>();
|
||||
|
||||
constructor(private readonly limitFor: (tenantId: string) => number) {}
|
||||
|
||||
/** Returns a release function. Throws RATE_LIMITED when the tenant is full. */
|
||||
acquire(tenantId: string): () => void {
|
||||
const limit = this.limitFor(tenantId);
|
||||
const current = this.inFlight.get(tenantId) ?? 0;
|
||||
if (current >= limit) {
|
||||
throw new SiaxError(
|
||||
"RATE_LIMITED",
|
||||
"0pt.analytics",
|
||||
`tenant already has ${current} analytics queries in flight (limit ${limit})`,
|
||||
);
|
||||
}
|
||||
this.inFlight.set(tenantId, current + 1);
|
||||
let released = false;
|
||||
return () => {
|
||||
if (released) return;
|
||||
released = true;
|
||||
const n = (this.inFlight.get(tenantId) ?? 1) - 1;
|
||||
if (n <= 0) this.inFlight.delete(tenantId);
|
||||
else this.inFlight.set(tenantId, n);
|
||||
};
|
||||
}
|
||||
|
||||
/** Observability: in-flight count for a tenant (0 when idle). */
|
||||
inFlightFor(tenantId: string): number {
|
||||
return this.inFlight.get(tenantId) ?? 0;
|
||||
}
|
||||
|
||||
/** Total in-flight across all tenants. */
|
||||
totalInFlight(): number {
|
||||
let n = 0;
|
||||
for (const v of this.inFlight.values()) n += v;
|
||||
return n;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,169 @@
|
||||
/**
|
||||
* src/analytics/engine/schema.ts
|
||||
* ===============================
|
||||
* BI-P010 — CLICKHOUSE PRODUCTION SCHEMA.
|
||||
*
|
||||
* BI-P010's explicit warning is "Avoid one giant unbounded table for every
|
||||
* analytics use case." The existing `opt_metric_events` (owned by
|
||||
* src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one
|
||||
* table for one job: low-cardinality capability metrics. It stays, and it is
|
||||
* NOT extended into the catch-all.
|
||||
*
|
||||
* Four tables, four jobs, four retention profiles:
|
||||
*
|
||||
* opt_metric_events (PRE-EXISTING, not defined here) — capability
|
||||
* metrics, Map(String,String) dimensions, 90d.
|
||||
* opt_analytics_event — high-volume per-tenant event stream. Wide
|
||||
* dimension arrays, SHORT TTL (raw events are not
|
||||
* the serving tier; they age into the rollups).
|
||||
* opt_metric_daily — AggregatingMergeTree rollup fed by a
|
||||
* MATERIALIZED VIEW off opt_analytics_event.
|
||||
* This is what a dashboard time-series reads;
|
||||
* long TTL, tiny footprint.
|
||||
* opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED
|
||||
* dataset: one row per (tenant, dataset, entity),
|
||||
* ReplacingMergeTree so a refresh is idempotent.
|
||||
*
|
||||
* Why the split is load-bearing, not cosmetic:
|
||||
* * different ORDER BY keys. A dashboard filtering (tenant, metric, day)
|
||||
* and a projection looked up by (tenant, dataset, entity_key) cannot
|
||||
* share one primary key without one of them degenerating to a full scan.
|
||||
* * different TTLs. Raw events must expire fast; rollups must not.
|
||||
* * different cardinality. `LowCardinality(String)` is a win on
|
||||
* tenant/metric/dataset and a LOSS on a free-form entity key, so they
|
||||
* cannot be the same column type in the same table.
|
||||
*
|
||||
* EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it
|
||||
* is what makes the forced tenant predicate (BI-P011) a primary-key range
|
||||
* scan instead of a filter applied after reading everyone's rows, and it is
|
||||
* what makes a per-tenant row policy cheap.
|
||||
*
|
||||
* LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this
|
||||
* schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts
|
||||
* ClickHouse pipelines or quarantined market data.
|
||||
*/
|
||||
|
||||
export interface AnalyticsSchemaOptions {
|
||||
database: string;
|
||||
/** Raw event retention. Deliberately short — rollups carry the history. */
|
||||
eventRetentionDays?: number;
|
||||
/** Rollup retention. */
|
||||
rollupRetentionDays?: number;
|
||||
/** Serving-projection retention. */
|
||||
projectionRetentionDays?: number;
|
||||
}
|
||||
|
||||
/** Physical tables this module owns (opt_metric_events is NOT one of them). */
|
||||
export const ANALYTICS_TABLES = [
|
||||
"opt_analytics_event",
|
||||
"opt_metric_daily",
|
||||
"opt_dataset_projection",
|
||||
] as const;
|
||||
export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number];
|
||||
|
||||
/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */
|
||||
export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const;
|
||||
|
||||
/**
|
||||
* DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so
|
||||
* applying it to a live database is idempotent (the same contract
|
||||
* `ClickHouseAnalytics.ensureSchema` already keeps).
|
||||
*/
|
||||
export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] {
|
||||
const db = opts.database;
|
||||
const eventTtl = Math.floor(opts.eventRetentionDays ?? 30);
|
||||
const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095);
|
||||
const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365);
|
||||
|
||||
return [
|
||||
`CREATE DATABASE IF NOT EXISTS ${db}`,
|
||||
|
||||
// ── raw event stream ────────────────────────────────────────────────
|
||||
// Dimensions as parallel arrays (not Map) so ClickHouse can prune on
|
||||
// dimension VALUES via the bloom filter index; Map(String,String) forces
|
||||
// reading the whole map. `event_id` makes deduplication possible on
|
||||
// replay without a ReplacingMergeTree's merge latency.
|
||||
`CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event (
|
||||
tenant_id LowCardinality(String),
|
||||
project_id LowCardinality(String),
|
||||
dataset_id LowCardinality(String),
|
||||
metric LowCardinality(String),
|
||||
event_id String,
|
||||
event_time DateTime64(3, 'UTC'),
|
||||
value Float64,
|
||||
dim_names Array(LowCardinality(String)),
|
||||
dim_values Array(String),
|
||||
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
||||
INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (tenant_id, project_id, dataset_id, metric, event_time)
|
||||
TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY
|
||||
SETTINGS index_granularity = 8192`,
|
||||
|
||||
// ── daily rollup (serving tier for time series) ─────────────────────
|
||||
`CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily (
|
||||
tenant_id LowCardinality(String),
|
||||
project_id LowCardinality(String),
|
||||
dataset_id LowCardinality(String),
|
||||
metric LowCardinality(String),
|
||||
day Date,
|
||||
sum_state AggregateFunction(sum, Float64),
|
||||
count_state AggregateFunction(count, Float64),
|
||||
min_state AggregateFunction(min, Float64),
|
||||
max_state AggregateFunction(max, Float64)
|
||||
)
|
||||
ENGINE = AggregatingMergeTree()
|
||||
PARTITION BY toYYYYMM(day)
|
||||
ORDER BY (tenant_id, project_id, dataset_id, metric, day)
|
||||
TTL day + INTERVAL ${rollupTtl} DAY`,
|
||||
|
||||
`CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv
|
||||
TO ${db}.opt_metric_daily AS
|
||||
SELECT
|
||||
tenant_id,
|
||||
project_id,
|
||||
dataset_id,
|
||||
metric,
|
||||
toDate(event_time) AS day,
|
||||
sumState(value) AS sum_state,
|
||||
countState(value) AS count_state,
|
||||
minState(value) AS min_state,
|
||||
maxState(value) AS max_state
|
||||
FROM ${db}.opt_analytics_event
|
||||
GROUP BY tenant_id, project_id, dataset_id, metric, day`,
|
||||
|
||||
// ── serving projection for a CURATED dataset ────────────────────────
|
||||
// ReplacingMergeTree(version) so refreshProjection is idempotent: the
|
||||
// same (tenant, dataset, entity_key) re-ingested at a higher version
|
||||
// supersedes rather than duplicates.
|
||||
`CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection (
|
||||
tenant_id LowCardinality(String),
|
||||
project_id LowCardinality(String),
|
||||
dataset_id LowCardinality(String),
|
||||
entity_key String,
|
||||
as_of DateTime64(3, 'UTC'),
|
||||
version UInt64,
|
||||
attributes Map(String, String),
|
||||
measures Map(String, Float64),
|
||||
INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4
|
||||
)
|
||||
ENGINE = ReplacingMergeTree(version)
|
||||
PARTITION BY (tenant_id)
|
||||
ORDER BY (tenant_id, project_id, dataset_id, entity_key)
|
||||
TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`,
|
||||
];
|
||||
}
|
||||
|
||||
/**
|
||||
* Statement that proves a table's ORDER BY leads with tenant_id — used by the
|
||||
* BI-P011 test suite so "tenant-first primary key" is asserted against the
|
||||
* live server, not against this file's own comments.
|
||||
*/
|
||||
export function sortingKeySql(db: string, table: string): string {
|
||||
return (
|
||||
`SELECT sorting_key FROM system.tables ` +
|
||||
`WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
|
||||
);
|
||||
}
|
||||
@@ -0,0 +1,180 @@
|
||||
/**
|
||||
* src/analytics/engine/siax-analytics-engine.ts
|
||||
* ==============================================
|
||||
* BI-P009 — the provider-neutral analytics engine interface.
|
||||
*
|
||||
* Eight capabilities, named exactly as BI-P009 lists them:
|
||||
*
|
||||
* health executeMetric executeQuery estimateQuery
|
||||
* explain cancelQuery ingestProjection refreshProjection
|
||||
*
|
||||
* ClickHouse is the CANONICAL provider (BI-P009); DuckDB (BI-P014) and Trino
|
||||
* (BI-P015) implement the same interface later. Nothing above this interface
|
||||
* may import a provider module directly — that is what makes "replaceable
|
||||
* provider" true rather than aspirational.
|
||||
*
|
||||
* NORMATIVE OWNERSHIP (BI program §0): every method takes an
|
||||
* `AnalyticsIdentity` that the CALLER cannot forge, because it is produced by
|
||||
* CL0UD verification upstream, not by request-body parsing. Product repos
|
||||
* (VehIQ) call an API in front of this interface; they never hold a provider
|
||||
* credential and never construct an engine.
|
||||
*/
|
||||
import type { CompiledQuery, MetricAggregate } from "./query-compiler";
|
||||
import type { AnalyticsTable } from "./schema";
|
||||
import type { BudgetTier, ResourceBudget } from "./resource-governance";
|
||||
|
||||
/**
|
||||
* The VERIFIED caller. `tenant_id` here is the one the engine forces into
|
||||
* every predicate — it comes from CL0UD identity verification (BI-P011 layer
|
||||
* 1), never from a request body. This estate has already been burned once by
|
||||
* a tenant read out of a request body (INF0 Wave 2, cross-tenant RAG access),
|
||||
* which is why the type is separate from the request rather than a field on
|
||||
* it.
|
||||
*/
|
||||
export interface AnalyticsIdentity {
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
/** CL0UD principal, for audit + per-principal quota. */
|
||||
principal_id: string;
|
||||
correlation_id?: string;
|
||||
/** Budget tier granted by CL0UD entitlements (BI-P012). */
|
||||
budget_tier?: BudgetTier;
|
||||
/** CL0UD quota, which may only TIGHTEN the tier. */
|
||||
quota?: Partial<ResourceBudget>;
|
||||
}
|
||||
|
||||
export interface MetricRequest {
|
||||
metric: string;
|
||||
aggregate: MetricAggregate;
|
||||
dataset_id?: string;
|
||||
group_by?: string[];
|
||||
filters?: Record<string, string>;
|
||||
from_time?: string;
|
||||
to_time?: string;
|
||||
limit?: number;
|
||||
table?: AnalyticsTable;
|
||||
}
|
||||
|
||||
export interface AnalyticsRow {
|
||||
/** Resolved group values, keyed by the CALLER-FACING dimension name. */
|
||||
groups?: Record<string, string>;
|
||||
value: number;
|
||||
}
|
||||
|
||||
export interface EngineResult {
|
||||
rows: AnalyticsRow[];
|
||||
/** Engine-side id; the handle `cancelQuery` needs. */
|
||||
query_id: string;
|
||||
/** True when the governance ceiling cut the result short (BI-P012). */
|
||||
truncated: boolean;
|
||||
stats: QueryStats;
|
||||
}
|
||||
|
||||
export interface QueryStats {
|
||||
duration_ms: number;
|
||||
rows_read?: number;
|
||||
bytes_read?: number;
|
||||
result_rows: number;
|
||||
}
|
||||
|
||||
export interface QueryEstimate {
|
||||
/** Rows the engine expects to SCAN. */
|
||||
estimated_rows_to_read: number;
|
||||
estimated_bytes_to_read: number;
|
||||
/** False when the estimate exceeds the caller's budget (BI-P012). */
|
||||
within_budget: boolean;
|
||||
budget: ResourceBudget;
|
||||
/** Why it was refused, when `within_budget` is false. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface QueryExplanation {
|
||||
/** Provider's plan text. Never contains caller-supplied SQL fragments. */
|
||||
plan: string;
|
||||
/** The predicate the compiler forced — assertable, not narrated. */
|
||||
forced_tenant_predicate: string;
|
||||
table: string;
|
||||
}
|
||||
|
||||
export interface ProjectionRow {
|
||||
entity_key: string;
|
||||
as_of?: string;
|
||||
version?: number;
|
||||
attributes?: Record<string, string>;
|
||||
measures?: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface ProjectionIngestRequest {
|
||||
dataset_id: string;
|
||||
rows: ProjectionRow[];
|
||||
}
|
||||
|
||||
export interface ProjectionRefreshResult {
|
||||
dataset_id: string;
|
||||
/** Rows visible after the refresh settled. */
|
||||
row_count: number;
|
||||
refreshed_at: string;
|
||||
}
|
||||
|
||||
export interface HealthReport {
|
||||
healthy: boolean;
|
||||
provider: string;
|
||||
/** Provider version when reachable. */
|
||||
version?: string;
|
||||
/** Short operator-facing reason. Never a raw engine body. */
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* A raw, ANALYST-facing query (BI-P017 will govern its SQL dialect). It is a
|
||||
* SEPARATE method from `executeMetric` on purpose: `executeMetric` has no
|
||||
* raw-SQL surface at all, and keeping them apart means the closed DSL cannot
|
||||
* silently grow one.
|
||||
*
|
||||
* The canonical ClickHouse provider in this pass DELIBERATELY refuses this
|
||||
* method (`CAPABILITY_UNAVAILABLE`): opening an analyst SQL surface before
|
||||
* BI-P017's dialect contract and BI-P033's authorization exist would be
|
||||
* exactly the "raw SQL attempt" BI-P011's adversarial list names. Declaring
|
||||
* it and refusing it is honest; implementing it early is not.
|
||||
*/
|
||||
export interface AnalystQueryRequest {
|
||||
sql: string;
|
||||
dataset_ids: string[];
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface SiaxAnalyticsEngine {
|
||||
readonly provider: string;
|
||||
|
||||
health(): Promise<HealthReport>;
|
||||
|
||||
executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise<EngineResult>;
|
||||
|
||||
executeQuery(identity: AnalyticsIdentity, request: AnalystQueryRequest): Promise<EngineResult>;
|
||||
|
||||
estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryEstimate>;
|
||||
|
||||
cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise<boolean>;
|
||||
|
||||
explain(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryExplanation>;
|
||||
|
||||
ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>;
|
||||
|
||||
refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise<ProjectionRefreshResult>;
|
||||
}
|
||||
|
||||
/** The eight capability names, for the provider contract test. */
|
||||
export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [
|
||||
"health",
|
||||
"executeMetric",
|
||||
"executeQuery",
|
||||
"estimateQuery",
|
||||
"cancelQuery",
|
||||
"explain",
|
||||
"ingestProjection",
|
||||
"refreshProjection",
|
||||
] as const;
|
||||
export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number];
|
||||
|
||||
/** Re-exported so provider modules do not reach across into the compiler. */
|
||||
export type { CompiledQuery };
|
||||
@@ -0,0 +1,183 @@
|
||||
/**
|
||||
* src/analytics/execution/cancellation.ts
|
||||
* ========================================
|
||||
* BI-P019 — QUERY CANCELLATION.
|
||||
*
|
||||
* "Integrate engine cancellation. DoD: User cancellation stops expensive
|
||||
* query within defined timeout."
|
||||
*
|
||||
* THE THING THAT MAKES THIS REAL: "stops" is VERIFIED, not requested.
|
||||
*
|
||||
* The obvious implementation aborts the HTTP request and issues a KILL. Both
|
||||
* of those are requests. The client abort stops US waiting; it does not stop
|
||||
* the server, which keeps burning CPU on a query nobody will read. `KILL QUERY`
|
||||
* without `SYNC` returns immediately and tells you nothing. Even `KILL QUERY
|
||||
* SYNC` can return while the query is still winding down.
|
||||
*
|
||||
* So `cancelAndVerify` does three things and reports on all three:
|
||||
* 1. abort the client side;
|
||||
* 2. issue the engine's kill;
|
||||
* 3. POLL the engine's process list until the query is GONE, or the deadline
|
||||
* passes — and if the deadline passes, it says so with
|
||||
* `verified_stopped: false` rather than returning a cheerful `true`.
|
||||
*
|
||||
* A caller that treats `cancelQuery()` returning `true` as "the query stopped"
|
||||
* is the failure this module exists to prevent, which is why the return type
|
||||
* is an outcome object and not a boolean.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
|
||||
/** Whatever can list and kill running queries. ClickHouse and Trino both can. */
|
||||
export interface EngineProcessControl {
|
||||
readonly engine: string;
|
||||
/** True when a query with this id is still running on the engine. */
|
||||
isRunning(queryId: string): Promise<boolean>;
|
||||
/** Ask the engine to stop it. Returning true means "accepted", not "stopped". */
|
||||
requestKill(queryId: string): Promise<boolean>;
|
||||
/** Live progress for the running query, when the engine exposes it (BI-P021). */
|
||||
progress?(queryId: string): Promise<QueryProgress | undefined>;
|
||||
}
|
||||
|
||||
export interface QueryProgress {
|
||||
rows_read: number;
|
||||
bytes_read: number;
|
||||
elapsed_ms: number;
|
||||
}
|
||||
|
||||
export interface CancellationOutcome {
|
||||
query_id: string;
|
||||
requested_at: string;
|
||||
/** The engine accepted the kill request. */
|
||||
kill_accepted: boolean;
|
||||
/**
|
||||
* The query was OBSERVED to be gone from the engine's process list.
|
||||
* `false` means it may still be running — the honest answer.
|
||||
*/
|
||||
verified_stopped: boolean;
|
||||
/** Milliseconds from the cancel request to the observation. */
|
||||
elapsed_ms: number;
|
||||
/** The deadline this cancellation was held to. */
|
||||
timeout_ms: number;
|
||||
/** Present when the query had already finished before the request landed. */
|
||||
already_finished: boolean;
|
||||
}
|
||||
|
||||
export interface CancellationOptions {
|
||||
/** The defined timeout BI-P019's DoD refers to. */
|
||||
timeoutMs?: number;
|
||||
pollIntervalMs?: number;
|
||||
/** Aborts the client-side wait. */
|
||||
abort?: () => void;
|
||||
now?: () => number;
|
||||
sleep?: (ms: number) => Promise<void>;
|
||||
}
|
||||
|
||||
export const DEFAULT_CANCELLATION_TIMEOUT_MS = 5_000;
|
||||
const DEFAULT_POLL_INTERVAL_MS = 100;
|
||||
|
||||
const defaultSleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
|
||||
|
||||
export async function cancelAndVerify(
|
||||
control: EngineProcessControl,
|
||||
queryId: string,
|
||||
opts: CancellationOptions = {},
|
||||
): Promise<CancellationOutcome> {
|
||||
if (!queryId?.trim()) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'cancellation requires a query id');
|
||||
}
|
||||
const timeoutMs = opts.timeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS;
|
||||
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
|
||||
const now = opts.now ?? Date.now;
|
||||
const sleep = opts.sleep ?? defaultSleep;
|
||||
const startedMs = now();
|
||||
const requestedAt = new Date(startedMs).toISOString();
|
||||
|
||||
// Was it even running? Cancelling a finished query is a no-op, and reporting
|
||||
// it as a successful stop would overstate what happened.
|
||||
//
|
||||
// FAIL CLOSED when the engine cannot be asked: an unreachable engine means
|
||||
// "unknown", and the safe reading of unknown is "still running" — so the
|
||||
// kill is still issued and the verification loop still has to observe the
|
||||
// query gone. Letting this throw would abandon the cancellation entirely at
|
||||
// the exact moment the operator most needs it to be attempted.
|
||||
let runningBefore: boolean;
|
||||
try {
|
||||
runningBefore = await control.isRunning(queryId);
|
||||
} catch {
|
||||
runningBefore = true;
|
||||
}
|
||||
|
||||
// 1. client side: stop waiting.
|
||||
opts.abort?.();
|
||||
|
||||
// 2. engine side: ask it to stop.
|
||||
let killAccepted = false;
|
||||
try {
|
||||
killAccepted = await control.requestKill(queryId);
|
||||
} catch {
|
||||
killAccepted = false;
|
||||
}
|
||||
|
||||
if (!runningBefore) {
|
||||
return {
|
||||
query_id: queryId,
|
||||
requested_at: requestedAt,
|
||||
kill_accepted: killAccepted,
|
||||
verified_stopped: true,
|
||||
elapsed_ms: now() - startedMs,
|
||||
timeout_ms: timeoutMs,
|
||||
already_finished: true,
|
||||
};
|
||||
}
|
||||
|
||||
// 3. VERIFY. This loop is the whole point of the module.
|
||||
let verified = false;
|
||||
for (;;) {
|
||||
let stillRunning: boolean;
|
||||
try {
|
||||
stillRunning = await control.isRunning(queryId);
|
||||
} catch {
|
||||
// An engine we cannot ask is an engine we cannot vouch for. Keep trying
|
||||
// until the deadline, then report unverified.
|
||||
stillRunning = true;
|
||||
}
|
||||
if (!stillRunning) {
|
||||
verified = true;
|
||||
break;
|
||||
}
|
||||
if (now() - startedMs >= timeoutMs) break;
|
||||
await sleep(pollIntervalMs);
|
||||
}
|
||||
|
||||
return {
|
||||
query_id: queryId,
|
||||
requested_at: requestedAt,
|
||||
kill_accepted: killAccepted,
|
||||
verified_stopped: verified,
|
||||
elapsed_ms: now() - startedMs,
|
||||
timeout_ms: timeoutMs,
|
||||
already_finished: false,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The same call, for a caller that wants a failure rather than a report.
|
||||
* Throws `RETRYABLE_FAILURE` when the query outlived the deadline — because
|
||||
* "we asked it to stop and it did not" is an operational condition someone
|
||||
* has to see, not a value to ignore.
|
||||
*/
|
||||
export async function cancelOrThrow(
|
||||
control: EngineProcessControl,
|
||||
queryId: string,
|
||||
opts: CancellationOptions = {},
|
||||
): Promise<CancellationOutcome> {
|
||||
const outcome = await cancelAndVerify(control, queryId, opts);
|
||||
if (!outcome.verified_stopped) {
|
||||
throw new SiaxError(
|
||||
'RETRYABLE_FAILURE',
|
||||
`0pt.analytics.${control.engine}`,
|
||||
`query was not observed to stop within ${outcome.timeout_ms}ms of cancellation`,
|
||||
);
|
||||
}
|
||||
return outcome;
|
||||
}
|
||||
@@ -0,0 +1,138 @@
|
||||
/**
|
||||
* src/analytics/execution/clickhouse-process-control.ts
|
||||
* ======================================================
|
||||
* BI-P019/BI-P021 — the ClickHouse implementation of `EngineProcessControl`.
|
||||
*
|
||||
* Three statements, each chosen for a measured reason:
|
||||
*
|
||||
* isRunning `SELECT count() FROM system.processes WHERE query_id = …`
|
||||
* — the authority on whether the server is still working. Note
|
||||
* that `KILL QUERY SYNC` returning is NOT this: a killed query
|
||||
* can still appear here briefly while it unwinds, which is
|
||||
* exactly why cancellation polls instead of trusting the kill.
|
||||
*
|
||||
* requestKill `KILL QUERY WHERE query_id = … ASYNC` — ASYNC, not SYNC,
|
||||
* because the verification loop is what establishes the stop and
|
||||
* a SYNC kill would block this call for up to the query's
|
||||
* remaining lifetime while providing no stronger guarantee.
|
||||
*
|
||||
* progress `read_rows`, `read_bytes` and `elapsed` from the same table —
|
||||
* live counters, which is what makes BI-P021's in-flight budget
|
||||
* enforcement possible at all.
|
||||
*
|
||||
* KILL and the system-table reads are ADMINISTRATIVE. They run on the engine's
|
||||
* own credential, never on the tenant's restricted user (which has no access
|
||||
* to system.processes — measured: ACCESS_DENIED, code 497). The tenant-scoping
|
||||
* check therefore happens HERE, on the query id, before any statement is sent.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { EngineProcessControl, QueryProgress } from './cancellation';
|
||||
|
||||
export interface ClickHouseProcessControlOptions {
|
||||
baseUrl: string;
|
||||
/** "user:password" for the ADMIN credential. From a secret ref. */
|
||||
basicAuth?: string;
|
||||
timeoutMs?: number;
|
||||
/**
|
||||
* Restricts every operation to query ids carrying this prefix, so a caller
|
||||
* cannot kill or inspect another tenant's work through this object.
|
||||
*/
|
||||
queryIdPrefix?: string;
|
||||
}
|
||||
|
||||
export class ClickHouseProcessControl implements EngineProcessControl {
|
||||
readonly engine = 'clickhouse';
|
||||
private readonly baseUrl: string;
|
||||
private readonly authHeader?: string;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly prefix?: string;
|
||||
|
||||
constructor(opts: ClickHouseProcessControlOptions) {
|
||||
this.baseUrl = opts.baseUrl.replace(/\/+$/, '');
|
||||
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined;
|
||||
this.timeoutMs = opts.timeoutMs ?? 10_000;
|
||||
this.prefix = opts.queryIdPrefix;
|
||||
}
|
||||
|
||||
private assertScoped(queryId: string): void {
|
||||
if (this.prefix && !queryId.startsWith(this.prefix)) {
|
||||
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this scope');
|
||||
}
|
||||
if (!/^[A-Za-z0-9_-]{1,128}$/.test(queryId)) {
|
||||
// The id reaches ClickHouse as a bound parameter, but a malformed id is
|
||||
// a bug worth surfacing rather than a string worth binding.
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'malformed query id');
|
||||
}
|
||||
}
|
||||
|
||||
async isRunning(queryId: string): Promise<boolean> {
|
||||
this.assertScoped(queryId);
|
||||
const body = await this.send(
|
||||
'SELECT count() AS n FROM system.processes WHERE query_id = {qid:String} FORMAT JSON',
|
||||
{ param_qid: queryId },
|
||||
);
|
||||
const n = Number((JSON.parse(body) as { data: Array<{ n: string | number }> }).data[0]?.n ?? 0);
|
||||
return n > 0;
|
||||
}
|
||||
|
||||
async requestKill(queryId: string): Promise<boolean> {
|
||||
this.assertScoped(queryId);
|
||||
await this.send(
|
||||
'KILL QUERY WHERE query_id = {qid:String} ASYNC FORMAT JSON',
|
||||
{ param_qid: queryId },
|
||||
false,
|
||||
);
|
||||
return true;
|
||||
}
|
||||
|
||||
async progress(queryId: string): Promise<QueryProgress | undefined> {
|
||||
this.assertScoped(queryId);
|
||||
const body = await this.send(
|
||||
'SELECT read_rows, read_bytes, toUInt64(elapsed * 1000) AS elapsed_ms FROM system.processes ' +
|
||||
'WHERE query_id = {qid:String} FORMAT JSON',
|
||||
{ param_qid: queryId },
|
||||
);
|
||||
const row = (JSON.parse(body) as { data: Array<Record<string, string | number>> }).data[0];
|
||||
if (!row) return undefined;
|
||||
return {
|
||||
rows_read: Number(row.read_rows ?? 0),
|
||||
bytes_read: Number(row.read_bytes ?? 0),
|
||||
elapsed_ms: Number(row.elapsed_ms ?? 0),
|
||||
};
|
||||
}
|
||||
|
||||
private async send(sql: string, params: Record<string, string>, expectJson = true): Promise<string> {
|
||||
const url = new URL(this.baseUrl);
|
||||
url.searchParams.set('query', sql);
|
||||
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
||||
// NOT readonly: KILL mutates server state. The other two would tolerate
|
||||
// readonly, but sharing one path keeps the credential story simple.
|
||||
url.searchParams.set('max_execution_time', '10');
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.authHeader) headers.Authorization = this.authHeader;
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url.toString(), {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: '',
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
|
||||
}
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status);
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'clickhouse',
|
||||
`process-control statement failed (CH code ${code})`,
|
||||
);
|
||||
}
|
||||
if (expectJson && !text.trim()) {
|
||||
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'empty response from process-control statement');
|
||||
}
|
||||
return text;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,274 @@
|
||||
/**
|
||||
* src/analytics/execution/query-execution-record.ts
|
||||
* ==================================================
|
||||
* BI-P018 — QUERY EXECUTION MODEL.
|
||||
*
|
||||
* Every field BI-P018 lists, verbatim, and one it lists by its absence:
|
||||
*
|
||||
* "Every production query has traceable execution metadata WITHOUT
|
||||
* unnecessarily storing sensitive raw SQL."
|
||||
*
|
||||
* So the record carries `compiled_query_hash` and never the statement. That is
|
||||
* enforced, not documented: `assertNoRawSql` scans a finished record for the
|
||||
* statement it describes, and
|
||||
* tests/analytics/query-execution-model.test.ts fails if any field carries it.
|
||||
*
|
||||
* The ONE place raw SQL is retained is the analyst statement history
|
||||
* (BI-P017), where a privileged human ran unreviewed SQL against tenant data
|
||||
* and "who ran what" has to survive. That history is a separate record with a
|
||||
* separate retention posture; this one is the per-execution telemetry that
|
||||
* every query in the estate produces, and it must be safe to keep for a long
|
||||
* time and to ship to AUD0.
|
||||
*
|
||||
* BI-P002's `QueryExecution` entity is the persisted row; this module is the
|
||||
* lifecycle that produces it, plus the four fields BI-P018 adds to it
|
||||
* (compiled_query_hash, datasets, metrics, cache_status, cost_units).
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { ExecutionState } from '../domain/entities';
|
||||
|
||||
/** BI-P020's outcome, recorded per execution so cache behaviour is measurable. */
|
||||
export const CACHE_STATUSES = ['hit', 'miss', 'bypass', 'stale_refused', 'not_cacheable'] as const;
|
||||
export type CacheStatus = (typeof CACHE_STATUSES)[number];
|
||||
|
||||
/** Which surface produced the query — the "user/workload" axis BI-P018 names. */
|
||||
export const WORKLOAD_KINDS = [
|
||||
'dashboard',
|
||||
'report',
|
||||
'api',
|
||||
'analyst_sql',
|
||||
'ai_analyst',
|
||||
'scheduled',
|
||||
'internal',
|
||||
] as const;
|
||||
export type WorkloadKind = (typeof WORKLOAD_KINDS)[number];
|
||||
|
||||
export interface QueryExecutionRecord {
|
||||
query_id: string;
|
||||
execution_id: string;
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
/** The CL0UD principal. A human, a service, or a job — never absent. */
|
||||
principal_id: string;
|
||||
workload: WorkloadKind;
|
||||
submitted_at: string;
|
||||
started_at?: string;
|
||||
completed_at?: string;
|
||||
engine: string;
|
||||
/**
|
||||
* sha256 of the COMPILED statement plus its bound parameter values.
|
||||
*
|
||||
* Parameters are included deliberately: two executions of the same SQL with
|
||||
* different filters are different work with different cost, and a hash that
|
||||
* collapsed them would make the cost record meaningless. They are HASHED,
|
||||
* not stored, so the record still carries no tenant data.
|
||||
*/
|
||||
compiled_query_hash: string;
|
||||
datasets: string[];
|
||||
metrics: string[];
|
||||
status: ExecutionState;
|
||||
/** Rows RETURNED to the caller. */
|
||||
rows?: number;
|
||||
/** Rows and bytes SCANNED — the cost axis, not the result size. */
|
||||
rows_scanned?: number;
|
||||
bytes_scanned?: number;
|
||||
duration_ms?: number;
|
||||
cache_status: CacheStatus;
|
||||
/** BI-P021 units. Present once the query has settled, including on failure. */
|
||||
cost_units?: number;
|
||||
/** A SiaxErrorCode. Never an engine body, never a fragment of the statement. */
|
||||
error?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Hash a compiled statement and its bound parameters into the one value the
|
||||
* record keeps.
|
||||
*
|
||||
* Parameters are sorted so two logically identical executions hash equal
|
||||
* regardless of key iteration order — otherwise the cache-key and cost
|
||||
* aggregations built on this hash would fragment for no reason.
|
||||
*/
|
||||
export function compiledQueryHash(sql: string, params: Record<string, string> = {}): string {
|
||||
const canonical = JSON.stringify([
|
||||
sql.trim(),
|
||||
Object.keys(params)
|
||||
.sort()
|
||||
.map((k) => [k, params[k]]),
|
||||
]);
|
||||
return crypto.createHash('sha256').update(canonical).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* The legal state machine. Written down because "status" fields drift into
|
||||
* meaninglessness when any value can follow any other — a `succeeded` row that
|
||||
* later becomes `running` makes every downstream aggregation a lie.
|
||||
*/
|
||||
const LEGAL_TRANSITIONS: Record<ExecutionState, readonly ExecutionState[]> = {
|
||||
queued: ['running', 'cancelled', 'rejected', 'failed'],
|
||||
running: ['succeeded', 'failed', 'cancelled'],
|
||||
succeeded: [],
|
||||
failed: [],
|
||||
cancelled: [],
|
||||
rejected: [],
|
||||
};
|
||||
|
||||
export function assertLegalTransition(from: ExecutionState, to: ExecutionState): void {
|
||||
if (!LEGAL_TRANSITIONS[from].includes(to)) {
|
||||
throw new SiaxError('CONFLICT', '0pt.analytics', `illegal execution transition ${from} -> ${to}`);
|
||||
}
|
||||
}
|
||||
|
||||
/** Terminal states: a record in one of these will never change again. */
|
||||
export const TERMINAL_STATES: readonly ExecutionState[] = ['succeeded', 'failed', 'cancelled', 'rejected'];
|
||||
|
||||
export interface ExecutionSink {
|
||||
write(record: QueryExecutionRecord): Promise<void> | void;
|
||||
}
|
||||
|
||||
export interface ExecutionRecorderOptions {
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
principal_id: string;
|
||||
workload: WorkloadKind;
|
||||
engine: string;
|
||||
correlation_id?: string;
|
||||
sink: ExecutionSink;
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
/**
|
||||
* Owns one execution's record from submission to a terminal state.
|
||||
*
|
||||
* Every transition WRITES. A recorder that only writes at the end loses
|
||||
* exactly the executions that matter: the ones that never finished.
|
||||
*/
|
||||
export class QueryExecutionRecorder {
|
||||
private record: QueryExecutionRecord;
|
||||
private readonly sink: ExecutionSink;
|
||||
private readonly now: () => number;
|
||||
|
||||
constructor(
|
||||
opts: ExecutionRecorderOptions,
|
||||
queryId: string,
|
||||
compiled: { hash: string; datasets: string[]; metrics: string[] },
|
||||
) {
|
||||
if (!opts.tenant_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics',
|
||||
'an execution record requires a verified tenant',
|
||||
);
|
||||
}
|
||||
if (!opts.principal_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics',
|
||||
'an execution record requires a verified principal',
|
||||
);
|
||||
}
|
||||
this.now = opts.now ?? Date.now;
|
||||
this.sink = opts.sink;
|
||||
this.record = {
|
||||
query_id: queryId,
|
||||
execution_id: crypto.randomUUID(),
|
||||
tenant_id: opts.tenant_id,
|
||||
project_id: opts.project_id,
|
||||
principal_id: opts.principal_id,
|
||||
workload: opts.workload,
|
||||
submitted_at: new Date(this.now()).toISOString(),
|
||||
engine: opts.engine,
|
||||
compiled_query_hash: compiled.hash,
|
||||
datasets: [...compiled.datasets],
|
||||
metrics: [...compiled.metrics],
|
||||
status: 'queued',
|
||||
cache_status: 'miss',
|
||||
correlation_id: opts.correlation_id,
|
||||
};
|
||||
}
|
||||
|
||||
get current(): QueryExecutionRecord {
|
||||
return { ...this.record, datasets: [...this.record.datasets], metrics: [...this.record.metrics] };
|
||||
}
|
||||
|
||||
async submitted(): Promise<void> {
|
||||
await this.sink.write(this.current);
|
||||
}
|
||||
|
||||
async started(): Promise<void> {
|
||||
assertLegalTransition(this.record.status, 'running');
|
||||
this.record.status = 'running';
|
||||
this.record.started_at = new Date(this.now()).toISOString();
|
||||
await this.sink.write(this.current);
|
||||
}
|
||||
|
||||
/** A cache hit never reached the engine; record it as such, not as a fast query. */
|
||||
async servedFromCache(rows: number, costUnits = 0): Promise<void> {
|
||||
assertLegalTransition(this.record.status, 'running');
|
||||
this.record.status = 'running';
|
||||
this.record.started_at = this.record.started_at ?? new Date(this.now()).toISOString();
|
||||
this.record.cache_status = 'hit';
|
||||
await this.finish('succeeded', { rows, rows_scanned: 0, bytes_scanned: 0, cost_units: costUnits });
|
||||
}
|
||||
|
||||
setCacheStatus(status: CacheStatus): void {
|
||||
this.record.cache_status = status;
|
||||
}
|
||||
|
||||
async finish(
|
||||
status: ExecutionState,
|
||||
stats: {
|
||||
rows?: number;
|
||||
rows_scanned?: number;
|
||||
bytes_scanned?: number;
|
||||
cost_units?: number;
|
||||
error?: string;
|
||||
},
|
||||
): Promise<QueryExecutionRecord> {
|
||||
assertLegalTransition(this.record.status, status);
|
||||
const completedMs = this.now();
|
||||
this.record.status = status;
|
||||
this.record.completed_at = new Date(completedMs).toISOString();
|
||||
const startedMs = this.record.started_at
|
||||
? Date.parse(this.record.started_at)
|
||||
: Date.parse(this.record.submitted_at);
|
||||
this.record.duration_ms = Math.max(0, completedMs - startedMs);
|
||||
if (stats.rows !== undefined) this.record.rows = stats.rows;
|
||||
if (stats.rows_scanned !== undefined) this.record.rows_scanned = stats.rows_scanned;
|
||||
if (stats.bytes_scanned !== undefined) this.record.bytes_scanned = stats.bytes_scanned;
|
||||
if (stats.cost_units !== undefined) this.record.cost_units = stats.cost_units;
|
||||
if (stats.error !== undefined) this.record.error = stats.error;
|
||||
assertNoRawSql(this.record);
|
||||
await this.sink.write(this.current);
|
||||
return this.current;
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Refuse to emit a record that carries statement text.
|
||||
*
|
||||
* Checked at runtime rather than trusted to the type, because the fields that
|
||||
* leak are the string ones a future change widens — `error` becoming an engine
|
||||
* body is the obvious one, and ClickHouse's error text echoes the query.
|
||||
*/
|
||||
export function assertNoRawSql(record: QueryExecutionRecord): void {
|
||||
const suspicious = /\b(select|insert|update|delete|drop|create|alter|from|where|join|union)\b/i;
|
||||
const stringFields: Array<[string, string | undefined]> = [
|
||||
['error', record.error],
|
||||
['query_id', record.query_id],
|
||||
['compiled_query_hash', record.compiled_query_hash],
|
||||
['correlation_id', record.correlation_id],
|
||||
...record.datasets.map((d, i) => [`datasets[${i}]`, d] as [string, string]),
|
||||
...record.metrics.map((m, i) => [`metrics[${i}]`, m] as [string, string]),
|
||||
];
|
||||
for (const [name, value] of stringFields) {
|
||||
if (value && suspicious.test(value)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics',
|
||||
`execution record field "${name}" looks like SQL; BI-P018 stores a hash, never the statement`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,143 @@
|
||||
/**
|
||||
* src/analytics/ingestion/analytics-ingestion-service.ts
|
||||
* =======================================================
|
||||
* BI-P129: one entry point that takes an `AnalyticsIngestionSpec` and lands
|
||||
* rows in a canonical analytics tier, whichever provider supplies the
|
||||
* connector.
|
||||
*
|
||||
* The sequence, and why it is this sequence:
|
||||
*
|
||||
* 1. plan — every gate checkable without touching the source
|
||||
* (tier, physical home, licence, provider readiness).
|
||||
* Cheap checks first: a spec that can never be legal
|
||||
* should not open a session to find that out.
|
||||
* 2. connect+discover— ask the source what it actually looks like.
|
||||
* 3. landability — the discovered shape must satisfy the BI-P004
|
||||
* contract. This is the check that cannot be done at
|
||||
* step 1, and doing it here is the whole reason the
|
||||
* service exists rather than a bare `planX()` call.
|
||||
* 4. run — hand the connector to the EXISTING P062 runner with
|
||||
* a landing sink. The runner owns paging, envelopes,
|
||||
* schema conformance, quality gating, the idempotency
|
||||
* ledger and the durable checkpoint. None of that is
|
||||
* re-implemented here.
|
||||
* 5. disconnect — always, including on failure.
|
||||
*
|
||||
* Step 4 is worth spelling out because it is where a "new ingestion system"
|
||||
* would normally get built by accident. There is no second runner. The
|
||||
* analytics plane contributes a TARGET and four GATES; the highway is the one
|
||||
* that already exists and already has crash/resume tests.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import { IngestionRunner, type IngestionRunReport } from '../../ingestion/ingestion-runner';
|
||||
import type { IngestionRepository } from '../../repositories/ingestion-repository';
|
||||
import type { SchemaRegistry } from '../../schema/schema-registry';
|
||||
import type { LineageStore } from '../../lineage/lineage-store';
|
||||
import type { QualityEnforcementPolicy } from '../../quality/quality-policy';
|
||||
import type { RecordSchemaFields } from '../../schema/record-schema';
|
||||
import {
|
||||
planAnalyticsIngestion,
|
||||
type AnalyticsConnectorProvider,
|
||||
type AnalyticsIngestionSpec,
|
||||
type PlannedAnalyticsIngestion,
|
||||
} from './connector-provider';
|
||||
import { assertDiscoveryLandable, landingSink, type LandingWriter } from './landing';
|
||||
|
||||
export interface AnalyticsIngestionDeps {
|
||||
repo: IngestionRepository;
|
||||
schemas: SchemaRegistry;
|
||||
writer: LandingWriter;
|
||||
lineage?: LineageStore;
|
||||
/** Runtime environment name; "production" turns gate 4 on. */
|
||||
runtimeEnv: string;
|
||||
}
|
||||
|
||||
export interface AnalyticsIngestionResult {
|
||||
plan: PlannedAnalyticsIngestion;
|
||||
/** What discovery reported, per partition — recorded so a later drift is provable. */
|
||||
discovered: Record<string, RecordSchemaFields>;
|
||||
run: IngestionRunReport;
|
||||
}
|
||||
|
||||
export interface AnalyticsIngestionOptions {
|
||||
run_id?: string;
|
||||
page_size?: number;
|
||||
max_pages?: number;
|
||||
partition_key?: string;
|
||||
quality_policy?: QualityEnforcementPolicy;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
export class AnalyticsIngestionService {
|
||||
constructor(private readonly deps: AnalyticsIngestionDeps) {}
|
||||
|
||||
/** Gates 1, 3 and 4 — no IO, no session. Safe to call from a UI pre-flight. */
|
||||
plan(provider: AnalyticsConnectorProvider, spec: AnalyticsIngestionSpec): PlannedAnalyticsIngestion {
|
||||
return planAnalyticsIngestion(provider, spec, this.deps.runtimeEnv);
|
||||
}
|
||||
|
||||
async ingest(
|
||||
provider: AnalyticsConnectorProvider,
|
||||
spec: AnalyticsIngestionSpec,
|
||||
options: AnalyticsIngestionOptions = {},
|
||||
): Promise<AnalyticsIngestionResult> {
|
||||
const plan = this.plan(provider, spec);
|
||||
const runId = options.run_id ?? `run_${spec.source_id}_${Date.now()}`;
|
||||
const connector = plan.connector;
|
||||
|
||||
const discovered: Record<string, RecordSchemaFields> = {};
|
||||
await connector.connect(spec.config, {
|
||||
tenant_id: spec.tenant_id,
|
||||
project_id: spec.project_id,
|
||||
source_id: spec.source_id,
|
||||
run_id: runId,
|
||||
});
|
||||
try {
|
||||
const discovery = await connector.discover();
|
||||
for (const partition of discovery.partitions) {
|
||||
if (options.partition_key && partition.partition_key !== options.partition_key) continue;
|
||||
discovered[partition.partition_key] = partition.schema;
|
||||
}
|
||||
// Gate 2 — the one that needs the source to have answered.
|
||||
assertDiscoveryLandable(spec.target, discovered);
|
||||
} finally {
|
||||
// The runner opens its own session; this discovery session is ours to close.
|
||||
await connector.disconnect();
|
||||
}
|
||||
|
||||
const runner = new IngestionRunner(this.deps.repo, this.deps.schemas, this.deps.lineage);
|
||||
const run = await runner.run({
|
||||
source_id: spec.source_id,
|
||||
tenant_id: spec.tenant_id,
|
||||
project_id: spec.project_id,
|
||||
mode: spec.mode,
|
||||
connector,
|
||||
config: spec.config,
|
||||
sink: landingSink({
|
||||
target: spec.target,
|
||||
tenant_id: spec.tenant_id,
|
||||
project_id: spec.project_id,
|
||||
run_id: runId,
|
||||
writer: this.deps.writer,
|
||||
}),
|
||||
run_id: runId,
|
||||
page_size: options.page_size,
|
||||
max_pages: options.max_pages,
|
||||
partition_key: options.partition_key,
|
||||
quality_policy: options.quality_policy,
|
||||
output_ref: `0pt:dataset:${spec.target.dataset_id}`,
|
||||
correlation_id: options.correlation_id,
|
||||
});
|
||||
|
||||
if (run.status === 'failed') {
|
||||
throw new SiaxError(
|
||||
'RETRYABLE_FAILURE',
|
||||
'0pt.analytics.ingestion',
|
||||
`ingestion run ${run.run_id} for dataset ${spec.target.dataset_id} failed: ` +
|
||||
`${run.failure?.code ?? 'unknown'} ${run.failure?.detail ?? ''}`.trim(),
|
||||
);
|
||||
}
|
||||
|
||||
return { plan, discovered, run };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,442 @@
|
||||
/**
|
||||
* src/analytics/ingestion/connector-provider.ts
|
||||
* ==============================================
|
||||
* BI-P129 — ANALYTICS DATA INGESTION.
|
||||
*
|
||||
* BI-P129 names four provider kinds for 0pt's `ConnectorProvider`:
|
||||
*
|
||||
* native — connectors 0pt implements and owns (P061 contract)
|
||||
* dlt — the dlt Python framework (BI-P130)
|
||||
* Meltano/Singer — the Singer tap ecosystem (BI-P131)
|
||||
* Airbyte — "only optional where legal/license architecture accepts it"
|
||||
*
|
||||
* WHAT THIS FILE IS. It is the seam between 0pt's EXISTING ingestion highway
|
||||
* (`src/connectors/connector.ts` P061 + `src/ingestion/ingestion-runner.ts`
|
||||
* P062) and the ANALYTICS tier/contract model (BI-P003 tiers, BI-P004
|
||||
* `DatasetContract`). It deliberately does NOT define a second connector
|
||||
* interface: a provider's job is to hand back a P061 `Connector`, and the
|
||||
* runner that already exists then does the reading, checkpointing and
|
||||
* quality gating. A parallel ingestion stack would be exactly the "do not
|
||||
* define parallel models" failure this programme keeps paying for.
|
||||
*
|
||||
* WHAT IT ADDS, which the P061 stack has no opinion about: an ingestion run
|
||||
* must LAND somewhere, and where it lands is an analytics contract question.
|
||||
* Four gates, all fail-closed, all attacked in
|
||||
* tests/analytics/analytics-ingestion-providers.test.ts:
|
||||
*
|
||||
* 1. assertLandingTierIngestible — source data may land in RAW, or in
|
||||
* NORMALIZED with a declared normalization. It may NEVER land directly
|
||||
* in CURATED or SERVING. BI-P003's rule ("no business dashboard reads
|
||||
* arbitrary raw ingestion tables by default") is trivially defeated by
|
||||
* relabelling the raw landing table SERVING; this gate is what stops
|
||||
* that, because the tier is checked at the point of WRITE, not only at
|
||||
* the point of read.
|
||||
*
|
||||
* 2. assertContractCoversDiscovery — every field the connector actually
|
||||
* discovered must be declared in the target `DatasetContract`, with a
|
||||
* compatible type. An UNDECLARED field is refused rather than landed:
|
||||
* an undeclared column is an unclassified column, and an unclassified
|
||||
* column carrying personal data is a GDPR incident that every freshness
|
||||
* and quality check would still report green.
|
||||
*
|
||||
* 3. assertPhysicalHomeMatchesTier — a RAW/NORMALIZED landing goes to
|
||||
* ST0RE Parquet/Iceberg (the shapes st0re#45 standardised), never
|
||||
* straight into ClickHouse. Writing ingestion output into the serving
|
||||
* engine skips the lake, and with it every replay, backfill and
|
||||
* erasure path that depends on the files existing.
|
||||
*
|
||||
* 4. assertProviderRunnable — a provider whose own `providerReadiness()`
|
||||
* says `verified: false` cannot execute in a production runtime without
|
||||
* a recorded exception naming a human. This is the structural version
|
||||
* of the honesty rule `0pt#35` set with MetricFlow: an unverified
|
||||
* provider may exist, may be planned, may be tested — but it cannot
|
||||
* quietly become production capability because someone forgot it was
|
||||
* unverified.
|
||||
*
|
||||
* ARCHITECTURE BOUNDARY (BI programme §0): 0pt owns ingestion into analytics
|
||||
* tiers. A product repo declares what the rows MEAN through
|
||||
* `BIApplicationBinding`; it never runs its own ingestion into these tiers
|
||||
* and never receives ClickHouse credentials.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { Connector, ConnectorConfig, ValidationResult } from '../../connectors/connector';
|
||||
import type { RecordSchemaFields, FieldType } from '../../schema/record-schema';
|
||||
import type { IngestionMode } from '../../types/ingestion';
|
||||
import type { AnalyticsFieldType } from '../domain/entities';
|
||||
import type { AnalyticsTier } from '../domain/tiers';
|
||||
import type { DatasetContract } from '../domain/contract';
|
||||
import type { ProviderReadiness } from '../metrics/semantic-provider';
|
||||
import type { N0dRequiredRole } from '../../clients/n0d-client';
|
||||
|
||||
/**
|
||||
* Deliberately the SAME `ProviderReadiness` shape BI-P029 introduced, not a
|
||||
* second one. An estate with two readiness vocabularies has none.
|
||||
*/
|
||||
export type { ProviderReadiness };
|
||||
|
||||
// ── provider kinds ─────────────────────────────────────────────────────────
|
||||
|
||||
export const CONNECTOR_PROVIDER_KINDS = ['native', 'dlt', 'singer', 'airbyte'] as const;
|
||||
export type ConnectorProviderKind = (typeof CONNECTOR_PROVIDER_KINDS)[number];
|
||||
|
||||
/**
|
||||
* BI-P129: "Airbyte only optional where legal/license architecture accepts
|
||||
* it." That acceptance has NOT been given in this estate — Airbyte's server
|
||||
* is Elastic License 2.0 (ELv2), which restricts offering the software as a
|
||||
* managed service to third parties, and 0pt's whole purpose is offering
|
||||
* managed data capability to customers. So `airbyte` is a declared kind with
|
||||
* no implementation, and registering one is refused here rather than
|
||||
* somewhere downstream. When a licence review says otherwise, this constant
|
||||
* is the single place that changes.
|
||||
*/
|
||||
export const PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE: readonly ConnectorProviderKind[] = ['airbyte'];
|
||||
|
||||
// ── landing target ─────────────────────────────────────────────────────────
|
||||
|
||||
/** Physical formats an analytics landing may use. st0re#45's shapes. */
|
||||
export const LANDING_FORMATS = ['parquet', 'iceberg'] as const;
|
||||
export type LandingFormat = (typeof LANDING_FORMATS)[number];
|
||||
|
||||
/** Where an ingestion run's output physically goes. */
|
||||
export interface LandingPhysical {
|
||||
format: LandingFormat;
|
||||
/** ST0RE lakehouse namespace. */
|
||||
namespace: string;
|
||||
table_name: string;
|
||||
}
|
||||
|
||||
export interface AnalyticsLandingTarget {
|
||||
dataset_id: string;
|
||||
tier: AnalyticsTier;
|
||||
/** The BI-P004 contract the landed rows must satisfy. */
|
||||
contract: DatasetContract;
|
||||
physical: LandingPhysical;
|
||||
/**
|
||||
* Required when `tier` is NORMALIZED: what normalization was applied
|
||||
* between the source bytes and the landed rows. Free text, recorded — an
|
||||
* empty or missing value means the dataset is really RAW wearing a
|
||||
* NORMALIZED label, and is refused.
|
||||
*/
|
||||
normalization?: string;
|
||||
}
|
||||
|
||||
/** One ingestion the analytics plane is asked to perform. */
|
||||
export interface AnalyticsIngestionSpec {
|
||||
source_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
mode: IngestionMode;
|
||||
provider: ConnectorProviderKind;
|
||||
/** Provider-specific connector selector (a native connector id, a tap name, a dlt source). */
|
||||
connector_ref: string;
|
||||
config: ConnectorConfig;
|
||||
target: AnalyticsLandingTarget;
|
||||
/**
|
||||
* Records the human who accepted running an UNVERIFIED provider in a
|
||||
* production runtime. Absent = an unverified provider is refused.
|
||||
*/
|
||||
unverified_provider_exception?: string;
|
||||
}
|
||||
|
||||
// ── execution plan ─────────────────────────────────────────────────────────
|
||||
|
||||
export type ProviderRuntime = 'in_process' | 'n0d';
|
||||
|
||||
/**
|
||||
* Where a provider's work actually runs. BI-P130 says dlt executes on N0D;
|
||||
* this is the type that carries that decision instead of a comment.
|
||||
*/
|
||||
export interface ProviderExecutionPlan {
|
||||
runtime: ProviderRuntime;
|
||||
/** N0D job_type; set iff runtime is "n0d". */
|
||||
job_type?: string;
|
||||
required_role?: N0dRequiredRole;
|
||||
/** Named profile from N0D_RESOURCE_PROFILES; set iff runtime is "n0d". */
|
||||
resource_profile?: string;
|
||||
/** Operator-facing reason, in plain words. */
|
||||
reason: string;
|
||||
}
|
||||
|
||||
// ── the port ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface AnalyticsConnectorProvider {
|
||||
readonly provider: ConnectorProviderKind;
|
||||
/** What this provider has and has NOT demonstrated. Never optimistic. */
|
||||
providerReadiness(): ProviderReadiness;
|
||||
/** Pure, no IO, never throws — the P061 `validate` posture, at spec level. */
|
||||
validateSpec(spec: AnalyticsIngestionSpec): ValidationResult;
|
||||
/** The P061 connector this spec ingests through. */
|
||||
connectorFor(spec: AnalyticsIngestionSpec): Connector;
|
||||
executionPlan(spec: AnalyticsIngestionSpec): ProviderExecutionPlan;
|
||||
}
|
||||
|
||||
// ── gate 1: landing tier ───────────────────────────────────────────────────
|
||||
|
||||
/** Tiers an ingestion run may write into at all. */
|
||||
export const INGESTIBLE_TIERS: readonly AnalyticsTier[] = ['RAW', 'NORMALIZED'];
|
||||
|
||||
export function assertLandingTierIngestible(target: AnalyticsLandingTarget): void {
|
||||
if (!INGESTIBLE_TIERS.includes(target.tier)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`ingestion may not land directly in ${target.tier} (dataset ${target.dataset_id}): ` +
|
||||
`CURATED and SERVING are DERIVED tiers, built from a landed RAW/NORMALIZED dataset. ` +
|
||||
`Landing source data there relabels raw ingestion as dashboard-readable and defeats BI-P003.`,
|
||||
);
|
||||
}
|
||||
if (target.tier === 'NORMALIZED' && !(target.normalization ?? '').trim()) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`dataset ${target.dataset_id} lands in NORMALIZED but declares no normalization — ` +
|
||||
`that is a RAW dataset wearing a NORMALIZED label (BI-P003)`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── gate 2: contract covers what was discovered ────────────────────────────
|
||||
|
||||
/**
|
||||
* How a connector's wire-level field type maps onto the analytics type
|
||||
* vocabulary. Conservative on purpose: a connector's `number` may be an
|
||||
* integer or a float, so it satisfies EITHER analytics numeric type, but
|
||||
* nothing else. Anything not listed here is a mismatch, not a guess.
|
||||
*/
|
||||
const CONNECTOR_TYPE_SATISFIES: Record<FieldType, readonly AnalyticsFieldType[]> = {
|
||||
string: ['string', 'decimal', 'date', 'timestamp'],
|
||||
number: ['int64', 'float64', 'decimal'],
|
||||
boolean: ['bool'],
|
||||
timestamp: ['timestamp', 'date'],
|
||||
object: ['json'],
|
||||
array: ['json'],
|
||||
};
|
||||
|
||||
export interface ContractCoverageProblem {
|
||||
code: 'undeclared_field' | 'type_mismatch' | 'missing_required_field' | 'nullability_mismatch';
|
||||
field: string;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare what the connector says the source LOOKS LIKE against what the
|
||||
* contract says the dataset IS. Returns every problem; never throws.
|
||||
*/
|
||||
export function contractCoverageProblems(
|
||||
contract: DatasetContract,
|
||||
discovered: RecordSchemaFields,
|
||||
): ContractCoverageProblem[] {
|
||||
const problems: ContractCoverageProblem[] = [];
|
||||
const byName = new Map(contract.fields.map((f) => [f.name, f]));
|
||||
|
||||
for (const [name, spec] of Object.entries(discovered)) {
|
||||
const field = byName.get(name);
|
||||
if (!field) {
|
||||
problems.push({
|
||||
code: 'undeclared_field',
|
||||
field: name,
|
||||
detail:
|
||||
`source field "${name}" is not declared in contract ${contract.dataset_id}@${contract.version}; ` +
|
||||
`an undeclared field is an unclassified field and is refused, not landed`,
|
||||
});
|
||||
continue;
|
||||
}
|
||||
const satisfies = CONNECTOR_TYPE_SATISFIES[spec.type] ?? [];
|
||||
if (!satisfies.includes(field.type)) {
|
||||
problems.push({
|
||||
code: 'type_mismatch',
|
||||
field: name,
|
||||
detail: `source field "${name}" is ${spec.type}; contract declares ${field.type}`,
|
||||
});
|
||||
}
|
||||
// `FieldSpec.required` is the connector-side vocabulary; the contract
|
||||
// speaks `nullable`. required:false === nullable:true.
|
||||
if (spec.required === false && field.nullable === false) {
|
||||
problems.push({
|
||||
code: 'nullability_mismatch',
|
||||
field: name,
|
||||
detail: `source field "${name}" is optional at the source; contract declares it NOT NULL`,
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
for (const field of contract.fields) {
|
||||
if (field.nullable === false && !(field.name in discovered)) {
|
||||
problems.push({
|
||||
code: 'missing_required_field',
|
||||
field: field.name,
|
||||
detail: `contract requires non-null field "${field.name}" but the source does not produce it`,
|
||||
});
|
||||
}
|
||||
}
|
||||
return problems;
|
||||
}
|
||||
|
||||
export function assertContractCoversDiscovery(
|
||||
contract: DatasetContract,
|
||||
discovered: RecordSchemaFields,
|
||||
): void {
|
||||
const problems = contractCoverageProblems(contract, discovered);
|
||||
if (problems.length > 0) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`source shape does not satisfy dataset contract ${contract.dataset_id}@${contract.version}: ` +
|
||||
problems.map((p) => `[${p.code}] ${p.detail}`).join('; '),
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── gate 3: physical home ──────────────────────────────────────────────────
|
||||
|
||||
export function assertPhysicalHomeMatchesTier(target: AnalyticsLandingTarget): void {
|
||||
if (!(LANDING_FORMATS as readonly string[]).includes(target.physical.format)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`landing format "${target.physical.format}" is not a lakehouse format; ` +
|
||||
`RAW/NORMALIZED land as Parquet or Iceberg in ST0RE (BI-P003, st0re#45)`,
|
||||
);
|
||||
}
|
||||
if (!target.physical.namespace.trim() || !target.physical.table_name.trim()) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`landing target for ${target.dataset_id} is missing a ST0RE namespace/table name`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── gate 4: unverified providers may not run in production ─────────────────
|
||||
|
||||
export function assertProviderRunnable(
|
||||
provider: AnalyticsConnectorProvider,
|
||||
spec: AnalyticsIngestionSpec,
|
||||
runtimeEnv: string,
|
||||
): void {
|
||||
const readiness = provider.providerReadiness();
|
||||
if (readiness.verified) return;
|
||||
const isProduction = runtimeEnv === 'production';
|
||||
const exception = (spec.unverified_provider_exception ?? '').trim();
|
||||
if (!isProduction) return;
|
||||
if (exception.length === 0) {
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.ingestion',
|
||||
`provider "${provider.provider}" reports verified: false ` +
|
||||
`(open: ${readiness.open.join('; ')}) and may not run in production ` +
|
||||
`without a recorded unverified_provider_exception`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── planning ───────────────────────────────────────────────────────────────
|
||||
|
||||
export interface PlannedAnalyticsIngestion {
|
||||
spec: AnalyticsIngestionSpec;
|
||||
connector: Connector;
|
||||
execution: ProviderExecutionPlan;
|
||||
readiness: ProviderReadiness;
|
||||
/** The tier the rows land in — echoed so a caller cannot claim a different one later. */
|
||||
landing_tier: AnalyticsTier;
|
||||
}
|
||||
|
||||
/**
|
||||
* Run every gate that can be checked WITHOUT touching the source, then hand
|
||||
* back the connector. `assertContractCoversDiscovery` is deliberately NOT
|
||||
* here: discovery requires a live session, so it runs in `landingSink`'s
|
||||
* caller / `AnalyticsIngestionService` once `discover()` has answered.
|
||||
*/
|
||||
export function planAnalyticsIngestion(
|
||||
provider: AnalyticsConnectorProvider,
|
||||
spec: AnalyticsIngestionSpec,
|
||||
runtimeEnv: string,
|
||||
): PlannedAnalyticsIngestion {
|
||||
if (provider.provider !== spec.provider) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`spec asks for provider "${spec.provider}" but was given "${provider.provider}"`,
|
||||
);
|
||||
}
|
||||
if (PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE.includes(spec.provider)) {
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.ingestion',
|
||||
`provider "${spec.provider}" requires a licence/architecture acceptance this estate has not given ` +
|
||||
`(BI-P129: "Airbyte only optional where legal/license architecture accepts it")`,
|
||||
);
|
||||
}
|
||||
const verdict = provider.validateSpec(spec);
|
||||
if (!verdict.valid) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`provider "${provider.provider}" rejected the spec: ${verdict.errors.join('; ')}`,
|
||||
);
|
||||
}
|
||||
assertLandingTierIngestible(spec.target);
|
||||
assertPhysicalHomeMatchesTier(spec.target);
|
||||
assertProviderRunnable(provider, spec, runtimeEnv);
|
||||
|
||||
return {
|
||||
spec,
|
||||
connector: provider.connectorFor(spec),
|
||||
execution: provider.executionPlan(spec),
|
||||
readiness: provider.providerReadiness(),
|
||||
landing_tier: spec.target.tier,
|
||||
};
|
||||
}
|
||||
|
||||
// ── registry ───────────────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The set of providers a runtime actually has. Registration is explicit:
|
||||
* there is no discovery-by-filename, because a provider appearing in the
|
||||
* registry because a file exists is how an unreviewed integration becomes
|
||||
* live.
|
||||
*/
|
||||
export class ConnectorProviderRegistry {
|
||||
private readonly providers = new Map<ConnectorProviderKind, AnalyticsConnectorProvider>();
|
||||
|
||||
register(provider: AnalyticsConnectorProvider): this {
|
||||
if (this.providers.has(provider.provider)) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.ingestion',
|
||||
`provider "${provider.provider}" is already registered`,
|
||||
);
|
||||
}
|
||||
if (PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE.includes(provider.provider)) {
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.ingestion',
|
||||
`provider "${provider.provider}" may not be registered: no licence/architecture acceptance exists ` +
|
||||
`for it in this estate (BI-P129)`,
|
||||
);
|
||||
}
|
||||
this.providers.set(provider.provider, provider);
|
||||
return this;
|
||||
}
|
||||
|
||||
get(kind: ConnectorProviderKind): AnalyticsConnectorProvider {
|
||||
const p = this.providers.get(kind);
|
||||
if (!p) {
|
||||
throw new SiaxError(
|
||||
'NOT_FOUND',
|
||||
'0pt.analytics.ingestion',
|
||||
`no connector provider registered for kind "${kind}"`,
|
||||
);
|
||||
}
|
||||
return p;
|
||||
}
|
||||
|
||||
has(kind: ConnectorProviderKind): boolean {
|
||||
return this.providers.has(kind);
|
||||
}
|
||||
|
||||
/** Readiness of every registered provider — the operator's one-screen answer. */
|
||||
readinessReport(): ProviderReadiness[] {
|
||||
return [...this.providers.values()].map((p) => p.providerReadiness());
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,160 @@
|
||||
/**
|
||||
* src/analytics/ingestion/landing.ts
|
||||
* ===================================
|
||||
* BI-P129: the write side. An ingestion run's envelopes have to become FILES
|
||||
* in a canonical tier, and this is the only place in the analytics plane that
|
||||
* decides what those files look like.
|
||||
*
|
||||
* WHERE THE BOUNDARY IS. 0pt does not own object storage. ST0RE owns the
|
||||
* lakehouse (namespaces, Iceberg tables, snapshots — `st0re#45`), and 0pt's
|
||||
* existing `src/services/iceberg-write-service.ts` is already the client for
|
||||
* it. So `LandingWriter` is a PORT, not an implementation: it names exactly
|
||||
* what the analytics plane needs from a lake writer (append a batch to one
|
||||
* tier-owned table, tell me what landed) and nothing more. The default
|
||||
* implementation here is in-memory and is used by tests and by dry runs; the
|
||||
* ST0RE-backed one is the existing service, injected.
|
||||
*
|
||||
* WHAT THIS FILE ENFORCES, and why each rule is not decoration:
|
||||
*
|
||||
* - Every landed batch carries `tier` and `dataset_id`. A batch of rows
|
||||
* with no tier is a batch nobody can decide the read rules for later.
|
||||
*
|
||||
* - `assertContractCoversDiscovery` (BI-P129 gate 2) runs HERE, at the
|
||||
* point discovery has actually answered, not at plan time when the
|
||||
* source has not been asked yet. A gate that runs before the fact it
|
||||
* checks is available is a gate that checks nothing.
|
||||
*
|
||||
* - Deletes are LANDED, not dropped. `IngestionEnvelope` with
|
||||
* `operation: "delete"` carries `payload: null`; a landing that silently
|
||||
* skipped it would produce exactly the failure this programme has already
|
||||
* paid for once — an aggregate that never sees a DELETE and overstates a
|
||||
* number while every freshness check reports green. Staleness is not
|
||||
* correctness, and a delete that never lands is not staleness at all,
|
||||
* it is permanent wrongness.
|
||||
*
|
||||
* - The batch records `envelope_id`s, so a replay is detectable downstream.
|
||||
* The runner's guarantee is at-least-once (documented in
|
||||
* src/ingestion/ingestion-runner.ts); a landing that pretended otherwise
|
||||
* would be lying about the pipeline it sits in.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { IngestionEnvelope } from '../../types/ingestion';
|
||||
import type { RecordSchemaFields } from '../../schema/record-schema';
|
||||
import type { AnalyticsTier } from '../domain/tiers';
|
||||
import {
|
||||
assertContractCoversDiscovery,
|
||||
type AnalyticsLandingTarget,
|
||||
type LandingFormat,
|
||||
} from './connector-provider';
|
||||
|
||||
/** One physical batch appended to a tier-owned table. */
|
||||
export interface LandedBatch {
|
||||
dataset_id: string;
|
||||
tier: AnalyticsTier;
|
||||
format: LandingFormat;
|
||||
namespace: string;
|
||||
table_name: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
run_id: string;
|
||||
partition_key: string;
|
||||
/** Rows written, including tombstones. */
|
||||
row_count: number;
|
||||
/** Rows whose payload was null (CDC deletes). Counted separately, never dropped. */
|
||||
tombstone_count: number;
|
||||
envelope_ids: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* The port. One method: append a batch. Deliberately narrow — every extra
|
||||
* method here is a piece of ST0RE's contract leaking into 0pt's.
|
||||
*/
|
||||
export interface LandingWriter {
|
||||
append(batch: LandedBatch): Promise<void>;
|
||||
}
|
||||
|
||||
/** Test/dry-run writer. Keeps everything it was given so a test can assert on it. */
|
||||
export class InMemoryLandingWriter implements LandingWriter {
|
||||
readonly batches: LandedBatch[] = [];
|
||||
async append(batch: LandedBatch): Promise<void> {
|
||||
this.batches.push(batch);
|
||||
}
|
||||
get rowsLanded(): number {
|
||||
return this.batches.reduce((n, b) => n + b.row_count, 0);
|
||||
}
|
||||
get tombstonesLanded(): number {
|
||||
return this.batches.reduce((n, b) => n + b.tombstone_count, 0);
|
||||
}
|
||||
}
|
||||
|
||||
export interface LandingSinkInput {
|
||||
target: AnalyticsLandingTarget;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
run_id: string;
|
||||
writer: LandingWriter;
|
||||
}
|
||||
|
||||
/**
|
||||
* Build an `IngestionSink` (the runner's own type) that lands envelopes into
|
||||
* the declared analytics tier.
|
||||
*
|
||||
* Note the sink is per-run: the runner commits its checkpoint only AFTER
|
||||
* this resolves, so a writer that throws leaves the page unconsumed and the
|
||||
* next run re-reads it. That ordering belongs to the runner and is not
|
||||
* re-implemented here — it is relied upon.
|
||||
*/
|
||||
export function landingSink(input: LandingSinkInput): (envelopes: IngestionEnvelope[]) => Promise<void> {
|
||||
return async (envelopes: IngestionEnvelope[]): Promise<void> => {
|
||||
if (envelopes.length === 0) return;
|
||||
const partitions = new Set(envelopes.map((e) => e.partition_key));
|
||||
for (const partition of partitions) {
|
||||
const forPartition = envelopes.filter((e) => e.partition_key === partition);
|
||||
await input.writer.append({
|
||||
dataset_id: input.target.dataset_id,
|
||||
tier: input.target.tier,
|
||||
format: input.target.physical.format,
|
||||
namespace: input.target.physical.namespace,
|
||||
table_name: input.target.physical.table_name,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
run_id: input.run_id,
|
||||
partition_key: partition,
|
||||
row_count: forPartition.length,
|
||||
tombstone_count: forPartition.filter((e) => e.payload === null).length,
|
||||
envelope_ids: forPartition.map((e) => e.envelope_id),
|
||||
});
|
||||
}
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P129 gate 2, applied once the connector's `discover()` has answered.
|
||||
* Called by `AnalyticsIngestionService`; exported so an operator pre-flight
|
||||
* can run it without starting a run.
|
||||
*/
|
||||
export function assertDiscoveryLandable(
|
||||
target: AnalyticsLandingTarget,
|
||||
discoveredByPartition: Record<string, RecordSchemaFields>,
|
||||
): void {
|
||||
const partitions = Object.keys(discoveredByPartition);
|
||||
if (partitions.length === 0) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`source for dataset ${target.dataset_id} discovered no partitions; there is nothing to land`,
|
||||
);
|
||||
}
|
||||
for (const partition of partitions) {
|
||||
try {
|
||||
assertContractCoversDiscovery(target.contract, discoveredByPartition[partition]);
|
||||
} catch (err) {
|
||||
const detail = err instanceof Error ? err.message : String(err);
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.ingestion',
|
||||
`partition "${partition}": ${detail}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,97 @@
|
||||
/**
|
||||
* src/analytics/ingestion/native-provider.ts
|
||||
* ===========================================
|
||||
* BI-P129, provider kind `native`: connectors 0pt implements and owns.
|
||||
*
|
||||
* There is nothing to integrate here — the connectors already exist and are
|
||||
* already conformance-tested (`src/connectors/conformance.ts`, P061). This
|
||||
* provider's whole job is to select one and report, honestly, that it is
|
||||
* verified: these connectors run in this repo's CI, in-process, with no
|
||||
* third-party runtime and no unreviewed dependency.
|
||||
*
|
||||
* The two shipped natives are deliberately heterogeneous (that was P061's
|
||||
* point and it is BI-P129's DoD too — "two real source types"):
|
||||
*
|
||||
* http_json — remote HTTP API, opaque server-issued page token, health
|
||||
* is a network round trip, record count unknown until the
|
||||
* server says it is done.
|
||||
* jsonl_file — local file drop, cursor is a line offset the connector
|
||||
* computes itself, health is a stat(), record count exact,
|
||||
* and it emits real CDC deletes.
|
||||
*
|
||||
* They share no code but the contract, so "both land in a canonical tier"
|
||||
* is a statement about the SEAM, not about one connector.
|
||||
*/
|
||||
import { HttpJsonConnector, type FetchImpl } from '../../connectors/http-json-connector';
|
||||
import { JsonlFileConnector } from '../../connectors/jsonl-file-connector';
|
||||
import type { Connector, ValidationResult } from '../../connectors/connector';
|
||||
import type {
|
||||
AnalyticsConnectorProvider,
|
||||
AnalyticsIngestionSpec,
|
||||
ProviderExecutionPlan,
|
||||
ProviderReadiness,
|
||||
} from './connector-provider';
|
||||
|
||||
/** Native connector ids this provider can build. */
|
||||
export const NATIVE_CONNECTOR_IDS = ['http_json', 'jsonl_file'] as const;
|
||||
export type NativeConnectorId = (typeof NATIVE_CONNECTOR_IDS)[number];
|
||||
|
||||
export class NativeConnectorProvider implements AnalyticsConnectorProvider {
|
||||
readonly provider = 'native' as const;
|
||||
|
||||
/**
|
||||
* Optional injection point so a caller can supply an `HttpJsonConnector`
|
||||
* with a specific `fetchImpl` (the connector's own documented test seam).
|
||||
* With nothing supplied, a real connector talking to the configured URL is
|
||||
* built — this is not a mock fallback.
|
||||
*/
|
||||
constructor(private readonly fetchImpl?: FetchImpl) {}
|
||||
|
||||
providerReadiness(): ProviderReadiness {
|
||||
return {
|
||||
provider: this.provider,
|
||||
pinned_version: 'in-repo',
|
||||
verified: true,
|
||||
proven: [
|
||||
'both native connectors pass the P061 conformance suite (tests/p0/connector-conformance.test.ts)',
|
||||
'both land into a canonical analytics tier through the P062 ingestion runner ' +
|
||||
'(tests/analytics/analytics-ingestion-providers.test.ts)',
|
||||
'no third-party runtime and no unreviewed dependency is involved',
|
||||
],
|
||||
open: [
|
||||
'only http_json and jsonl_file exist natively; every other source type needs dlt/Singer or a new native connector',
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
validateSpec(spec: AnalyticsIngestionSpec): ValidationResult {
|
||||
const errors: string[] = [];
|
||||
if (!(NATIVE_CONNECTOR_IDS as readonly string[]).includes(spec.connector_ref)) {
|
||||
errors.push(
|
||||
`unknown native connector "${spec.connector_ref}" (have: ${NATIVE_CONNECTOR_IDS.join(', ')})`,
|
||||
);
|
||||
return { valid: false, errors };
|
||||
}
|
||||
// Delegate the config check to the connector itself — it is the authority
|
||||
// on its own config, and its `validate` is pure and never throws.
|
||||
const connector = this.connectorFor(spec);
|
||||
const verdict = connector.validate(spec.config);
|
||||
errors.push(...verdict.errors);
|
||||
return { valid: errors.length === 0, errors };
|
||||
}
|
||||
|
||||
connectorFor(spec: AnalyticsIngestionSpec): Connector {
|
||||
if (spec.connector_ref === 'http_json') {
|
||||
return this.fetchImpl ? new HttpJsonConnector(this.fetchImpl) : new HttpJsonConnector();
|
||||
}
|
||||
if (spec.connector_ref === 'jsonl_file') return new JsonlFileConnector();
|
||||
throw new Error(`unknown native connector "${spec.connector_ref}"`);
|
||||
}
|
||||
|
||||
executionPlan(_spec: AnalyticsIngestionSpec): ProviderExecutionPlan {
|
||||
return {
|
||||
runtime: 'in_process',
|
||||
reason: 'native connectors are Node code in this repo; they need no separate runtime and no N0D job',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,170 @@
|
||||
/**
|
||||
* src/analytics/materialization/drift-detector.ts
|
||||
* ================================================
|
||||
* BI-P105 — is the pre-aggregate still telling the truth?
|
||||
*
|
||||
* The comparison is bucket-by-bucket between the aggregate and a recomputation
|
||||
* from the source over the same window. Three things make it a real detector
|
||||
* rather than a ceremony:
|
||||
*
|
||||
* 1. A MISSING BUCKET IS DRIFT. The nastiest pre-aggregate failure is not a
|
||||
* wrong value, it is an absent row: late-arriving data, a materialized view
|
||||
* that was created after the backfill, a refresh that failed silently. Every
|
||||
* value it DOES hold is correct, so a value-only comparison passes while the
|
||||
* total is wrong. Set difference is checked first, before any arithmetic.
|
||||
*
|
||||
* 2. ZERO BUCKETS IS `insufficient_data`, NEVER `match`. A comparison over an
|
||||
* empty window finds no discrepancies — the same false green BI-P025 found
|
||||
* in a quality check over an empty table. The router treats
|
||||
* `insufficient_data` as "do not serve", not as "fine".
|
||||
*
|
||||
* 3. RELATIVE AND ABSOLUTE TOLERANCE BOTH HAVE TO HOLD. A 0.1% tolerance on a
|
||||
* revenue bucket of 40M is 40 000 kr of invisible error; an absolute-only
|
||||
* tolerance flags every rounding difference on a large bucket. Requiring
|
||||
* both means a discrepancy has to be small in kronor AND small in
|
||||
* proportion.
|
||||
*/
|
||||
|
||||
import type { DriftTolerance, DriftVerdict, MaterializationStatus } from './materialization';
|
||||
|
||||
/** One aggregated row, keyed by its dimension tuple. */
|
||||
export interface AggregateBucket {
|
||||
/** Stable key for the dimension tuple, e.g. "2026-09-01|SE". */
|
||||
key: string;
|
||||
/** metric name -> value. */
|
||||
values: Record<string, number>;
|
||||
}
|
||||
|
||||
export interface DriftDiscrepancy {
|
||||
key: string;
|
||||
metric: string;
|
||||
source_value: number;
|
||||
aggregate_value: number;
|
||||
absolute_delta: number;
|
||||
/** |delta| / |source|, or Infinity when the source value is 0 and the aggregate is not. */
|
||||
relative_delta: number;
|
||||
}
|
||||
|
||||
export interface DriftReport {
|
||||
verdict: DriftVerdict;
|
||||
checked_at: string;
|
||||
buckets_compared: number;
|
||||
/** In the source recomputation, absent from the aggregate. Always drift. */
|
||||
missing_in_aggregate: string[];
|
||||
/** In the aggregate, absent from the source. Always drift (deleted rows). */
|
||||
extra_in_aggregate: string[];
|
||||
discrepancies: DriftDiscrepancy[];
|
||||
worst?: DriftDiscrepancy;
|
||||
tolerance: DriftTolerance;
|
||||
/**
|
||||
* What a caller would have been told if it had trusted the aggregate: the
|
||||
* total the aggregate reports vs the total the source computes, per metric.
|
||||
* This is the number that makes a drift report actionable — "0.4% off" means
|
||||
* nothing until it is "1 284 500 kr reported, 1 271 900 kr true".
|
||||
*/
|
||||
totals: Record<string, { source: number; aggregate: number; delta: number }>;
|
||||
}
|
||||
|
||||
const MAX_REPORTED_DISCREPANCIES = 50;
|
||||
|
||||
function relative(sourceValue: number, delta: number): number {
|
||||
if (sourceValue === 0) return delta === 0 ? 0 : Number.POSITIVE_INFINITY;
|
||||
return Math.abs(delta / sourceValue);
|
||||
}
|
||||
|
||||
/**
|
||||
* Compare a recomputation from source against the pre-aggregate.
|
||||
*
|
||||
* `fromSource` is the truth. Both sides must cover the same window; the caller
|
||||
* is responsible for that, and `missing_in_aggregate` will catch it loudly if
|
||||
* they do not.
|
||||
*/
|
||||
export function detectDrift(args: {
|
||||
fromSource: AggregateBucket[];
|
||||
fromAggregate: AggregateBucket[];
|
||||
tolerance: DriftTolerance;
|
||||
metrics: string[];
|
||||
now?: Date;
|
||||
}): DriftReport {
|
||||
const { fromSource, fromAggregate, tolerance, metrics } = args;
|
||||
const checked_at = (args.now ?? new Date()).toISOString();
|
||||
const sourceByKey = new Map(fromSource.map((b) => [b.key, b]));
|
||||
const aggByKey = new Map(fromAggregate.map((b) => [b.key, b]));
|
||||
|
||||
const missing_in_aggregate = [...sourceByKey.keys()].filter((k) => !aggByKey.has(k)).sort();
|
||||
const extra_in_aggregate = [...aggByKey.keys()].filter((k) => !sourceByKey.has(k)).sort();
|
||||
|
||||
const discrepancies: DriftDiscrepancy[] = [];
|
||||
let buckets_compared = 0;
|
||||
const totals: DriftReport['totals'] = {};
|
||||
for (const m of metrics) totals[m] = { source: 0, aggregate: 0, delta: 0 };
|
||||
|
||||
for (const [key, sourceBucket] of sourceByKey) {
|
||||
for (const m of metrics) totals[m].source += sourceBucket.values[m] ?? 0;
|
||||
const aggBucket = aggByKey.get(key);
|
||||
if (!aggBucket) continue;
|
||||
buckets_compared += 1;
|
||||
for (const m of metrics) {
|
||||
const sv = sourceBucket.values[m] ?? 0;
|
||||
const av = aggBucket.values[m] ?? 0;
|
||||
const delta = av - sv;
|
||||
const rel = relative(sv, delta);
|
||||
if (Math.abs(delta) > tolerance.absolute || rel > tolerance.relative) {
|
||||
discrepancies.push({
|
||||
key,
|
||||
metric: m,
|
||||
source_value: sv,
|
||||
aggregate_value: av,
|
||||
absolute_delta: delta,
|
||||
relative_delta: rel,
|
||||
});
|
||||
}
|
||||
}
|
||||
}
|
||||
for (const bucket of aggByKey.values()) {
|
||||
for (const m of metrics) totals[m].aggregate += bucket.values[m] ?? 0;
|
||||
}
|
||||
for (const m of metrics) totals[m].delta = totals[m].aggregate - totals[m].source;
|
||||
|
||||
discrepancies.sort((a, b) => Math.abs(b.absolute_delta) - Math.abs(a.absolute_delta));
|
||||
|
||||
// A comparison that compared nothing has not proved anything. This is the
|
||||
// BI-P025 false green, restated for pre-aggregates.
|
||||
const verdict: DriftVerdict =
|
||||
buckets_compared === 0 && missing_in_aggregate.length === 0 && extra_in_aggregate.length === 0
|
||||
? 'insufficient_data'
|
||||
: discrepancies.length > 0 || missing_in_aggregate.length > 0 || extra_in_aggregate.length > 0
|
||||
? 'drifted'
|
||||
: 'match';
|
||||
|
||||
const report: DriftReport = {
|
||||
verdict,
|
||||
checked_at,
|
||||
buckets_compared,
|
||||
missing_in_aggregate,
|
||||
extra_in_aggregate,
|
||||
discrepancies: discrepancies.slice(0, MAX_REPORTED_DISCREPANCIES),
|
||||
tolerance,
|
||||
totals,
|
||||
};
|
||||
if (discrepancies.length > 0) report.worst = discrepancies[0];
|
||||
return report;
|
||||
}
|
||||
|
||||
/**
|
||||
* Fold a drift report into the runtime status. A drifted aggregate is taken
|
||||
* OUT of service here rather than merely annotated: leaving it `serving` with
|
||||
* a bad verdict relies on every future reader remembering to check the
|
||||
* verdict, and one that forgets serves the wrong number.
|
||||
*
|
||||
* `insufficient_data` also stops service. It means the check could not prove
|
||||
* anything, and an unproven aggregate is not a current one.
|
||||
*/
|
||||
export function applyDriftReport(status: MaterializationStatus, report: DriftReport): MaterializationStatus {
|
||||
return {
|
||||
...status,
|
||||
state: report.verdict === 'match' ? status.state : 'drifted',
|
||||
last_drift_check_at: report.checked_at,
|
||||
last_drift_verdict: report.verdict,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,309 @@
|
||||
/**
|
||||
* src/analytics/materialization/materialization.ts
|
||||
* =================================================
|
||||
* BI-P105 — MATERIALIZATION / PRE-AGGREGATION.
|
||||
*
|
||||
* A pre-aggregate is the one optimisation that fails SILENTLY. A slow query is
|
||||
* a complaint; a pre-aggregate that has drifted from its source answers
|
||||
* instantly, with confidence, and is wrong — no error, no timeout, nothing in
|
||||
* a log. So this module is built around one rule:
|
||||
*
|
||||
* A MATERIALIZATION SERVES ONLY WHILE IT IS PROVABLY CURRENT AND PROVABLY
|
||||
* UNDRIFTED. Every unknown routes to the source table.
|
||||
*
|
||||
* Six refusal reasons, each of which is a real way the aggregate becomes a
|
||||
* confident liar:
|
||||
*
|
||||
* coverage_miss it does not carry the metric/dimension/grain
|
||||
* asked for (a filter on a dimension the
|
||||
* aggregate collapsed cannot be applied after
|
||||
* the fact, and applying it anyway is how a
|
||||
* pre-aggregate returns another slice's number)
|
||||
* watermark_unknown nobody knows how current it is. BI-P026 already
|
||||
* refuses to render UNKNOWN age as current; the
|
||||
* same doctrine applies here.
|
||||
* staleness_exceeded it is older than its own freshness contract
|
||||
* window_beyond_watermark the question reaches past the newest data the
|
||||
* aggregate contains — it would silently answer
|
||||
* with a shorter period
|
||||
* drift_never_verified it has never been compared to its source
|
||||
* drift_verification_stale the last comparison is older than the interval
|
||||
* the spec requires
|
||||
* drift_detected the last comparison FAILED
|
||||
*
|
||||
* `drift_never_verified` is the one people leave out, and it is the one that
|
||||
* matters most on day one: a brand-new materialization is exactly as unproven
|
||||
* as a drifted one, and a router that defaults new aggregates to "serving"
|
||||
* ships the bug on the day the feature launches.
|
||||
*
|
||||
* The DoD also asks for measured improvement and documented freshness
|
||||
* semantics: scripts/bi-p105-materialization-drift.mjs measures both against a
|
||||
* live ClickHouse and writes evidence/bi-p105/materialization-drift.json,
|
||||
* including what happens WITHOUT this router (the drifted aggregate served).
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { AnalyticsQueryRequest, TimeGrain } from '../contract/query-request';
|
||||
|
||||
export const MATERIALIZATION_STRATEGIES = [
|
||||
'clickhouse_materialized_view',
|
||||
'clickhouse_projection',
|
||||
'dbt_incremental',
|
||||
'cube_preaggregation',
|
||||
] as const;
|
||||
export type MaterializationStrategy = (typeof MATERIALIZATION_STRATEGIES)[number];
|
||||
|
||||
export const MATERIALIZATION_STATES = ['building', 'serving', 'drifted', 'disabled'] as const;
|
||||
export type MaterializationState = (typeof MATERIALIZATION_STATES)[number];
|
||||
|
||||
/**
|
||||
* Grains ordered coarsest-last. An aggregate at `day` can answer `day` and
|
||||
* anything coarser by re-aggregating; it can NEVER answer `hour`, because the
|
||||
* detail was thrown away when the rows were rolled up.
|
||||
*/
|
||||
const GRAIN_ORDER: readonly TimeGrain[] = ['hour', 'day', 'week', 'month', 'quarter', 'year'];
|
||||
|
||||
/**
|
||||
* `week` does not divide `month` and `month` does not divide `quarter` by row
|
||||
* count, but both roll up cleanly from `day`. Grains that CANNOT be derived by
|
||||
* re-aggregating a coarser one are listed here so the router refuses rather
|
||||
* than quietly returning a mis-bucketed series.
|
||||
*/
|
||||
function canRollUp(from: TimeGrain, to: TimeGrain): boolean {
|
||||
if (from === to) return true;
|
||||
const fi = GRAIN_ORDER.indexOf(from);
|
||||
const ti = GRAIN_ORDER.indexOf(to);
|
||||
if (fi < 0 || ti < 0 || ti < fi) return false;
|
||||
// Weekly buckets do not compose into calendar months/quarters/years.
|
||||
if (from === 'week' && to !== 'week') return false;
|
||||
return true;
|
||||
}
|
||||
|
||||
export interface DriftTolerance {
|
||||
/** Absolute delta allowed per metric per bucket. */
|
||||
absolute: number;
|
||||
/** Relative delta allowed (0.001 = 0.1%). Both must hold. */
|
||||
relative: number;
|
||||
}
|
||||
|
||||
/** What a materialization IS. Declarative; the engine adapters read it. */
|
||||
export interface MaterializationSpec {
|
||||
materialization_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
/** The dataset it pre-aggregates (BI-P003). */
|
||||
dataset: string;
|
||||
strategy: MaterializationStrategy;
|
||||
/** Physical target (ClickHouse table, dbt model, Cube pre-agg name). */
|
||||
target_ref: string;
|
||||
metrics: string[];
|
||||
dimensions: string[];
|
||||
time_dimension?: string;
|
||||
grain?: TimeGrain;
|
||||
/** The freshness contract. Older than this and it stops serving. */
|
||||
max_staleness_seconds: number;
|
||||
/** How often the aggregate must be re-proved against its source. */
|
||||
drift_check_interval_seconds: number;
|
||||
tolerance: DriftTolerance;
|
||||
}
|
||||
|
||||
export const DRIFT_VERDICTS = ['match', 'drifted', 'insufficient_data'] as const;
|
||||
export type DriftVerdict = (typeof DRIFT_VERDICTS)[number];
|
||||
|
||||
/** Runtime state. Separate from the spec: the spec is config, this is fact. */
|
||||
export interface MaterializationStatus {
|
||||
materialization_id: string;
|
||||
state: MaterializationState;
|
||||
/**
|
||||
* The newest source timestamp the aggregate represents. UNDEFINED means
|
||||
* UNKNOWN, and unknown is never treated as current.
|
||||
*/
|
||||
data_watermark?: string;
|
||||
last_refresh_at?: string;
|
||||
last_drift_check_at?: string;
|
||||
last_drift_verdict?: DriftVerdict;
|
||||
}
|
||||
|
||||
export const ROUTING_REFUSALS = [
|
||||
'coverage_miss',
|
||||
'watermark_unknown',
|
||||
'staleness_exceeded',
|
||||
'window_beyond_watermark',
|
||||
'drift_never_verified',
|
||||
'drift_verification_stale',
|
||||
'drift_detected',
|
||||
'state_not_serving',
|
||||
] as const;
|
||||
export type RoutingRefusal = (typeof ROUTING_REFUSALS)[number];
|
||||
|
||||
export type RoutingDecision =
|
||||
| {
|
||||
source: 'materialization';
|
||||
materialization_id: string;
|
||||
target_ref: string;
|
||||
/** Reported so a caller can show it; never inferred, always measured. */
|
||||
staleness_seconds: number;
|
||||
}
|
||||
| { source: 'base_table'; refusal: RoutingRefusal; detail: string };
|
||||
|
||||
function refuse(refusal: RoutingRefusal, detail: string): RoutingDecision {
|
||||
return { source: 'base_table', refusal, detail };
|
||||
}
|
||||
|
||||
/**
|
||||
* Staleness of an aggregate, in seconds, or `undefined` when it cannot be
|
||||
* computed. Deliberately NOT defaulted to 0: "we could not tell" and "it is
|
||||
* current" are different facts, and conflating them is the bug.
|
||||
*/
|
||||
export function materializationStaleness(status: MaterializationStatus, now: Date): number | undefined {
|
||||
if (!status.data_watermark) return undefined;
|
||||
const t = Date.parse(status.data_watermark);
|
||||
if (Number.isNaN(t)) return undefined;
|
||||
return Math.max(0, Math.floor((now.getTime() - t) / 1000));
|
||||
}
|
||||
|
||||
/**
|
||||
* Decide whether a request may be answered from the pre-aggregate.
|
||||
*
|
||||
* Fail-closed by construction: the function returns `materialization` only
|
||||
* after every check has passed, and every unknown is a refusal with a named
|
||||
* reason. Callers get the reason so it can be logged and counted (BI-P103
|
||||
* reads the counts).
|
||||
*/
|
||||
export function selectQuerySource(args: {
|
||||
request: AnalyticsQueryRequest;
|
||||
spec: MaterializationSpec;
|
||||
status: MaterializationStatus;
|
||||
now: Date;
|
||||
}): RoutingDecision {
|
||||
const { request, spec, status, now } = args;
|
||||
|
||||
if (request.dataset !== spec.dataset) {
|
||||
return refuse('coverage_miss', `materialization covers "${spec.dataset}", not "${request.dataset}"`);
|
||||
}
|
||||
if (status.state !== 'serving') {
|
||||
return refuse('state_not_serving', `materialization state is "${status.state}"`);
|
||||
}
|
||||
|
||||
// ── coverage ────────────────────────────────────────────────────────────
|
||||
const metrics = new Set(spec.metrics);
|
||||
for (const m of request.metrics) {
|
||||
if (!metrics.has(m)) return refuse('coverage_miss', `metric "${m}" is not in the pre-aggregate`);
|
||||
}
|
||||
const dimensions = new Set(spec.dimensions);
|
||||
for (const d of request.dimensions) {
|
||||
if (!dimensions.has(d)) return refuse('coverage_miss', `dimension "${d}" is not in the pre-aggregate`);
|
||||
}
|
||||
for (const f of request.filters) {
|
||||
if (!dimensions.has(f.dimension)) {
|
||||
// The aggregate summed across this dimension. Filtering afterwards would
|
||||
// return a number computed over rows the filter should have excluded.
|
||||
return refuse(
|
||||
'coverage_miss',
|
||||
`filter on "${f.dimension}" cannot be applied: the pre-aggregate has already summed across it`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const requestedGrain = request.timeRange?.grain;
|
||||
if (requestedGrain) {
|
||||
if (!spec.grain) return refuse('coverage_miss', 'the pre-aggregate has no time grain');
|
||||
if (!canRollUp(spec.grain, requestedGrain)) {
|
||||
return refuse(
|
||||
'coverage_miss',
|
||||
`a "${spec.grain}" pre-aggregate cannot answer at "${requestedGrain}" grain`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── freshness ───────────────────────────────────────────────────────────
|
||||
const staleness = materializationStaleness(status, now);
|
||||
if (staleness === undefined) {
|
||||
return refuse(
|
||||
'watermark_unknown',
|
||||
'the pre-aggregate has no data watermark — unknown age is not current age',
|
||||
);
|
||||
}
|
||||
if (staleness > spec.max_staleness_seconds) {
|
||||
return refuse(
|
||||
'staleness_exceeded',
|
||||
`${staleness}s stale, contract allows ${spec.max_staleness_seconds}s`,
|
||||
);
|
||||
}
|
||||
const to = request.timeRange?.to;
|
||||
if (to) {
|
||||
const toMs = Date.parse(to);
|
||||
const watermarkMs = Date.parse(status.data_watermark!);
|
||||
if (!Number.isNaN(toMs) && toMs > watermarkMs) {
|
||||
return refuse(
|
||||
'window_beyond_watermark',
|
||||
`the question runs to ${to} but the pre-aggregate only contains data to ${status.data_watermark}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── drift ───────────────────────────────────────────────────────────────
|
||||
if (!status.last_drift_check_at || !status.last_drift_verdict) {
|
||||
return refuse(
|
||||
'drift_never_verified',
|
||||
'the pre-aggregate has never been compared to its source; a new aggregate is as unproven as a drifted one',
|
||||
);
|
||||
}
|
||||
if (status.last_drift_verdict !== 'match') {
|
||||
return refuse('drift_detected', `last drift check returned "${status.last_drift_verdict}"`);
|
||||
}
|
||||
const checkedAgo = Math.floor((now.getTime() - Date.parse(status.last_drift_check_at)) / 1000);
|
||||
if (Number.isNaN(checkedAgo)) {
|
||||
return refuse('drift_never_verified', 'last_drift_check_at is not a timestamp');
|
||||
}
|
||||
if (checkedAgo > spec.drift_check_interval_seconds) {
|
||||
return refuse(
|
||||
'drift_verification_stale',
|
||||
`last verified ${checkedAgo}s ago, spec requires every ${spec.drift_check_interval_seconds}s`,
|
||||
);
|
||||
}
|
||||
|
||||
return {
|
||||
source: 'materialization',
|
||||
materialization_id: spec.materialization_id,
|
||||
target_ref: spec.target_ref,
|
||||
staleness_seconds: staleness,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The routing decision as a freshness envelope for the result (BI-P026 shape):
|
||||
* a consumer must be able to see WHICH source answered and how old it was.
|
||||
* A result that does not say is a result nobody can audit.
|
||||
*/
|
||||
export function describeRouting(decision: RoutingDecision): {
|
||||
served_from: 'materialization' | 'base_table';
|
||||
materialization_id?: string;
|
||||
staleness_seconds?: number;
|
||||
refusal?: RoutingRefusal;
|
||||
detail?: string;
|
||||
} {
|
||||
return decision.source === 'materialization'
|
||||
? {
|
||||
served_from: 'materialization',
|
||||
materialization_id: decision.materialization_id,
|
||||
staleness_seconds: decision.staleness_seconds,
|
||||
}
|
||||
: { served_from: 'base_table', refusal: decision.refusal, detail: decision.detail };
|
||||
}
|
||||
|
||||
export function assertSpec(spec: MaterializationSpec): void {
|
||||
const invalid = (d: string): never => {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.materialization', d);
|
||||
};
|
||||
if (!spec.materialization_id?.trim()) invalid('materialization_id is required');
|
||||
if (!spec.tenant_id?.trim() || !spec.project_id?.trim()) invalid('a materialization is tenant-scoped');
|
||||
if (spec.metrics.length === 0) invalid('a materialization with no metrics pre-aggregates nothing');
|
||||
if (spec.max_staleness_seconds <= 0) invalid('max_staleness_seconds must be positive');
|
||||
if (spec.drift_check_interval_seconds <= 0) {
|
||||
invalid(
|
||||
'drift_check_interval_seconds must be positive — an aggregate that is never re-verified is one ' +
|
||||
'source mutation away from serving wrong numbers forever',
|
||||
);
|
||||
}
|
||||
if (spec.tolerance.absolute < 0 || spec.tolerance.relative < 0) invalid('tolerance must be non-negative');
|
||||
if (spec.grain && !GRAIN_ORDER.includes(spec.grain)) invalid('grain is not a time grain');
|
||||
}
|
||||
@@ -0,0 +1,419 @@
|
||||
/**
|
||||
* src/analytics/metrics/metric-model.ts
|
||||
* ======================================
|
||||
* BI-P028 — METRIC DOMAIN MODEL, and BI-P031 — METRIC GOVERNANCE.
|
||||
*
|
||||
* BI-P028 names a canonical metric with fifteen fields. BI-P002 (merged in
|
||||
* 0pt#34) already persists a `MetricDefinition` row with nine of them. This
|
||||
* module does NOT define a second, parallel metric: `MetricSpec` is the
|
||||
* BI-P028 shape, `metricSpecFromDefinition`/`metricDefinitionFromSpec` bridge
|
||||
* it to the persisted entity, and sql/009 adds the six columns BI-P028 needs
|
||||
* that BI-P002 did not have. A second metric type would be the exact failure
|
||||
* BI-P028's DoD is about.
|
||||
*
|
||||
* ── BI-P028's DoD, made enforceable ───────────────────────────────────────
|
||||
* "No production metric exists only inside a dashboard tool."
|
||||
*
|
||||
* A statement of intent unless something refuses. `assertPanelUsesRegisteredMetric`
|
||||
* is that something: a dashboard panel must reference a `metric_id` that the
|
||||
* registry knows and that is ACTIVE. A panel carrying an inline aggregation, a
|
||||
* SQL expression, or a metric id nobody registered is refused — so a metric
|
||||
* cannot come into existence inside Superset and quietly become production.
|
||||
*
|
||||
* ── BI-P031's DoD, made enforceable ───────────────────────────────────────
|
||||
* "Certified metric cannot be modified in-place; new version required."
|
||||
*
|
||||
* `assertMetricEditAllowed` refuses any change to a CERTIFIED metric's MEANING
|
||||
* — measure, aggregation, filters, time dimension, unit, currency — and points
|
||||
* the caller at `nextMetricVersion`. Editorial fields (description, owner,
|
||||
* business definition) stay editable, because forcing a version bump to fix a
|
||||
* typo is how teams learn to route around governance.
|
||||
*
|
||||
* The TypeScript check is layer 1. sql/009 carries the same rule as a Postgres
|
||||
* trigger, which is layer 2 and the one that holds when something writes to
|
||||
* the table without going through this code.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { MetricAggregation, MetricDefinition } from '../domain/entities';
|
||||
|
||||
// ── BI-P028 vocabulary ─────────────────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* `type` in BI-P028's field list. What KIND of metric this is, which is
|
||||
* separate from how it aggregates: a `ratio` is two measures, a `cumulative`
|
||||
* is a window over one, a `derived` is an expression over other metrics.
|
||||
*/
|
||||
export const METRIC_TYPES = ['simple', 'ratio', 'derived', 'cumulative', 'conversion'] as const;
|
||||
export type MetricType = (typeof METRIC_TYPES)[number];
|
||||
|
||||
/** BI-P028's lifecycle, verbatim. */
|
||||
export const METRIC_STATUSES = ['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED'] as const;
|
||||
export type MetricStatus = (typeof METRIC_STATUSES)[number];
|
||||
|
||||
/** BI-P031's certification axis. Orthogonal to lifecycle status, on purpose. */
|
||||
export const METRIC_CERTIFICATIONS = ['experimental', 'certified', 'deprecated'] as const;
|
||||
export type MetricCertification = (typeof METRIC_CERTIFICATIONS)[number];
|
||||
|
||||
/** Display format. A number without one is a number a dashboard guesses at. */
|
||||
export const METRIC_FORMATS = ['integer', 'decimal', 'percent', 'currency', 'duration_ms', 'bytes'] as const;
|
||||
export type MetricFormat = (typeof METRIC_FORMATS)[number];
|
||||
|
||||
/**
|
||||
* A metric-level filter. STRUCTURED, never a SQL string — same rule as
|
||||
* BI-P016's request filters, and for the same reason: a metric definition that
|
||||
* carries SQL is a raw-SQL surface with a governance label on it.
|
||||
*/
|
||||
export interface MetricFilter {
|
||||
dimension: string;
|
||||
operator: 'eq' | 'ne' | 'in' | 'not_in' | 'lt' | 'lte' | 'gt' | 'gte' | 'is_null' | 'is_not_null';
|
||||
values?: string[];
|
||||
}
|
||||
|
||||
/** BI-P028's canonical metric, field for field. */
|
||||
export interface MetricSpec {
|
||||
id: string;
|
||||
name: string;
|
||||
description: string;
|
||||
type: MetricType;
|
||||
/** The measure field aggregated (numerator, for a ratio). */
|
||||
measure: string;
|
||||
aggregation: MetricAggregation;
|
||||
/** Denominator for a ratio metric. */
|
||||
denominator?: string;
|
||||
/** Dimension names this metric may legitimately be sliced by. */
|
||||
dimensions: string[];
|
||||
/** Filters baked into the metric's definition. */
|
||||
filters: MetricFilter[];
|
||||
/** The dimension time ranges and grains apply to. */
|
||||
timeDimension: string;
|
||||
unit?: string;
|
||||
/** ISO-4217, required when `format` is "currency". */
|
||||
currency?: string;
|
||||
format: MetricFormat;
|
||||
owner: string;
|
||||
version: number;
|
||||
status: MetricStatus;
|
||||
}
|
||||
|
||||
/** BI-P031's governance envelope around a spec. */
|
||||
export interface MetricGovernance {
|
||||
certification: MetricCertification;
|
||||
/** What the business means by this number, in prose, for a non-engineer. */
|
||||
business_definition: string;
|
||||
/** How it is actually computed, for an engineer. */
|
||||
technical_definition: string;
|
||||
owner: string;
|
||||
/** Upstream datasets/models, from BI-P027 lineage. */
|
||||
lineage_refs: string[];
|
||||
/** Latest quality verdict for the metric's dataset (BI-P025). */
|
||||
quality_status?: 'passing' | 'warning' | 'failing' | 'unknown';
|
||||
/** Usage, from AUD0's BI-P076 events. Drives deprecation decisions. */
|
||||
usage_30d?: { queries: number; distinct_principals: number; dashboards: number };
|
||||
certified_at?: string;
|
||||
certified_by?: string;
|
||||
}
|
||||
|
||||
export interface GovernedMetric {
|
||||
spec: MetricSpec;
|
||||
governance: MetricGovernance;
|
||||
}
|
||||
|
||||
// ── validation ─────────────────────────────────────────────────────────────
|
||||
|
||||
const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/;
|
||||
|
||||
function invalid(detail: string): never {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.metrics', detail);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate a spec. Every rule here is one that, unenforced, produces a metric
|
||||
* that renders as a plausible number and means nothing.
|
||||
*/
|
||||
export function assertValidMetricSpec(spec: MetricSpec): void {
|
||||
if (!NAME_RE.test(spec.name)) {
|
||||
invalid(`metric name "${spec.name}" must be a lowercase snake_case identifier`);
|
||||
}
|
||||
if (!spec.description?.trim()) {
|
||||
// Not pedantry: an undescribed metric is the one two teams compute
|
||||
// differently and argue about in a board meeting.
|
||||
invalid(`metric "${spec.name}" has no description`);
|
||||
}
|
||||
if (!spec.owner?.trim()) invalid(`metric "${spec.name}" has no owner`);
|
||||
if (!spec.timeDimension?.trim()) {
|
||||
invalid(
|
||||
`metric "${spec.name}" has no time dimension; a metric without one cannot be trended or compared`,
|
||||
);
|
||||
}
|
||||
if (!(METRIC_TYPES as readonly string[]).includes(spec.type)) invalid(`unknown metric type "${spec.type}"`);
|
||||
if (!(METRIC_STATUSES as readonly string[]).includes(spec.status))
|
||||
invalid(`unknown metric status "${spec.status}"`);
|
||||
if (!(METRIC_FORMATS as readonly string[]).includes(spec.format))
|
||||
invalid(`unknown metric format "${spec.format}"`);
|
||||
if (spec.version < 1 || !Number.isInteger(spec.version))
|
||||
invalid('metric version must be a positive integer');
|
||||
|
||||
if (spec.type === 'ratio' && !spec.denominator) {
|
||||
invalid(`ratio metric "${spec.name}" has no denominator`);
|
||||
}
|
||||
if (spec.type !== 'ratio' && spec.denominator) {
|
||||
invalid(`metric "${spec.name}" is not a ratio but carries a denominator`);
|
||||
}
|
||||
if (spec.format === 'currency' && !spec.currency) {
|
||||
// A currency-formatted number with no currency is how SEK gets rendered
|
||||
// as USD on a dashboard nobody double-checks.
|
||||
invalid(`metric "${spec.name}" is currency-formatted but names no currency`);
|
||||
}
|
||||
if (spec.currency && !/^[A-Z]{3}$/.test(spec.currency)) {
|
||||
invalid(`metric "${spec.name}" currency must be an ISO-4217 code`);
|
||||
}
|
||||
if (spec.format !== 'currency' && spec.currency) {
|
||||
invalid(`metric "${spec.name}" names a currency but is not currency-formatted`);
|
||||
}
|
||||
if (new Set(spec.dimensions).size !== spec.dimensions.length) {
|
||||
invalid(`metric "${spec.name}" lists a dimension twice`);
|
||||
}
|
||||
for (const f of spec.filters) {
|
||||
if (!spec.dimensions.includes(f.dimension) && f.dimension !== spec.timeDimension) {
|
||||
invalid(`metric "${spec.name}" filters on "${f.dimension}", which is not one of its dimensions`);
|
||||
}
|
||||
const needsValues = !['is_null', 'is_not_null'].includes(f.operator);
|
||||
if (needsValues && (!f.values || f.values.length === 0)) {
|
||||
invalid(`metric "${spec.name}" filter on "${f.dimension}" has no values`);
|
||||
}
|
||||
if (!needsValues && f.values?.length) {
|
||||
invalid(`metric "${spec.name}" filter operator "${f.operator}" takes no values`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The hash that makes "did the meaning change?" answerable.
|
||||
*
|
||||
* Deliberately covers ONLY the semantic fields. Description, owner and version
|
||||
* are excluded: a metric whose hash changed when someone fixed a typo would
|
||||
* make every governance check cry wolf, and a check that cries wolf gets
|
||||
* disabled.
|
||||
*/
|
||||
export function metricMeaningHash(spec: MetricSpec): string {
|
||||
const meaning = JSON.stringify([
|
||||
spec.type,
|
||||
spec.measure,
|
||||
spec.aggregation,
|
||||
spec.denominator ?? '',
|
||||
[...spec.dimensions].sort(),
|
||||
[...spec.filters]
|
||||
.map((f) => [f.dimension, f.operator, [...(f.values ?? [])].sort()])
|
||||
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
||||
spec.timeDimension,
|
||||
spec.unit ?? '',
|
||||
spec.currency ?? '',
|
||||
]);
|
||||
return crypto.createHash('sha256').update(meaning).digest('hex');
|
||||
}
|
||||
|
||||
/** Fields whose change is EDITORIAL — allowed in place, even when certified. */
|
||||
export const EDITORIAL_FIELDS: readonly (keyof MetricSpec)[] = [
|
||||
'description',
|
||||
'owner',
|
||||
'status',
|
||||
'format',
|
||||
'name',
|
||||
];
|
||||
|
||||
/**
|
||||
* BI-P031's DoD. Refuses an in-place change to a CERTIFIED metric's meaning.
|
||||
*
|
||||
* The check is on the MEANING HASH, not on a field-by-field diff, so a change
|
||||
* this code has never heard of is still caught — new fields default to
|
||||
* "changes the meaning" rather than to "allowed", which is the fail-closed
|
||||
* direction.
|
||||
*/
|
||||
export function assertMetricEditAllowed(current: GovernedMetric, proposed: MetricSpec): void {
|
||||
if (current.spec.id !== proposed.id) {
|
||||
invalid('a metric edit must not change the metric id');
|
||||
}
|
||||
if (current.governance.certification !== 'certified') return;
|
||||
if (proposed.version !== current.spec.version) {
|
||||
// A version bump is the sanctioned path; it is not an "edit".
|
||||
return;
|
||||
}
|
||||
if (metricMeaningHash(current.spec) !== metricMeaningHash(proposed)) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.metrics',
|
||||
`metric "${current.spec.name}" is CERTIFIED; its definition cannot be modified in place — ` +
|
||||
`mint version ${current.spec.version + 1} instead`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Mint the next version. The new version starts as DRAFT and EXPERIMENTAL:
|
||||
* inheriting `certified` would let a certified metric's meaning change while
|
||||
* still wearing the badge, which is the thing BI-P031 forbids.
|
||||
*/
|
||||
export function nextMetricVersion(
|
||||
current: GovernedMetric,
|
||||
changes: Partial<MetricSpec>,
|
||||
reason: string,
|
||||
): { spec: MetricSpec; governance: MetricGovernance; change_reason: string } {
|
||||
if (!reason?.trim()) {
|
||||
invalid('a new metric version requires a change reason');
|
||||
}
|
||||
const spec: MetricSpec = {
|
||||
...current.spec,
|
||||
...changes,
|
||||
id: current.spec.id,
|
||||
version: current.spec.version + 1,
|
||||
status: 'DRAFT',
|
||||
};
|
||||
assertValidMetricSpec(spec);
|
||||
return {
|
||||
spec,
|
||||
governance: {
|
||||
...current.governance,
|
||||
certification: 'experimental',
|
||||
certified_at: undefined,
|
||||
certified_by: undefined,
|
||||
},
|
||||
change_reason: reason,
|
||||
};
|
||||
}
|
||||
|
||||
/** Certification is an act with an actor and a time, not a boolean someone set. */
|
||||
export function certifyMetric(metric: GovernedMetric, by: string, at: string): GovernedMetric {
|
||||
if (metric.spec.status !== 'ACTIVE') {
|
||||
invalid(`only an ACTIVE metric can be certified; "${metric.spec.name}" is ${metric.spec.status}`);
|
||||
}
|
||||
if (!metric.governance.business_definition?.trim() || !metric.governance.technical_definition?.trim()) {
|
||||
invalid(
|
||||
`metric "${metric.spec.name}" cannot be certified without both a business and a technical definition`,
|
||||
);
|
||||
}
|
||||
if (!by?.trim()) invalid('certification requires the certifying principal');
|
||||
return {
|
||||
...metric,
|
||||
governance: { ...metric.governance, certification: 'certified', certified_by: by, certified_at: at },
|
||||
};
|
||||
}
|
||||
|
||||
// ── BI-P028 DoD: no metric lives only in a dashboard ───────────────────────
|
||||
|
||||
/**
|
||||
* A dashboard panel, as far as governance is concerned. It may name a metric.
|
||||
* It may not BE one.
|
||||
*/
|
||||
export interface DashboardPanelRef {
|
||||
panel_id: string;
|
||||
metric_id?: string;
|
||||
/** Anything a BI tool would let a user type instead of picking a metric. */
|
||||
inline_expression?: string;
|
||||
inline_aggregation?: string;
|
||||
inline_sql?: string;
|
||||
}
|
||||
|
||||
export interface MetricLookup {
|
||||
(metricId: string): GovernedMetric | undefined;
|
||||
}
|
||||
|
||||
export function assertPanelUsesRegisteredMetric(
|
||||
panel: DashboardPanelRef,
|
||||
lookup: MetricLookup,
|
||||
): GovernedMetric {
|
||||
for (const field of ['inline_expression', 'inline_aggregation', 'inline_sql'] as const) {
|
||||
if (panel[field]) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.metrics',
|
||||
`panel "${panel.panel_id}" defines a metric inline via "${field}" — BI-P028: no production metric ` +
|
||||
`may exist only inside a dashboard tool. Register it and reference its metric_id.`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!panel.metric_id) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.metrics',
|
||||
`panel "${panel.panel_id}" references no metric_id`,
|
||||
);
|
||||
}
|
||||
const metric = lookup(panel.metric_id);
|
||||
if (!metric) {
|
||||
throw new SiaxError(
|
||||
'NOT_FOUND',
|
||||
'0pt.analytics.metrics',
|
||||
`panel "${panel.panel_id}" references unregistered metric "${panel.metric_id}"`,
|
||||
);
|
||||
}
|
||||
if (metric.spec.status === 'RETIRED') {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.metrics',
|
||||
`panel "${panel.panel_id}" references RETIRED metric "${metric.spec.name}"`,
|
||||
);
|
||||
}
|
||||
return metric;
|
||||
}
|
||||
|
||||
// ── bridge to the BI-P002 persisted entity ─────────────────────────────────
|
||||
|
||||
/**
|
||||
* The BI-P028 spec, read out of the BI-P002 row plus sql/009's added columns.
|
||||
* Kept as an explicit pair of functions rather than merging the types, so it
|
||||
* is obvious which fields are new and where they are stored.
|
||||
*/
|
||||
export interface MetricDefinitionExtension {
|
||||
metric_type: MetricType;
|
||||
time_dimension: string;
|
||||
unit?: string;
|
||||
currency?: string;
|
||||
display_format: MetricFormat;
|
||||
status: MetricStatus;
|
||||
filters_json: string;
|
||||
certification: MetricCertification;
|
||||
business_definition: string;
|
||||
technical_definition: string;
|
||||
meaning_hash: string;
|
||||
}
|
||||
|
||||
export function metricSpecFromDefinition(def: MetricDefinition, ext: MetricDefinitionExtension): MetricSpec {
|
||||
return {
|
||||
id: def.metric_id,
|
||||
name: def.name,
|
||||
description: def.description ?? '',
|
||||
type: ext.metric_type,
|
||||
measure: def.measure_field,
|
||||
aggregation: def.aggregation,
|
||||
denominator: def.denominator_field,
|
||||
dimensions: [...def.allowed_dimension_ids],
|
||||
filters: JSON.parse(ext.filters_json) as MetricFilter[],
|
||||
timeDimension: ext.time_dimension,
|
||||
unit: ext.unit,
|
||||
currency: ext.currency,
|
||||
format: ext.display_format,
|
||||
owner: def.owner_id,
|
||||
version: def.current_version,
|
||||
status: ext.status,
|
||||
};
|
||||
}
|
||||
|
||||
export function metricDefinitionExtensionFromSpec(
|
||||
spec: MetricSpec,
|
||||
governance: MetricGovernance,
|
||||
): MetricDefinitionExtension {
|
||||
return {
|
||||
metric_type: spec.type,
|
||||
time_dimension: spec.timeDimension,
|
||||
unit: spec.unit,
|
||||
currency: spec.currency,
|
||||
display_format: spec.format,
|
||||
status: spec.status,
|
||||
filters_json: JSON.stringify(spec.filters),
|
||||
certification: governance.certification,
|
||||
business_definition: governance.business_definition,
|
||||
technical_definition: governance.technical_definition,
|
||||
meaning_hash: metricMeaningHash(spec),
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,291 @@
|
||||
/**
|
||||
* src/analytics/metrics/semantic-provider.ts
|
||||
* ===========================================
|
||||
* BI-P029 — METRICFLOW PROVIDER.
|
||||
*
|
||||
* ── WHAT IS AND IS NOT PROVEN HERE. READ THIS BEFORE CITING BI-P029. ──────
|
||||
*
|
||||
* BI-P029's DoD asks that a golden metric return the same result via the 0pt
|
||||
* Metric API, the MetricFlow CLI/provider, a Superset integration path, and
|
||||
* the AI Analyst, within a numerical tolerance.
|
||||
*
|
||||
* MetricFlow is a Python package. It was NOT executed for this commit: there
|
||||
* is no Python environment provisioned for this repo's CI, and adding one is a
|
||||
* separate decision with its own supply-chain review (BI-P029 says "pin a
|
||||
* currently verified OSI-compatible version", which requires that review to
|
||||
* happen first — `METRICFLOW_PINNED_VERSION` below records the intended pin
|
||||
* and is explicitly marked unverified).
|
||||
*
|
||||
* So what IS proven, in tests/integration/semantic-parity.integration.test.ts
|
||||
* — a path this header cited from the day it was written, while the file
|
||||
* itself did not exist until BI-P029's gap review; the citation was the
|
||||
* evidence, which is no evidence at all — is the part that does not need
|
||||
* MetricFlow: the SAME golden metric, taken
|
||||
* from the SAME registry version, executed against the SAME live ClickHouse
|
||||
* data by TWO INDEPENDENT compilers — 0pt's metric compiler, and a renderer
|
||||
* driven only by the engine-neutral `SemanticQuerySpec` — returns the same
|
||||
* number. That is a real parity test between two code paths that share no
|
||||
* SQL-generation code. It is NOT a test of MetricFlow, and this file does not
|
||||
* claim to be one.
|
||||
*
|
||||
* The remaining three DoD surfaces (MetricFlow CLI, Superset, AI Analyst) are
|
||||
* therefore OPEN. Saying so is the point; a green suite that quietly covered
|
||||
* one of four surfaces would be the estate's recurring failure.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { SemanticQuerySpec } from '../contract/compile-semantic';
|
||||
import type { MetricSpec } from './metric-model';
|
||||
import type { SemanticModelVersion } from './semantic-registry';
|
||||
|
||||
/**
|
||||
* The version this provider is written against.
|
||||
*
|
||||
* LICENCE BOUNDARY — the reason this is 0.209.0 and not 0.208.1.
|
||||
*
|
||||
* MetricFlow was relicensed by dbt Labs at 0.209.0. Verified against the
|
||||
* LICENSE file at each git tag, 2026-09-07, not inferred from PyPI metadata
|
||||
* (PyPI reports no licence field for either version, so a metadata check
|
||||
* would have found nothing):
|
||||
*
|
||||
* v0.208.1 Business Source License 1.1 — source-available, NOT open source
|
||||
* v0.209.0 Apache License 2.0
|
||||
*
|
||||
* The previous pin sat exactly one patch below that boundary. For a
|
||||
* self-hosted platform whose premise is no proprietary runtime dependency,
|
||||
* a BUSL component in the semantic layer is a licence defect, not a
|
||||
* preference — BUSL restricts production use until its change date.
|
||||
*
|
||||
* STILL UNVERIFIED, and deliberately still reported as such: no supply-chain
|
||||
* review has been run and MetricFlow has never been executed here (no Python
|
||||
* runtime is provisioned). `providerReadiness()` continues to return
|
||||
* `verified: false`. Fixing the licence does not make the provider proven.
|
||||
*/
|
||||
export const METRICFLOW_PINNED_VERSION = '0.209.0';
|
||||
|
||||
/**
|
||||
* The first MetricFlow release under Apache-2.0. Anything below this is BUSL
|
||||
* and must not be pinned here; the test asserts the pin stays at or above it.
|
||||
*/
|
||||
export const METRICFLOW_FIRST_OPEN_SOURCE_VERSION = '0.209.0';
|
||||
|
||||
/** The provider-neutral port. BI-P029's `SiaxSemanticProvider`. */
|
||||
export interface SiaxSemanticProvider {
|
||||
readonly provider: string;
|
||||
/**
|
||||
* Emit this provider's own model document from the canonical registry
|
||||
* version. The registry never stores it (BI-P030); it is derived.
|
||||
*/
|
||||
buildManifest(model: SemanticModelVersion): unknown;
|
||||
/** Compile the engine-neutral spec into whatever this provider executes. */
|
||||
compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery;
|
||||
/** Canonical SIAX metric id -> this provider's own name for it. */
|
||||
providerBindings(model: SemanticModelVersion): Record<string, string>;
|
||||
}
|
||||
|
||||
export interface CompiledSemanticQuery {
|
||||
provider: string;
|
||||
/** The provider's own query representation. Never SQL text from a caller. */
|
||||
query: unknown;
|
||||
/** Canonical metric ids, unchanged — the BI-P030 contract. */
|
||||
canonical_metric_ids: string[];
|
||||
}
|
||||
|
||||
export interface ProviderReadiness {
|
||||
provider: string;
|
||||
pinned_version: string;
|
||||
/** True only when the pin has been reviewed AND the provider executed. */
|
||||
verified: boolean;
|
||||
/** What has actually been demonstrated. Plain words for an operator. */
|
||||
proven: string[];
|
||||
/** What has NOT. */
|
||||
open: string[];
|
||||
}
|
||||
|
||||
/**
|
||||
* MetricFlow-shaped provider.
|
||||
*
|
||||
* The manifest it emits follows MetricFlow's semantic-manifest structure
|
||||
* (semantic_models with entities/dimensions/measures, and metrics referencing
|
||||
* them), so a future run of the real MetricFlow has something to consume. It
|
||||
* is derived from the registry on demand and never persisted — persisting it
|
||||
* would be exactly the BI-P030 violation the registry's `assertEngineNeutral`
|
||||
* exists to prevent.
|
||||
*/
|
||||
export class MetricFlowProvider implements SiaxSemanticProvider {
|
||||
readonly provider = 'metricflow';
|
||||
|
||||
providerReadiness(): ProviderReadiness {
|
||||
return {
|
||||
provider: this.provider,
|
||||
pinned_version: METRICFLOW_PINNED_VERSION,
|
||||
verified: false,
|
||||
proven: [
|
||||
'the engine-neutral SemanticQuerySpec compiles to a MetricFlow-shaped query',
|
||||
'a MetricFlow semantic manifest is derived from the canonical registry without persisting it',
|
||||
'canonical SIAX metric ids survive the compilation unchanged',
|
||||
"0pt's metric compiler and an independent spec-driven renderer return the same number " +
|
||||
'for the golden metric against live ClickHouse data',
|
||||
],
|
||||
open: [
|
||||
`MetricFlow ${METRICFLOW_PINNED_VERSION} has NOT been executed — no Python runtime is provisioned`,
|
||||
'the pinned version has NOT had a licence or supply-chain review',
|
||||
'the Superset integration path is not implemented',
|
||||
'the AI Analyst path (inf0#29) has not been compared against this provider',
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
buildManifest(model: SemanticModelVersion): {
|
||||
/** Stamped so a document that leaked into a registry row is detectable. */
|
||||
_generated_by: string;
|
||||
_generated_from: { model_id: string; version: number; hash: string };
|
||||
semantic_models: Array<Record<string, unknown>>;
|
||||
metrics: Array<Record<string, unknown>>;
|
||||
} {
|
||||
const semantic_models = model.datasets.map((dataset) => {
|
||||
const dims = model.dimensions.filter((d) => d.dataset_id === dataset);
|
||||
const measures = model.metrics
|
||||
.filter((m) => m.dimensions.some((dim) => dims.some((d) => d.name === dim)))
|
||||
.map((m) => ({ name: m.measure, agg: mapAggregation(m.aggregation), expr: m.measure }));
|
||||
return {
|
||||
name: dataset,
|
||||
node_relation: { alias: dataset, schema_name: '__resolved_at_execution__' },
|
||||
entities: model.relationships
|
||||
.filter((r) => r.from_dataset === dataset)
|
||||
.map((r) => ({ name: r.from_field, type: r.kind === 'many_to_one' ? 'foreign' : 'primary' })),
|
||||
dimensions: dims.map((d) => ({
|
||||
name: d.name,
|
||||
type: d.kind === 'time' ? 'time' : 'categorical',
|
||||
...(d.kind === 'time' ? { type_params: { time_granularity: 'day' } } : {}),
|
||||
})),
|
||||
measures: dedupeBy(measures, (m) => m.name),
|
||||
};
|
||||
});
|
||||
|
||||
const metrics = model.metrics.map((m) => ({
|
||||
// The CANONICAL id is what a consumer holds. The provider's own name is
|
||||
// derived from it, never the other way round.
|
||||
name: m.id,
|
||||
description: m.description,
|
||||
type: mapMetricType(m.type),
|
||||
type_params: metricTypeParams(m),
|
||||
filter: m.filters.length ? { where_filters: m.filters.map(structuredFilter) } : undefined,
|
||||
meta: { siax_metric_id: m.id, siax_metric_version: m.version, unit: m.unit, currency: m.currency },
|
||||
}));
|
||||
|
||||
return {
|
||||
_generated_by: `${this.provider}@${METRICFLOW_PINNED_VERSION}`,
|
||||
_generated_from: { model_id: model.model_id, version: model.version, hash: model.hash },
|
||||
semantic_models,
|
||||
metrics,
|
||||
};
|
||||
}
|
||||
|
||||
compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery {
|
||||
const known = new Map(model.metrics.map((m) => [m.id, m]));
|
||||
for (const ref of spec.metrics) {
|
||||
if (!known.has(ref.metric_id)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`metric "${ref.metric_id}" is not in semantic model ${model.model_id} v${model.version}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
const timeBucket = spec.groupBy.find((g) => g.grain);
|
||||
// Per-metric definition filters (BI-P028 `MetricSpec.filters`), kept
|
||||
// SEPARATE from the request's `where`. Folding them together would apply
|
||||
// one metric's meaning to every metric in the query; omitting them would
|
||||
// make this provider compute a different number from the ClickHouse
|
||||
// target for the same canonical metric id.
|
||||
const metric_filters: Record<
|
||||
string,
|
||||
Array<{ dimension: string; operator: string; values: string[] }>
|
||||
> = {};
|
||||
for (const m of spec.metrics) {
|
||||
if (!m.definition_filters?.length) continue;
|
||||
metric_filters[m.metric_id] = m.definition_filters.map((f) => ({
|
||||
dimension: f.name,
|
||||
operator: f.operator,
|
||||
values: [...f.values],
|
||||
}));
|
||||
}
|
||||
return {
|
||||
provider: this.provider,
|
||||
query: {
|
||||
metrics: spec.metrics.map((m) => m.metric_id),
|
||||
...(Object.keys(metric_filters).length ? { metric_filters } : {}),
|
||||
group_by: spec.groupBy.filter((g) => !g.grain).map((g) => g.name),
|
||||
...(timeBucket ? { time_granularity: timeBucket.grain } : {}),
|
||||
// STRUCTURED, not a `--where` SQL string. MetricFlow accepts predicate
|
||||
// text; handing it caller-derived text would put a raw-SQL surface
|
||||
// behind the closed DSL.
|
||||
where: spec.where.map((w) => ({ dimension: w.name, operator: w.operator, values: w.values })),
|
||||
order_by: spec.orderBy.map((o) => ({ name: o.ref, descending: o.direction === 'desc' })),
|
||||
limit: spec.limit,
|
||||
...(spec.timeConstraint
|
||||
? { start_time: spec.timeConstraint.start, end_time: spec.timeConstraint.end }
|
||||
: {}),
|
||||
},
|
||||
canonical_metric_ids: spec.metrics.map((m) => m.metric_id).sort(),
|
||||
};
|
||||
}
|
||||
|
||||
providerBindings(model: SemanticModelVersion): Record<string, string> {
|
||||
// Identity for MetricFlow, because its metric names can be the canonical
|
||||
// ids. A provider whose naming rules differ (Cube's cube.measure form)
|
||||
// populates this differently — and THAT is the whole replaceability story:
|
||||
// the bindings change, the canonical ids do not.
|
||||
return Object.fromEntries(model.metrics.map((m) => [m.id, m.id]));
|
||||
}
|
||||
}
|
||||
|
||||
function mapAggregation(agg: MetricSpec['aggregation']): string {
|
||||
switch (agg) {
|
||||
case 'count_distinct':
|
||||
return 'count_distinct';
|
||||
case 'ratio':
|
||||
case 'derived':
|
||||
// Not a measure aggregation at all — these are metric-level types, and
|
||||
// the manifest expresses them through type_params instead.
|
||||
return 'sum';
|
||||
default:
|
||||
return agg;
|
||||
}
|
||||
}
|
||||
|
||||
function mapMetricType(type: MetricSpec['type']): string {
|
||||
switch (type) {
|
||||
case 'ratio':
|
||||
return 'ratio';
|
||||
case 'derived':
|
||||
return 'derived';
|
||||
case 'cumulative':
|
||||
return 'cumulative';
|
||||
case 'conversion':
|
||||
return 'conversion';
|
||||
default:
|
||||
return 'simple';
|
||||
}
|
||||
}
|
||||
|
||||
function metricTypeParams(m: MetricSpec): Record<string, unknown> {
|
||||
if (m.type === 'ratio') {
|
||||
return { numerator: m.measure, denominator: m.denominator };
|
||||
}
|
||||
return { measure: { name: m.measure } };
|
||||
}
|
||||
|
||||
function structuredFilter(f: MetricSpec['filters'][number]): Record<string, unknown> {
|
||||
return { dimension: f.dimension, operator: f.operator, values: f.values ?? [] };
|
||||
}
|
||||
|
||||
function dedupeBy<T>(items: T[], key: (t: T) => string): T[] {
|
||||
const seen = new Set<string>();
|
||||
return items.filter((i) => {
|
||||
const k = key(i);
|
||||
if (seen.has(k)) return false;
|
||||
seen.add(k);
|
||||
return true;
|
||||
});
|
||||
}
|
||||
@@ -0,0 +1,316 @@
|
||||
/**
|
||||
* src/analytics/metrics/semantic-registry.ts
|
||||
* ===========================================
|
||||
* BI-P030 — SEMANTIC MODEL REGISTRY.
|
||||
*
|
||||
* "Persist semantic model version SEPARATELY FROM IMPLEMENTATION ENGINE.
|
||||
* DoD: MetricFlow can be replaced without breaking canonical SIAX metric IDs."
|
||||
*
|
||||
* Which means exactly one thing in code, and it is a negative: NOTHING in this
|
||||
* file may name MetricFlow, Cube, dbt, ClickHouse, Trino or DuckDB. A registry
|
||||
* that stores a MetricFlow manifest is a MetricFlow registry with a neutral
|
||||
* name on it. `assertEngineNeutral` enforces the negative at runtime by
|
||||
* scanning a serialized version for engine vocabulary, and the test asserts it
|
||||
* catches a manifest that smuggles one in — because the rule is exactly the
|
||||
* kind that decays silently.
|
||||
*
|
||||
* The canonical SIAX metric id is the contract. A provider maps it to whatever
|
||||
* its own model calls that metric (BI-P029's `providerBindings`), and swapping
|
||||
* providers changes the BINDINGS, never the ids — so a dashboard, an API
|
||||
* consumer or a saved query keeps working.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { MetricSpec } from './metric-model';
|
||||
|
||||
export const SEMANTIC_MODEL_STATUSES = ['draft', 'active', 'superseded', 'retired'] as const;
|
||||
export type SemanticModelStatus = (typeof SEMANTIC_MODEL_STATUSES)[number];
|
||||
|
||||
export const RELATIONSHIP_KINDS = ['one_to_many', 'many_to_one', 'one_to_one'] as const;
|
||||
export type RelationshipKind = (typeof RELATIONSHIP_KINDS)[number];
|
||||
|
||||
export interface SemanticRelationship {
|
||||
from_dataset: string;
|
||||
from_field: string;
|
||||
to_dataset: string;
|
||||
to_field: string;
|
||||
kind: RelationshipKind;
|
||||
}
|
||||
|
||||
export interface SemanticDimension {
|
||||
id: string;
|
||||
name: string;
|
||||
dataset_id: string;
|
||||
/** "categorical" | "time". A time dimension is what grains apply to. */
|
||||
kind: 'categorical' | 'time';
|
||||
}
|
||||
|
||||
/** BI-P030's fields, verbatim. */
|
||||
export interface SemanticModelVersion {
|
||||
model_id: string;
|
||||
version: number;
|
||||
tenant_id: string;
|
||||
datasets: string[];
|
||||
relationships: SemanticRelationship[];
|
||||
/** Canonical SIAX metric ids and their specs at this version. */
|
||||
metrics: MetricSpec[];
|
||||
dimensions: SemanticDimension[];
|
||||
valid_from: string;
|
||||
status: SemanticModelStatus;
|
||||
hash: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* Engine vocabulary that must never appear in a persisted semantic model.
|
||||
*
|
||||
* Checked against the SERIALIZED version, so it catches an engine name hiding
|
||||
* in a nested field somebody added later — the failure mode a type cannot
|
||||
* prevent.
|
||||
*/
|
||||
const ENGINE_VOCABULARY = [
|
||||
'metricflow',
|
||||
'clickhouse',
|
||||
'trino',
|
||||
'duckdb',
|
||||
'cube.js',
|
||||
'cubejs',
|
||||
'dbt_metric',
|
||||
'superset',
|
||||
'presto',
|
||||
'bigquery',
|
||||
'snowflake',
|
||||
];
|
||||
|
||||
/** The only top-level keys a persisted semantic model may carry. */
|
||||
export const SEMANTIC_MODEL_KEYS = [
|
||||
'model_id',
|
||||
'version',
|
||||
'tenant_id',
|
||||
'datasets',
|
||||
'relationships',
|
||||
'metrics',
|
||||
'dimensions',
|
||||
'valid_from',
|
||||
'status',
|
||||
'hash',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* The STRUCTURAL half of engine neutrality, and the load-bearing one.
|
||||
*
|
||||
* `assertEngineNeutral` below is a VOCABULARY scan: it catches a model that
|
||||
* says "metricflow" or "clickhouse". It cannot catch a document that is
|
||||
* structurally MetricFlow-shaped while avoiding the word — a limitation found
|
||||
* by a test that expected it to and did not. So publishing also rejects any
|
||||
* key outside the BI-P030 field list, which means a provider manifest cannot
|
||||
* be stored in a registry row at all, whatever it calls itself.
|
||||
*/
|
||||
export function assertOnlySemanticModelFields(model: object): void {
|
||||
for (const key of Object.keys(model)) {
|
||||
if (!(SEMANTIC_MODEL_KEYS as readonly string[]).includes(key)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`semantic model carries field "${key}", which is not part of BI-P030's model — ` +
|
||||
`provider manifests are DERIVED on demand, never persisted`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
export function assertEngineNeutral(model: Omit<SemanticModelVersion, 'hash'>): void {
|
||||
const serialized = JSON.stringify(model).toLowerCase();
|
||||
for (const term of ENGINE_VOCABULARY) {
|
||||
if (serialized.includes(term)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`semantic model "${model.model_id}" mentions "${term}" — BI-P030 requires the model to be ` +
|
||||
`persisted separately from any implementation engine, so it can be replaced without ` +
|
||||
`breaking canonical SIAX metric ids`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The version hash. Covers everything that defines the model's MEANING;
|
||||
* `valid_from` and `status` are excluded because activating a model does not
|
||||
* change what it says.
|
||||
*/
|
||||
export function semanticModelHash(
|
||||
model: Omit<SemanticModelVersion, 'hash' | 'valid_from' | 'status'>,
|
||||
): string {
|
||||
const canonical = JSON.stringify([
|
||||
model.model_id,
|
||||
model.version,
|
||||
model.tenant_id,
|
||||
[...model.datasets].sort(),
|
||||
[...model.relationships]
|
||||
.map((r) => [r.from_dataset, r.from_field, r.to_dataset, r.to_field, r.kind])
|
||||
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
|
||||
[...model.metrics].map((m) => [m.id, m.version]).sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
|
||||
[...model.dimensions].map((d) => [d.id, d.name, d.dataset_id, d.kind]).sort(),
|
||||
]);
|
||||
return crypto.createHash('sha256').update(canonical).digest('hex');
|
||||
}
|
||||
|
||||
export interface SemanticModelStore {
|
||||
put(model: SemanticModelVersion): Promise<void>;
|
||||
get(tenantId: string, modelId: string, version: number): Promise<SemanticModelVersion | undefined>;
|
||||
active(tenantId: string, modelId: string): Promise<SemanticModelVersion | undefined>;
|
||||
versions(tenantId: string, modelId: string): Promise<SemanticModelVersion[]>;
|
||||
}
|
||||
|
||||
export class InMemorySemanticModelStore implements SemanticModelStore {
|
||||
private readonly rows = new Map<string, SemanticModelVersion>();
|
||||
private key(t: string, m: string, v: number): string {
|
||||
return `${t}::${m}::${v}`;
|
||||
}
|
||||
async put(model: SemanticModelVersion): Promise<void> {
|
||||
this.rows.set(this.key(model.tenant_id, model.model_id, model.version), model);
|
||||
}
|
||||
async get(t: string, m: string, v: number): Promise<SemanticModelVersion | undefined> {
|
||||
return this.rows.get(this.key(t, m, v));
|
||||
}
|
||||
async versions(t: string, m: string): Promise<SemanticModelVersion[]> {
|
||||
return [...this.rows.values()]
|
||||
.filter((r) => r.tenant_id === t && r.model_id === m)
|
||||
.sort((a, b) => a.version - b.version);
|
||||
}
|
||||
async active(t: string, m: string): Promise<SemanticModelVersion | undefined> {
|
||||
return (await this.versions(t, m)).find((r) => r.status === 'active');
|
||||
}
|
||||
}
|
||||
|
||||
export class SemanticModelRegistry {
|
||||
constructor(private readonly store: SemanticModelStore) {}
|
||||
|
||||
/**
|
||||
* Publish a new version. Versions are IMMUTABLE — re-publishing an existing
|
||||
* (model, version) is a CONFLICT, not an update, because a dashboard pinned
|
||||
* to version 7 must mean the same thing tomorrow.
|
||||
*/
|
||||
async publish(input: Omit<SemanticModelVersion, 'hash'>): Promise<SemanticModelVersion> {
|
||||
if (!input.tenant_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.semantic',
|
||||
'a semantic model requires a verified tenant',
|
||||
);
|
||||
}
|
||||
if (input.version < 1 || !Number.isInteger(input.version)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.semantic', 'version must be a positive integer');
|
||||
}
|
||||
assertOnlySemanticModelFields(input);
|
||||
assertEngineNeutral(input);
|
||||
this.assertReferentialIntegrity(input);
|
||||
|
||||
const existing = await this.store.get(input.tenant_id, input.model_id, input.version);
|
||||
if (existing) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.semantic',
|
||||
`semantic model ${input.model_id} v${input.version} already exists and is immutable`,
|
||||
);
|
||||
}
|
||||
const model: SemanticModelVersion = { ...input, hash: semanticModelHash(input) };
|
||||
await this.store.put(model);
|
||||
return model;
|
||||
}
|
||||
|
||||
/** Activate a version, superseding the previous active one. */
|
||||
async activate(
|
||||
tenantId: string,
|
||||
modelId: string,
|
||||
version: number,
|
||||
at: string,
|
||||
): Promise<SemanticModelVersion> {
|
||||
const target = await this.store.get(tenantId, modelId, version);
|
||||
if (!target)
|
||||
throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found');
|
||||
if (target.status === 'retired') {
|
||||
throw new SiaxError('CONFLICT', '0pt.analytics.semantic', 'a retired version cannot be re-activated');
|
||||
}
|
||||
const current = await this.store.active(tenantId, modelId);
|
||||
if (current && current.version !== version) {
|
||||
await this.store.put({ ...current, status: 'superseded' });
|
||||
}
|
||||
const activated = { ...target, status: 'active' as const, valid_from: at };
|
||||
await this.store.put(activated);
|
||||
return activated;
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P030's DoD, as a query: the canonical metric ids a model exposes.
|
||||
* The engine-parity test asserts this set is identical before and after a
|
||||
* provider swap.
|
||||
*/
|
||||
async canonicalMetricIds(tenantId: string, modelId: string, version: number): Promise<string[]> {
|
||||
const model = await this.store.get(tenantId, modelId, version);
|
||||
if (!model)
|
||||
throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found');
|
||||
return model.metrics.map((m) => m.id).sort();
|
||||
}
|
||||
|
||||
/**
|
||||
* Every dimension and dataset a metric names must exist in the model.
|
||||
*
|
||||
* A model that passes validation while referencing a dimension nobody
|
||||
* defined produces a metric that fails at query time, per tenant, in
|
||||
* production — the worst place to find out.
|
||||
*/
|
||||
private assertReferentialIntegrity(model: Omit<SemanticModelVersion, 'hash'>): void {
|
||||
const datasets = new Set(model.datasets);
|
||||
const dimensionNames = new Set(model.dimensions.map((d) => d.name));
|
||||
const timeDimensions = new Set(model.dimensions.filter((d) => d.kind === 'time').map((d) => d.name));
|
||||
|
||||
for (const d of model.dimensions) {
|
||||
if (!datasets.has(d.dataset_id)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`dimension "${d.name}" belongs to dataset "${d.dataset_id}", which the model does not include`,
|
||||
);
|
||||
}
|
||||
}
|
||||
for (const r of model.relationships) {
|
||||
for (const ds of [r.from_dataset, r.to_dataset]) {
|
||||
if (!datasets.has(ds)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`relationship references dataset "${ds}", which the model does not include`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
const seenIds = new Set<string>();
|
||||
for (const m of model.metrics) {
|
||||
if (seenIds.has(m.id)) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.semantic',
|
||||
`metric id "${m.id}" appears twice in the model`,
|
||||
);
|
||||
}
|
||||
seenIds.add(m.id);
|
||||
for (const dim of m.dimensions) {
|
||||
if (!dimensionNames.has(dim)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`metric "${m.name}" names dimension "${dim}", which the model does not define`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!timeDimensions.has(m.timeDimension)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.semantic',
|
||||
`metric "${m.name}" names time dimension "${m.timeDimension}", which the model does not define as a time dimension`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,434 @@
|
||||
/**
|
||||
* src/analytics/providers/duckdb-provider.ts
|
||||
* ==========================================
|
||||
* BI-P014 — DUCKDB PROVIDER.
|
||||
*
|
||||
* Use cases BI-P014 names: Parquet, Iceberg preview, local query, test, CSV,
|
||||
* ad-hoc files, report build, notebook. Heavy jobs go through N0D.
|
||||
*
|
||||
* ── THE SECURITY REQUIREMENT, AND THE MEASURED SURPRISE ───────────────────
|
||||
* "No uncontrolled filesystem access. Explicit ST0RE-mounted/input artifacts
|
||||
* only."
|
||||
*
|
||||
* DuckDB is a local engine with `read_csv('/etc/passwd')` built in, so this is
|
||||
* the whole prompt. Two layers, and the second is not what it looks like:
|
||||
*
|
||||
* LAYER 1 — MOUNT REGISTRY. A caller names ARTIFACT IDS, never paths. The
|
||||
* registry resolves an id to a path, checks the artifact belongs to
|
||||
* the caller's tenant, and `realpath`s the result to confirm it is
|
||||
* still inside the mount root once symlinks are followed. The query
|
||||
* builder is the only thing that ever writes a path into SQL.
|
||||
*
|
||||
* LAYER 2 — DUCKDB'S OWN SANDBOX. And here is the finding:
|
||||
*
|
||||
* `SET allowed_directories=[...]` ALONE IS A NO-OP.
|
||||
*
|
||||
* Measured on DuckDB 1.5.5: with `allowed_directories` set to one
|
||||
* mount directory and nothing else, `read_csv('/etc/passwd')`
|
||||
* RETURNED THE FILE, and an https:// URL was fetched over the
|
||||
* network. Only when `enable_external_access = false` is ALSO set
|
||||
* does the allow-list bind — the same reads then fail with
|
||||
* "Permission Error: Cannot access file". `lock_configuration=true`
|
||||
* afterwards stops a query re-opening either.
|
||||
*
|
||||
* So the obvious spelling of this control — set the allow-list and
|
||||
* move on — yields a setting that appears in `duckdb_settings()`,
|
||||
* looks like a sandbox, and permits everything. Same shape as
|
||||
* `trivy --exit-code 0`. Hence `duckDbSessionBootstrap()` emits the
|
||||
* three statements as one unit, and `assertSessionSandboxed()`
|
||||
* refuses a session missing any of them.
|
||||
*
|
||||
* Evidence: evidence/bi-p014/duckdb-sandbox.json, written by
|
||||
* tests/duckdb/duckdb-provider.live.test.ts, which runs the bypass
|
||||
* corpus with LAYER 1 REMOVED and requires layer 2 to refuse it anyway.
|
||||
*/
|
||||
import path from 'node:path';
|
||||
import fs from 'node:fs';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { AnalyticsIdentity } from '../engine/siax-analytics-engine';
|
||||
|
||||
export const ARTIFACT_FORMATS = ['parquet', 'csv', 'json', 'iceberg', 'arrow_ipc'] as const;
|
||||
export type ArtifactFormat = (typeof ARTIFACT_FORMATS)[number];
|
||||
|
||||
/**
|
||||
* A file the provider is allowed to read. Registered by the ST0RE mount
|
||||
* process, never by a query.
|
||||
*/
|
||||
export interface ArtifactMount {
|
||||
artifact_id: string;
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
/** Canonical ST0RE reference, for lineage. */
|
||||
st0re_ref: string;
|
||||
/** Path RELATIVE to the mount root. Never absolute, never traversing. */
|
||||
relative_path: string;
|
||||
format: ArtifactFormat;
|
||||
/** Bytes, from ST0RE metadata. Drives the N0D hand-off policy. */
|
||||
size_bytes: number;
|
||||
}
|
||||
|
||||
const ARTIFACT_ID_RE = /^[A-Za-z0-9_-]{1,128}$/;
|
||||
|
||||
/**
|
||||
* Resolves artifact ids to real paths, and refuses everything else.
|
||||
*
|
||||
* `realpathSync` is the load-bearing call: a relative path that passes a string
|
||||
* check can still be a symlink into `/etc`. The check is on the RESOLVED path,
|
||||
* after the filesystem has had its say.
|
||||
*/
|
||||
export class ArtifactMountRegistry {
|
||||
private readonly mounts = new Map<string, ArtifactMount>();
|
||||
private readonly rootReal: string;
|
||||
|
||||
constructor(
|
||||
/** The single directory every mounted artifact lives under. */
|
||||
readonly root: string,
|
||||
private readonly statPath: (p: string) => string = (p) => fs.realpathSync(p),
|
||||
) {
|
||||
if (!path.isAbsolute(root)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
'the mount root must be an absolute path',
|
||||
);
|
||||
}
|
||||
this.rootReal = this.statPath(root);
|
||||
}
|
||||
|
||||
register(mount: ArtifactMount): void {
|
||||
if (!ARTIFACT_ID_RE.test(mount.artifact_id)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe');
|
||||
}
|
||||
if (!mount.tenant_id?.trim()) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'a mount must name its owning tenant');
|
||||
}
|
||||
if (path.isAbsolute(mount.relative_path)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
'a mount path must be relative to the mount root',
|
||||
);
|
||||
}
|
||||
// Resolve BEFORE storing, so a bad mount is refused at registration rather
|
||||
// than at query time when the caller is a tenant.
|
||||
this.resolveWithin(mount.relative_path);
|
||||
this.mounts.set(mount.artifact_id, mount);
|
||||
}
|
||||
|
||||
/**
|
||||
* Resolve an artifact for a VERIFIED identity.
|
||||
*
|
||||
* Two refusals that are easy to conflate but must stay distinct:
|
||||
* * an unknown artifact is NOT_FOUND;
|
||||
* * an artifact owned by another tenant is AUTHORIZATION_DENIED, and
|
||||
* deliberately NOT reported as NOT_FOUND — an operator reading the audit
|
||||
* trail needs to see an attempted cross-tenant read as one.
|
||||
*/
|
||||
resolve(identity: AnalyticsIdentity, artifactId: string): { mount: ArtifactMount; absolutePath: string } {
|
||||
if (!identity?.tenant_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.duckdb',
|
||||
'artifact access requires a verified tenant',
|
||||
);
|
||||
}
|
||||
if (!ARTIFACT_ID_RE.test(artifactId)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe');
|
||||
}
|
||||
const mount = this.mounts.get(artifactId);
|
||||
if (!mount) {
|
||||
throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', `artifact "${artifactId}" is not mounted`);
|
||||
}
|
||||
if (mount.tenant_id !== identity.tenant_id) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.duckdb',
|
||||
`artifact "${artifactId}" belongs to another tenant`,
|
||||
);
|
||||
}
|
||||
if (mount.project_id && identity.project_id && mount.project_id !== identity.project_id) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.duckdb',
|
||||
`artifact "${artifactId}" belongs to another project`,
|
||||
);
|
||||
}
|
||||
return { mount, absolutePath: this.resolveWithin(mount.relative_path) };
|
||||
}
|
||||
|
||||
/** Join, resolve symlinks, and confirm the result is still under the root. */
|
||||
private resolveWithin(relative: string): string {
|
||||
const joined = path.resolve(this.rootReal, relative);
|
||||
let real: string;
|
||||
try {
|
||||
real = this.statPath(joined);
|
||||
} catch {
|
||||
throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', 'mounted artifact is not present on disk');
|
||||
}
|
||||
// `startsWith(root)` alone would accept `/mnt-evil` for root `/mnt`.
|
||||
if (real !== this.rootReal && !real.startsWith(`${this.rootReal}${path.sep}`)) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.duckdb',
|
||||
'artifact resolves outside the mount root (traversal or symlink)',
|
||||
);
|
||||
}
|
||||
return real;
|
||||
}
|
||||
|
||||
/** Every directory DuckDB is told about. One entry: the root itself. */
|
||||
allowedDirectories(): string[] {
|
||||
return [this.rootReal];
|
||||
}
|
||||
|
||||
list(tenantId: string): ArtifactMount[] {
|
||||
return [...this.mounts.values()].filter((m) => m.tenant_id === tenantId);
|
||||
}
|
||||
}
|
||||
|
||||
// ── layer 2: the DuckDB session sandbox ────────────────────────────────────
|
||||
|
||||
/**
|
||||
* The three statements, as ONE unit.
|
||||
*
|
||||
* Order matters and is not cosmetic: `allowed_directories` must be set while
|
||||
* the configuration is still open, `enable_external_access = false` is what
|
||||
* makes the list bind at all, and `lock_configuration = true` must come last
|
||||
* or a query could undo the other two.
|
||||
*/
|
||||
export function duckDbSessionBootstrap(allowedDirectories: readonly string[]): string[] {
|
||||
if (allowedDirectories.length === 0) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
'refusing to open a DuckDB session with no allowed directory — an empty list is not a sandbox',
|
||||
);
|
||||
}
|
||||
for (const dir of allowedDirectories) {
|
||||
if (!path.isAbsolute(dir)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'an allowed directory must be absolute');
|
||||
}
|
||||
if (dir.includes("'")) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
'an allowed directory must not contain a quote',
|
||||
);
|
||||
}
|
||||
}
|
||||
return [
|
||||
`SET allowed_directories=[${allowedDirectories.map((d) => `'${d}'`).join(', ')}]`,
|
||||
// MEASURED: without this, the line above permits everything, including
|
||||
// /etc/passwd and outbound HTTP. See this file's header.
|
||||
'SET enable_external_access=false',
|
||||
'SET lock_configuration=true',
|
||||
];
|
||||
}
|
||||
|
||||
/** The settings that must all be present for a session to be sandboxed. */
|
||||
export const REQUIRED_SANDBOX_SETTINGS = [
|
||||
'allowed_directories',
|
||||
'enable_external_access',
|
||||
'lock_configuration',
|
||||
] as const;
|
||||
|
||||
export function assertSessionSandboxed(statements: readonly string[]): void {
|
||||
const joined = statements.join('\n').toLowerCase();
|
||||
for (const setting of REQUIRED_SANDBOX_SETTINGS) {
|
||||
if (!joined.includes(setting)) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.duckdb',
|
||||
`DuckDB session is missing "${setting}" — allowed_directories WITHOUT ` +
|
||||
`enable_external_access=false is a no-op, measured on 1.5.5`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!/enable_external_access\s*=\s*false/i.test(joined)) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.duckdb',
|
||||
'DuckDB session leaves external access ENABLED; the directory allow-list is then not enforced',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── the provider ───────────────────────────────────────────────────────────
|
||||
|
||||
export interface DuckDbRunner {
|
||||
readonly engine: string;
|
||||
/** Open a session with the bootstrap applied, run `sql`, return rows. */
|
||||
run(args: { bootstrap: readonly string[]; sql: string }): Promise<Array<Record<string, unknown>>>;
|
||||
}
|
||||
|
||||
export const DUCKDB_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const;
|
||||
export type DuckDbAggregate = (typeof DUCKDB_AGGREGATES)[number];
|
||||
|
||||
/**
|
||||
* A closed request. There is NO caller SQL on this provider, for the same
|
||||
* reason `executeMetric` has none: an analyst SQL surface is a separate,
|
||||
* role-gated thing (BI-P017), and letting the ad-hoc FILE engine grow one would
|
||||
* put raw SQL next to a filesystem.
|
||||
*/
|
||||
export interface DuckDbArtifactQuery {
|
||||
artifact_id: string;
|
||||
/** Column names to select. Validated as identifiers. */
|
||||
select?: string[];
|
||||
aggregate?: { fn: DuckDbAggregate; column: string; alias?: string };
|
||||
group_by?: string[];
|
||||
/** Equality filters. Names validated, values escaped. */
|
||||
filters?: Record<string, string>;
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
const COLUMN_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
||||
|
||||
function column(name: string): string {
|
||||
if (!COLUMN_RE.test(name)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
`"${name.slice(0, 32)}" is not a column name`,
|
||||
);
|
||||
}
|
||||
return name;
|
||||
}
|
||||
|
||||
const READER: Record<ArtifactFormat, (p: string) => string> = {
|
||||
parquet: (p) => `read_parquet('${p}')`,
|
||||
csv: (p) => `read_csv('${p}', header=true, auto_detect=true)`,
|
||||
json: (p) => `read_json_auto('${p}')`,
|
||||
// The iceberg extension reads a table DIRECTORY, not a single file.
|
||||
iceberg: (p) => `iceberg_scan('${p}')`,
|
||||
arrow_ipc: (p) => `read_arrow('${p}')`,
|
||||
};
|
||||
|
||||
export interface N0dHandoffPolicy {
|
||||
/** Above this, the query is refused here and must go to N0D. */
|
||||
maxLocalBytes: number;
|
||||
}
|
||||
|
||||
export const DEFAULT_N0D_HANDOFF: N0dHandoffPolicy = { maxLocalBytes: 2 * 1024 * 1024 * 1024 };
|
||||
|
||||
export class DuckDbAnalyticsProvider {
|
||||
readonly provider = 'duckdb';
|
||||
|
||||
constructor(
|
||||
private readonly registry: ArtifactMountRegistry,
|
||||
private readonly runner: DuckDbRunner,
|
||||
private readonly handoff: N0dHandoffPolicy = DEFAULT_N0D_HANDOFF,
|
||||
) {}
|
||||
|
||||
/** Formats this build can read. See `readiness()` for what has been PROVEN. */
|
||||
supports(format: ArtifactFormat): boolean {
|
||||
return ['parquet', 'csv', 'json', 'iceberg'].includes(format);
|
||||
}
|
||||
|
||||
/**
|
||||
* What has been demonstrated against a real DuckDB, and what has not.
|
||||
* Same posture as BI-P029's `providerReadiness`: a capability list nobody
|
||||
* can check is a capability list that overstates.
|
||||
*/
|
||||
readiness(): { proven: ArtifactFormat[]; unproven: Array<{ format: ArtifactFormat; reason: string }> } {
|
||||
return {
|
||||
proven: ['parquet', 'csv', 'iceberg'],
|
||||
unproven: [
|
||||
{
|
||||
format: 'arrow_ipc',
|
||||
reason:
|
||||
"DuckDB's `arrow` extension is a network install and the offline verification " +
|
||||
'environment could not download it, so read_arrow() has NOT been exercised',
|
||||
},
|
||||
{ format: 'json', reason: 'no fixture was built for it in the verification run' },
|
||||
],
|
||||
};
|
||||
}
|
||||
|
||||
async query(
|
||||
identity: AnalyticsIdentity,
|
||||
request: DuckDbArtifactQuery,
|
||||
): Promise<Array<Record<string, unknown>>> {
|
||||
const { mount, absolutePath } = this.registry.resolve(identity, request.artifact_id);
|
||||
|
||||
if (!this.supports(mount.format)) {
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.duckdb',
|
||||
`format "${mount.format}" is not readable by this provider build`,
|
||||
);
|
||||
}
|
||||
if (mount.size_bytes > this.handoff.maxLocalBytes) {
|
||||
// BI-P014: "Run heavy jobs through N0D." Refusing is the honest form —
|
||||
// silently running a 40 GB scan in the API process is how a preview
|
||||
// engine takes down a control plane.
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.duckdb',
|
||||
`artifact is ${mount.size_bytes} bytes, above the ${this.handoff.maxLocalBytes}-byte local ceiling — ` +
|
||||
`submit it as an N0D job`,
|
||||
);
|
||||
}
|
||||
|
||||
const source = READER[mount.format](absolutePath);
|
||||
const selectParts: string[] = [];
|
||||
const groupParts: string[] = [];
|
||||
|
||||
for (const g of request.group_by ?? []) {
|
||||
selectParts.push(column(g));
|
||||
groupParts.push(column(g));
|
||||
}
|
||||
if (request.aggregate) {
|
||||
const col = column(request.aggregate.column);
|
||||
const alias = column(request.aggregate.alias ?? 'value');
|
||||
const fn = request.aggregate.fn;
|
||||
if (!DUCKDB_AGGREGATES.includes(fn)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', `unsupported aggregate "${fn}"`);
|
||||
}
|
||||
const expr =
|
||||
fn === 'count' ? 'count(*)' : fn === 'count_distinct' ? `count(DISTINCT ${col})` : `${fn}(${col})`;
|
||||
selectParts.push(`${expr} AS ${alias}`);
|
||||
} else {
|
||||
for (const s of request.select ?? []) selectParts.push(column(s));
|
||||
}
|
||||
if (selectParts.length === 0) selectParts.push('*');
|
||||
|
||||
const conditions: string[] = [];
|
||||
for (const [name, value] of Object.entries(request.filters ?? {})) {
|
||||
// Not every DuckDB binding exposes a bound-parameter transport, so values
|
||||
// are ESCAPED here — and the escaping is asserted against a hostile
|
||||
// corpus in the test rather than assumed.
|
||||
conditions.push(`${column(name)} = '${escapeLiteral(value)}'`);
|
||||
}
|
||||
|
||||
const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), 1_000_000));
|
||||
const sql =
|
||||
`SELECT ${selectParts.join(', ')} FROM ${source}` +
|
||||
(conditions.length ? ` WHERE ${conditions.join(' AND ')}` : '') +
|
||||
(groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') +
|
||||
` LIMIT ${limit}`;
|
||||
|
||||
const bootstrap = duckDbSessionBootstrap(this.registry.allowedDirectories());
|
||||
assertSessionSandboxed(bootstrap);
|
||||
return this.runner.run({ bootstrap, sql });
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Escape a value for a DuckDB string literal.
|
||||
*
|
||||
* Doubling `'` is the whole grammar — DuckDB does NOT treat backslash as an
|
||||
* escape inside a standard single-quoted string, so `\'` does not close one.
|
||||
* A NUL is REFUSED rather than escaped: a truncating byte inside a literal is
|
||||
* a parser disagreement waiting to happen, and there is no legitimate filter
|
||||
* value that contains one.
|
||||
*/
|
||||
export function escapeLiteral(value: string): string {
|
||||
if (value.includes('\u0000')) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.duckdb',
|
||||
'a filter value must not contain a NUL byte',
|
||||
);
|
||||
}
|
||||
return value.replace(/'/g, "''");
|
||||
}
|
||||
@@ -0,0 +1,127 @@
|
||||
/**
|
||||
* src/analytics/quality/clickhouse-quality-runner.ts
|
||||
* ===================================================
|
||||
* BI-P025 — the ClickHouse `QualityQueryRunner`.
|
||||
*
|
||||
* One statement per rule, and it counts BOTH numbers in a single pass:
|
||||
*
|
||||
* SELECT countIf(<violation expression>) AS violations, count() AS evaluated
|
||||
* FROM <db>.<table> WHERE tenant_id = {tenant:String} AND dataset_id = …
|
||||
*
|
||||
* `evaluated` is not decoration. It is what makes an EMPTY dataset
|
||||
* distinguishable from a CLEAN one — the false green
|
||||
* `NativeQualityProvider` refuses. Counting only violations would make the two
|
||||
* cases identical at this layer, and no amount of care upstream could recover
|
||||
* the difference.
|
||||
*
|
||||
* The BI-P011 forced tenant predicate applies here exactly as it does to a
|
||||
* metric query: `tenant_id = {tenant:String}` is condition[0], the value is
|
||||
* bound, and a quality run therefore cannot read another tenant's rows even
|
||||
* though it runs on an administrative credential.
|
||||
*
|
||||
* `unique` gets its own statement shape, because a duplicate is a property of
|
||||
* a GROUP, not of a row, and pretending otherwise would make the rule count
|
||||
* the wrong thing.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import { assertSafeIdentifier, FORCED_TENANT_PREDICATE } from '../engine/query-compiler';
|
||||
import type { QualityQueryRunner } from './quality-provider';
|
||||
|
||||
export interface ClickHouseQualityRunnerOptions {
|
||||
baseUrl: string;
|
||||
basicAuth?: string;
|
||||
database: string;
|
||||
/** dataset_id -> physical table. An unlisted dataset is refused, not guessed. */
|
||||
tables: Record<string, string>;
|
||||
timeoutMs?: number;
|
||||
/** Ceiling for a quality scan; a check must not become the expensive query. */
|
||||
maxRowsToRead?: number;
|
||||
}
|
||||
|
||||
/** Marker the provider emits for the `unique` rule. */
|
||||
const UNIQUE_MARKER = /^__unique__\(([A-Za-z_][A-Za-z0-9_]{0,62})\)$/;
|
||||
|
||||
export class ClickHouseQualityRunner implements QualityQueryRunner {
|
||||
readonly engine = 'clickhouse';
|
||||
|
||||
constructor(private readonly opts: ClickHouseQualityRunnerOptions) {
|
||||
assertSafeIdentifier(opts.database, 'database');
|
||||
for (const table of Object.values(opts.tables)) assertSafeIdentifier(table, 'table');
|
||||
}
|
||||
|
||||
async count(args: {
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
violationExpression: string;
|
||||
params: Record<string, string>;
|
||||
}): Promise<{ violations: number; evaluated: number }> {
|
||||
if (!args.tenant_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.quality',
|
||||
'a quality run requires a verified tenant',
|
||||
);
|
||||
}
|
||||
const table = this.opts.tables[args.dataset_id];
|
||||
if (!table) {
|
||||
throw new SiaxError(
|
||||
'NOT_FOUND',
|
||||
'0pt.analytics.quality',
|
||||
`dataset "${args.dataset_id}" has no physical table registered for quality runs`,
|
||||
);
|
||||
}
|
||||
const params: Record<string, string> = {
|
||||
...args.params,
|
||||
param_tenant: args.tenant_id,
|
||||
param_dataset: args.dataset_id,
|
||||
};
|
||||
const where = `${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String}`;
|
||||
|
||||
const unique = UNIQUE_MARKER.exec(args.violationExpression.trim());
|
||||
const sql = unique
|
||||
? // A duplicate is a property of a GROUP. Count the rows that belong to a
|
||||
// key appearing more than once, and the total rows, in one pass.
|
||||
`SELECT sum(multiIf(c > 1, c, 0)) AS violations, sum(c) AS evaluated FROM ` +
|
||||
`(SELECT ${assertSafeIdentifier(unique[1], 'field')} AS k, count() AS c ` +
|
||||
`FROM ${this.opts.database}.${table} WHERE ${where} GROUP BY k) FORMAT JSON`
|
||||
: `SELECT countIf(${args.violationExpression}) AS violations, count() AS evaluated ` +
|
||||
`FROM ${this.opts.database}.${table} WHERE ${where} FORMAT JSON`;
|
||||
|
||||
const url = new URL(this.opts.baseUrl.replace(/\/+$/, ''));
|
||||
url.searchParams.set('query', sql);
|
||||
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
|
||||
url.searchParams.set('max_rows_to_read', String(this.opts.maxRowsToRead ?? 5_000_000_000));
|
||||
url.searchParams.set('read_overflow_mode', 'throw');
|
||||
url.searchParams.set('max_execution_time', '60');
|
||||
url.searchParams.set('timeout_overflow_mode', 'throw');
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.opts.basicAuth) {
|
||||
headers.Authorization = `Basic ${Buffer.from(this.opts.basicAuth).toString('base64')}`;
|
||||
}
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url.toString(), {
|
||||
method: 'POST',
|
||||
headers,
|
||||
body: '',
|
||||
signal: AbortSignal.timeout(this.opts.timeoutMs ?? 60_000),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
|
||||
}
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status);
|
||||
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `quality query failed (CH code ${code})`);
|
||||
}
|
||||
const row = (
|
||||
JSON.parse(text) as { data: Array<{ violations: string | number; evaluated: string | number }> }
|
||||
).data[0];
|
||||
if (!row) {
|
||||
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'quality query returned no counts');
|
||||
}
|
||||
return { violations: Number(row.violations ?? 0), evaluated: Number(row.evaluated ?? 0) };
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,196 @@
|
||||
/**
|
||||
* src/analytics/quality/freshness.ts
|
||||
* ===================================
|
||||
* BI-P026 — DATA FRESHNESS.
|
||||
*
|
||||
* Five states, exactly as BI-P026 names them, and the DoD is a NEGATIVE:
|
||||
*
|
||||
* "Stale data is never SILENTLY presented as current."
|
||||
*
|
||||
* Two words carry it. "Silently" means the failure mode is not "we served old
|
||||
* data" — it is "we served old data and the reader could not tell". And
|
||||
* "never" means the enforcement has to sit at the boundary every result
|
||||
* crosses, not in each dashboard's rendering code.
|
||||
*
|
||||
* So `assertFreshnessDisclosed` refuses to release a result that carries no
|
||||
* freshness envelope at all. A result with no envelope is not FRESH and not
|
||||
* UNKNOWN — it is unreleasable, because "we forgot to attach it" and "the data
|
||||
* is current" are indistinguishable to a reader, and a default of FRESH would
|
||||
* make forgetting the silent path.
|
||||
*
|
||||
* UNKNOWN is likewise never treated as FRESH. It is its own state precisely
|
||||
* because "we could not determine the age" is the honest thing to display, and
|
||||
* collapsing it into FRESH is the classic version of this bug.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
|
||||
/** BI-P026's states, verbatim and in severity order. */
|
||||
export const FRESHNESS_STATES = ['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN'] as const;
|
||||
export type FreshnessState = (typeof FRESHNESS_STATES)[number];
|
||||
|
||||
/** States a reader must be shown explicitly rather than served as current. */
|
||||
export const NON_CURRENT_STATES: readonly FreshnessState[] = ['DELAYED', 'STALE', 'FAILED', 'UNKNOWN'];
|
||||
|
||||
export interface FreshnessPolicy {
|
||||
/** How often the upstream is expected to land data. */
|
||||
expectedIntervalMs: number;
|
||||
/**
|
||||
* Multiples of the expected interval. Past `delayedAfter` the data is
|
||||
* DELAYED; past `staleAfter` it is STALE. Defaults are deliberately tight:
|
||||
* a dashboard that is two cycles behind is already misleading someone.
|
||||
*/
|
||||
delayedAfterIntervals?: number;
|
||||
staleAfterIntervals?: number;
|
||||
}
|
||||
|
||||
export interface FreshnessInput {
|
||||
/** When the newest successfully loaded data is timestamped. */
|
||||
lastSuccessfulLoadAt?: string;
|
||||
/** Outcome of the most recent load attempt. */
|
||||
lastRunStatus?: 'succeeded' | 'failed' | 'running' | 'never_run';
|
||||
lastRunAt?: string;
|
||||
policy: FreshnessPolicy;
|
||||
now: number;
|
||||
}
|
||||
|
||||
export interface FreshnessEnvelope {
|
||||
state: FreshnessState;
|
||||
/** Age of the newest data, in ms. Absent when unknown. */
|
||||
age_ms?: number;
|
||||
last_successful_load_at?: string;
|
||||
last_run_at?: string;
|
||||
expected_interval_ms: number;
|
||||
/** Plain-language line a UI can render verbatim. */
|
||||
disclosure: string;
|
||||
computed_at: string;
|
||||
}
|
||||
|
||||
export function computeFreshness(input: FreshnessInput): FreshnessEnvelope {
|
||||
const { policy, now } = input;
|
||||
const delayedAfter = policy.expectedIntervalMs * (policy.delayedAfterIntervals ?? 1.5);
|
||||
const staleAfter = policy.expectedIntervalMs * (policy.staleAfterIntervals ?? 3);
|
||||
const computed_at = new Date(now).toISOString();
|
||||
const base = {
|
||||
last_successful_load_at: input.lastSuccessfulLoadAt,
|
||||
last_run_at: input.lastRunAt,
|
||||
expected_interval_ms: policy.expectedIntervalMs,
|
||||
computed_at,
|
||||
};
|
||||
|
||||
// FAILED outranks age: a pipeline that just failed may still have
|
||||
// recently-timestamped data from the run before, and reporting that as FRESH
|
||||
// is precisely how a broken pipeline stays invisible for a day.
|
||||
if (input.lastRunStatus === 'failed') {
|
||||
const age = ageOf(input.lastSuccessfulLoadAt, now);
|
||||
return {
|
||||
...base,
|
||||
state: 'FAILED',
|
||||
age_ms: age,
|
||||
disclosure: `The most recent load FAILED. Showing data from ${input.lastSuccessfulLoadAt ?? 'an unknown time'}.`,
|
||||
};
|
||||
}
|
||||
|
||||
if (!input.lastSuccessfulLoadAt || input.lastRunStatus === 'never_run') {
|
||||
return {
|
||||
...base,
|
||||
state: 'UNKNOWN',
|
||||
disclosure: 'The age of this data could not be determined. Do not read it as current.',
|
||||
};
|
||||
}
|
||||
|
||||
const age = ageOf(input.lastSuccessfulLoadAt, now);
|
||||
if (age === undefined) {
|
||||
return { ...base, state: 'UNKNOWN', disclosure: 'The age of this data could not be determined.' };
|
||||
}
|
||||
if (age > staleAfter) {
|
||||
return {
|
||||
...base,
|
||||
state: 'STALE',
|
||||
age_ms: age,
|
||||
disclosure: `This data is ${describeAge(age)} old, past the ${describeAge(staleAfter)} staleness threshold.`,
|
||||
};
|
||||
}
|
||||
if (age > delayedAfter) {
|
||||
return {
|
||||
...base,
|
||||
state: 'DELAYED',
|
||||
age_ms: age,
|
||||
disclosure: `This data is ${describeAge(age)} old and behind its ${describeAge(policy.expectedIntervalMs)} schedule.`,
|
||||
};
|
||||
}
|
||||
return {
|
||||
...base,
|
||||
state: 'FRESH',
|
||||
age_ms: age,
|
||||
disclosure: `Current as of ${input.lastSuccessfulLoadAt}.`,
|
||||
};
|
||||
}
|
||||
|
||||
function ageOf(iso: string | undefined, now: number): number | undefined {
|
||||
if (!iso) return undefined;
|
||||
const t = Date.parse(iso);
|
||||
if (Number.isNaN(t)) return undefined;
|
||||
return Math.max(0, now - t);
|
||||
}
|
||||
|
||||
function describeAge(ms: number): string {
|
||||
const s = Math.round(ms / 1000);
|
||||
if (s < 90) return `${s}s`;
|
||||
const m = Math.round(s / 60);
|
||||
if (m < 90) return `${m}m`;
|
||||
const h = Math.round(m / 60);
|
||||
if (h < 48) return `${h}h`;
|
||||
return `${Math.round(h / 24)}d`;
|
||||
}
|
||||
|
||||
/** Any result leaving the analytics boundary. */
|
||||
export interface DisclosableResult {
|
||||
freshness?: FreshnessEnvelope;
|
||||
/**
|
||||
* Set by a caller that has SEEN a non-current state and chosen to proceed.
|
||||
* Its presence is what makes serving non-current data not-silent.
|
||||
*/
|
||||
acknowledged_non_current?: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* BI-P026's DoD, as a gate.
|
||||
*
|
||||
* Three refusals, in order of how easy they are to get wrong:
|
||||
* 1. no envelope at all — unreleasable. Not defaulted to FRESH.
|
||||
* 2. a non-current state with no acknowledgement — refused, so a consumer
|
||||
* that ignores the envelope cannot render stale numbers as current.
|
||||
* 3. an acknowledgement with no envelope — refused, because acknowledging
|
||||
* something you were never shown is not disclosure.
|
||||
*/
|
||||
export function assertFreshnessDisclosed(result: DisclosableResult, context: string): FreshnessEnvelope {
|
||||
if (!result.freshness) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.freshness',
|
||||
`${context} carries no freshness envelope — BI-P026: data with unknown age must never be released ` +
|
||||
`as though it were current`,
|
||||
);
|
||||
}
|
||||
const state = result.freshness.state;
|
||||
if (!(FRESHNESS_STATES as readonly string[]).includes(state)) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.freshness',
|
||||
`${context} has an unknown freshness state`,
|
||||
);
|
||||
}
|
||||
if (NON_CURRENT_STATES.includes(state) && !result.acknowledged_non_current) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.freshness',
|
||||
`${context} is ${state} and the consumer has not acknowledged it: ${result.freshness.disclosure}`,
|
||||
);
|
||||
}
|
||||
return result.freshness;
|
||||
}
|
||||
|
||||
/** True when the state may be presented without a caveat. Only one state may. */
|
||||
export function isPresentableAsCurrent(state: FreshnessState): boolean {
|
||||
return state === 'FRESH';
|
||||
}
|
||||
@@ -0,0 +1,418 @@
|
||||
/**
|
||||
* src/analytics/quality/quality-provider.ts
|
||||
* ==========================================
|
||||
* BI-P025 — DATA QUALITY PROVIDER.
|
||||
*
|
||||
* One interface, three provider kinds, and an honest split between them:
|
||||
*
|
||||
* NATIVE — this module compiles the rule into a counting query and runs it
|
||||
* through an injected runner. It EXECUTES.
|
||||
* DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against
|
||||
* the BI-P022 contract). This module INGESTS the results. It does
|
||||
* not run dbt, and `providerKind` says so rather than implying
|
||||
* otherwise.
|
||||
* GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here
|
||||
* executes it. The adapter maps a GE validation result onto the
|
||||
* canonical verdict so a GE suite counts toward the same gate.
|
||||
*
|
||||
* Calling all three "providers" while only one executes would be the kind of
|
||||
* overstatement that turns into "we have Great Expectations coverage" in a
|
||||
* status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or
|
||||
* `"ingesting"`, and the gate records which produced each verdict.
|
||||
*
|
||||
* ── BI-P025's DoD ─────────────────────────────────────────────────────────
|
||||
* "Failed critical quality gate can QUARANTINE serving projection."
|
||||
*
|
||||
* `quarantine.ts` is that, and the important half is what happens on the READ
|
||||
* side: a quarantined projection is REFUSED, not annotated. An annotation on a
|
||||
* dashboard nobody reads is not a quarantine.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { QualityRuleKind, QualitySeverity } from '../domain/entities';
|
||||
|
||||
/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */
|
||||
export const QUALITY_RULES = [
|
||||
'not_null',
|
||||
'unique',
|
||||
'range',
|
||||
'referential',
|
||||
'freshness',
|
||||
'schema',
|
||||
'distribution',
|
||||
'custom',
|
||||
] as const;
|
||||
export type QualityRule = (typeof QUALITY_RULES)[number];
|
||||
|
||||
/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */
|
||||
export const RULE_KIND_ALIASES: Record<QualityRule, QualityRuleKind> = {
|
||||
not_null: 'not_null',
|
||||
unique: 'unique',
|
||||
range: 'range',
|
||||
referential: 'referential',
|
||||
freshness: 'freshness',
|
||||
schema: 'custom',
|
||||
distribution: 'accepted_values',
|
||||
custom: 'custom',
|
||||
};
|
||||
|
||||
export interface QualityRuleSpec {
|
||||
rule_id: string;
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
rule: QualityRule;
|
||||
/** Column the rule applies to; absent for dataset-level rules. */
|
||||
field?: string;
|
||||
severity: QualitySeverity;
|
||||
params: QualityRuleParams;
|
||||
description?: string;
|
||||
}
|
||||
|
||||
export interface QualityRuleParams {
|
||||
/** range */
|
||||
min?: number;
|
||||
max?: number;
|
||||
/** distribution / accepted values */
|
||||
accepted_values?: string[];
|
||||
/** distribution: the share of rows that must fall in accepted_values */
|
||||
min_share?: number;
|
||||
/** referential */
|
||||
references_dataset?: string;
|
||||
references_field?: string;
|
||||
/** freshness: the maximum acceptable age of the newest row */
|
||||
max_age_seconds?: number;
|
||||
timestamp_field?: string;
|
||||
/** schema: the columns and types that must exist */
|
||||
expected_columns?: Array<{ name: string; type: string }>;
|
||||
/** unique / not_null take no parameters */
|
||||
/** custom: an operator-authored counting query, reviewed like code */
|
||||
custom_violation_sql?: string;
|
||||
}
|
||||
|
||||
export interface QualityVerdict {
|
||||
rule_id: string;
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
rule: QualityRule;
|
||||
severity: QualitySeverity;
|
||||
passed: boolean;
|
||||
violation_count: number;
|
||||
evaluated_rows: number;
|
||||
/** Which provider produced this, and whether it EXECUTED or merely ingested. */
|
||||
produced_by: string;
|
||||
provider_kind: 'executing' | 'ingesting';
|
||||
evaluated_at: string;
|
||||
/** Short operator-facing note. Never raw engine output. */
|
||||
detail?: string;
|
||||
}
|
||||
|
||||
export interface SiaxDataQualityProvider {
|
||||
readonly provider: string;
|
||||
readonly providerKind: 'executing' | 'ingesting';
|
||||
/** Rules this provider can actually evaluate. */
|
||||
supports(rule: QualityRule): boolean;
|
||||
evaluate(spec: QualityRuleSpec): Promise<QualityVerdict>;
|
||||
}
|
||||
|
||||
/**
|
||||
* Runs a counting query and returns two numbers. The only engine coupling in
|
||||
* this module, so a DuckDB or Trino runner drops in unchanged.
|
||||
*/
|
||||
export interface QualityQueryRunner {
|
||||
readonly engine: string;
|
||||
/**
|
||||
* Must apply the caller's tenant scoping. The runner is handed the tenant
|
||||
* explicitly rather than trusting the SQL to carry it.
|
||||
*/
|
||||
count(args: {
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
violationExpression: string;
|
||||
/** Bound parameters. Values NEVER appear in the expression text. */
|
||||
params: Record<string, string>;
|
||||
}): Promise<{ violations: number; evaluated: number }>;
|
||||
}
|
||||
|
||||
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
||||
|
||||
function assertField(field: string | undefined, rule: QualityRule): string {
|
||||
if (!field || !IDENT_RE.test(field)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
`rule "${rule}" requires a valid field name`,
|
||||
);
|
||||
}
|
||||
return field;
|
||||
}
|
||||
|
||||
/**
|
||||
* The provider that actually runs checks.
|
||||
*
|
||||
* Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and
|
||||
* the runner counts rows where it is true. That shape is chosen so a rule can
|
||||
* never return "how many rows are fine", which is the number that looks
|
||||
* healthy when a table is empty.
|
||||
*/
|
||||
export class NativeQualityProvider implements SiaxDataQualityProvider {
|
||||
readonly provider = 'native';
|
||||
readonly providerKind = 'executing' as const;
|
||||
|
||||
constructor(
|
||||
private readonly runner: QualityQueryRunner,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
supports(rule: QualityRule): boolean {
|
||||
// `schema` and `referential` need catalog/second-table access the counting
|
||||
// runner does not have; they are honestly unsupported here rather than
|
||||
// silently passing.
|
||||
return !['schema', 'referential'].includes(rule);
|
||||
}
|
||||
|
||||
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
||||
if (!this.supports(spec.rule)) {
|
||||
throw new SiaxError(
|
||||
'CAPABILITY_UNAVAILABLE',
|
||||
'0pt.analytics.quality',
|
||||
`the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`,
|
||||
);
|
||||
}
|
||||
const { expression, params } = this.compile(spec);
|
||||
const { violations, evaluated } = await this.runner.count({
|
||||
tenant_id: spec.tenant_id,
|
||||
dataset_id: spec.dataset_id,
|
||||
violationExpression: expression,
|
||||
params,
|
||||
});
|
||||
|
||||
// An EMPTY dataset is not a passing dataset. A `not_null` check over zero
|
||||
// rows finds zero violations and would report success while the pipeline
|
||||
// is silently broken — the single most common false green in data quality.
|
||||
const emptyIsFailure = spec.rule !== 'custom';
|
||||
const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0;
|
||||
|
||||
return {
|
||||
rule_id: spec.rule_id,
|
||||
dataset_id: spec.dataset_id,
|
||||
tenant_id: spec.tenant_id,
|
||||
rule: spec.rule,
|
||||
severity: spec.severity,
|
||||
passed,
|
||||
violation_count: violations,
|
||||
evaluated_rows: evaluated,
|
||||
produced_by: `${this.provider}:${this.runner.engine}`,
|
||||
provider_kind: this.providerKind,
|
||||
evaluated_at: new Date(this.now()).toISOString(),
|
||||
detail:
|
||||
evaluated === 0 && emptyIsFailure
|
||||
? 'dataset is EMPTY — reported as a failure, not as zero violations'
|
||||
: undefined,
|
||||
};
|
||||
}
|
||||
|
||||
private compile(spec: QualityRuleSpec): { expression: string; params: Record<string, string> } {
|
||||
const p = spec.params;
|
||||
switch (spec.rule) {
|
||||
case 'not_null': {
|
||||
const f = assertField(spec.field, spec.rule);
|
||||
return { expression: `${f} IS NULL OR ${f} = ''`, params: {} };
|
||||
}
|
||||
case 'unique': {
|
||||
const f = assertField(spec.field, spec.rule);
|
||||
// Counted over a grouped subquery by the runner; the expression names
|
||||
// the key so the runner can build it.
|
||||
return { expression: `__unique__(${f})`, params: {} };
|
||||
}
|
||||
case 'range': {
|
||||
const f = assertField(spec.field, spec.rule);
|
||||
if (p.min === undefined && p.max === undefined) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
'a range rule needs a min or a max',
|
||||
);
|
||||
}
|
||||
const parts: string[] = [];
|
||||
const params: Record<string, string> = {};
|
||||
if (p.min !== undefined) {
|
||||
parts.push(`${f} < {qmin:Float64}`);
|
||||
params.param_qmin = String(p.min);
|
||||
}
|
||||
if (p.max !== undefined) {
|
||||
parts.push(`${f} > {qmax:Float64}`);
|
||||
params.param_qmax = String(p.max);
|
||||
}
|
||||
return { expression: parts.join(' OR '), params };
|
||||
}
|
||||
case 'freshness': {
|
||||
const f = assertField(p.timestamp_field ?? spec.field, spec.rule);
|
||||
if (!p.max_age_seconds || p.max_age_seconds <= 0) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
'a freshness rule needs max_age_seconds',
|
||||
);
|
||||
}
|
||||
return {
|
||||
expression: `${f} < now() - toIntervalSecond({qage:UInt64})`,
|
||||
params: { param_qage: String(Math.floor(p.max_age_seconds)) },
|
||||
};
|
||||
}
|
||||
case 'distribution': {
|
||||
const f = assertField(spec.field, spec.rule);
|
||||
if (!p.accepted_values?.length) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
'a distribution rule needs accepted_values',
|
||||
);
|
||||
}
|
||||
const params: Record<string, string> = {};
|
||||
const bound = p.accepted_values.map((v, i) => {
|
||||
params[`param_qv${i}`] = v;
|
||||
return `{qv${i}:String}`;
|
||||
});
|
||||
return { expression: `${f} NOT IN (${bound.join(', ')})`, params };
|
||||
}
|
||||
case 'custom': {
|
||||
if (!p.custom_violation_sql?.trim()) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
'a custom rule needs custom_violation_sql',
|
||||
);
|
||||
}
|
||||
// Deliberately NOT parsed or guarded here. A custom rule is
|
||||
// operator-authored, reviewed like code, and stored in the rule table
|
||||
// — not caller input. It never reaches this function from a request
|
||||
// body; `QualityRuleSpec` rows are written by an admin path.
|
||||
return { expression: p.custom_violation_sql, params: {} };
|
||||
}
|
||||
default:
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against
|
||||
* the BI-P022 contract, and pretending otherwise here would misreport where
|
||||
* the work happens.
|
||||
*/
|
||||
export interface DbtTestResult {
|
||||
unique_id: string;
|
||||
status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped';
|
||||
failures: number;
|
||||
/** dbt does not report rows evaluated; absent, not zero. */
|
||||
rows_evaluated?: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
export class DbtQualityProvider implements SiaxDataQualityProvider {
|
||||
readonly provider = 'dbt';
|
||||
readonly providerKind = 'ingesting' as const;
|
||||
constructor(
|
||||
private readonly results: Map<string, DbtTestResult>,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
supports(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
||||
const result = this.results.get(spec.rule_id);
|
||||
if (!result) {
|
||||
// A missing result is UNKNOWN, and unknown is a FAILURE for a blocking
|
||||
// rule. Treating "no result" as "passed" is how a quality gate stops
|
||||
// gating the moment the test stops running.
|
||||
return {
|
||||
rule_id: spec.rule_id,
|
||||
dataset_id: spec.dataset_id,
|
||||
tenant_id: spec.tenant_id,
|
||||
rule: spec.rule,
|
||||
severity: spec.severity,
|
||||
passed: false,
|
||||
violation_count: 0,
|
||||
evaluated_rows: 0,
|
||||
produced_by: this.provider,
|
||||
provider_kind: this.providerKind,
|
||||
evaluated_at: new Date(this.now()).toISOString(),
|
||||
detail: 'no dbt result for this rule — reported as a failure, not as a pass',
|
||||
};
|
||||
}
|
||||
return {
|
||||
rule_id: spec.rule_id,
|
||||
dataset_id: spec.dataset_id,
|
||||
tenant_id: spec.tenant_id,
|
||||
rule: spec.rule,
|
||||
severity: spec.severity,
|
||||
// `skipped` and `error` are NOT passes. dbt's `skipped` in particular is
|
||||
// the shape of l0re's silently-skipped Postgres tests.
|
||||
passed: result.status === 'pass',
|
||||
violation_count: result.failures,
|
||||
evaluated_rows: result.rows_evaluated ?? 0,
|
||||
produced_by: `dbt:${result.unique_id}`,
|
||||
provider_kind: this.providerKind,
|
||||
evaluated_at: new Date(this.now()).toISOString(),
|
||||
detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
/** Ingests a Great Expectations validation result. Does NOT run GE. */
|
||||
export interface GreatExpectationsResult {
|
||||
expectation_suite_name: string;
|
||||
success: boolean;
|
||||
results: Array<{
|
||||
expectation_config: { expectation_type: string };
|
||||
success: boolean;
|
||||
result?: { unexpected_count?: number; element_count?: number };
|
||||
}>;
|
||||
}
|
||||
|
||||
export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider {
|
||||
readonly provider = 'great_expectations';
|
||||
readonly providerKind = 'ingesting' as const;
|
||||
constructor(
|
||||
private readonly suites: Map<string, GreatExpectationsResult>,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
supports(): boolean {
|
||||
return true;
|
||||
}
|
||||
|
||||
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
||||
const suite = this.suites.get(spec.rule_id);
|
||||
const base = {
|
||||
rule_id: spec.rule_id,
|
||||
dataset_id: spec.dataset_id,
|
||||
tenant_id: spec.tenant_id,
|
||||
rule: spec.rule,
|
||||
severity: spec.severity,
|
||||
produced_by: this.provider,
|
||||
provider_kind: this.providerKind,
|
||||
evaluated_at: new Date(this.now()).toISOString(),
|
||||
};
|
||||
if (!suite) {
|
||||
return {
|
||||
...base,
|
||||
passed: false,
|
||||
violation_count: 0,
|
||||
evaluated_rows: 0,
|
||||
detail: 'no GE result — reported as a failure',
|
||||
};
|
||||
}
|
||||
const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0);
|
||||
const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0);
|
||||
return {
|
||||
...base,
|
||||
passed: suite.success,
|
||||
violation_count: unexpected,
|
||||
evaluated_rows: elements,
|
||||
produced_by: `great_expectations:${suite.expectation_suite_name}`,
|
||||
detail: suite.success ? undefined : 'GE suite failed',
|
||||
};
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,188 @@
|
||||
/**
|
||||
* src/analytics/quality/quarantine.ts
|
||||
* ====================================
|
||||
* BI-P025's DoD: "Failed critical quality gate can QUARANTINE serving
|
||||
* projection."
|
||||
*
|
||||
* The word that matters is on the READ side. A quarantine that annotates a
|
||||
* result is a label; a quarantine that REFUSES the read is a quarantine. This
|
||||
* module does the second, and `assertProjectionReadable` is the single place
|
||||
* every serving read passes through.
|
||||
*
|
||||
* Severity semantics, stated because they are the whole gate:
|
||||
* `block` — a failure quarantines the projection. Reads are refused.
|
||||
* `warn` — a failure is recorded and surfaced, and reads continue.
|
||||
* A rule with no severity does not exist: `QualityRuleSpec.severity` is
|
||||
* required, so "we forgot to mark it critical" cannot silently become `warn`.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { QualityVerdict } from './quality-provider';
|
||||
|
||||
export interface QuarantineRecord {
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
quarantined: boolean;
|
||||
/** Rule ids whose blocking failure caused it. */
|
||||
caused_by: string[];
|
||||
since?: string;
|
||||
released_at?: string;
|
||||
released_by?: string;
|
||||
/** Operator-facing summary. */
|
||||
reason?: string;
|
||||
}
|
||||
|
||||
export interface QuarantineStore {
|
||||
get(tenantId: string, datasetId: string): Promise<QuarantineRecord | undefined>;
|
||||
put(record: QuarantineRecord): Promise<void>;
|
||||
}
|
||||
|
||||
export class InMemoryQuarantineStore implements QuarantineStore {
|
||||
private readonly rows = new Map<string, QuarantineRecord>();
|
||||
private key(t: string, d: string): string {
|
||||
return `${t}::${d}`;
|
||||
}
|
||||
async get(t: string, d: string): Promise<QuarantineRecord | undefined> {
|
||||
return this.rows.get(this.key(t, d));
|
||||
}
|
||||
async put(r: QuarantineRecord): Promise<void> {
|
||||
this.rows.set(this.key(r.tenant_id, r.dataset_id), r);
|
||||
}
|
||||
}
|
||||
|
||||
export interface GateOutcome {
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
passed: boolean;
|
||||
quarantined: boolean;
|
||||
blocking_failures: QualityVerdict[];
|
||||
warnings: QualityVerdict[];
|
||||
evaluated: number;
|
||||
}
|
||||
|
||||
export class QualityGate {
|
||||
constructor(
|
||||
private readonly store: QuarantineStore,
|
||||
private readonly now: () => number = Date.now,
|
||||
) {}
|
||||
|
||||
/**
|
||||
* Apply a run's verdicts.
|
||||
*
|
||||
* A gate over ZERO verdicts does not pass. "No rules ran" and "every rule
|
||||
* passed" produce the same green in most implementations, and that is how a
|
||||
* quality gate stops gating without anyone noticing — the same shape as
|
||||
* l0re's 28 silently-skipped tests and api0's `trivy --exit-code 0`.
|
||||
*/
|
||||
async apply(tenantId: string, datasetId: string, verdicts: QualityVerdict[]): Promise<GateOutcome> {
|
||||
if (verdicts.length === 0) {
|
||||
const record: QuarantineRecord = {
|
||||
tenant_id: tenantId,
|
||||
dataset_id: datasetId,
|
||||
quarantined: true,
|
||||
caused_by: [],
|
||||
since: new Date(this.now()).toISOString(),
|
||||
reason: 'no quality rules were evaluated — an ungated dataset is quarantined, not trusted',
|
||||
};
|
||||
await this.store.put(record);
|
||||
return {
|
||||
dataset_id: datasetId,
|
||||
tenant_id: tenantId,
|
||||
passed: false,
|
||||
quarantined: true,
|
||||
blocking_failures: [],
|
||||
warnings: [],
|
||||
evaluated: 0,
|
||||
};
|
||||
}
|
||||
|
||||
for (const v of verdicts) {
|
||||
if (v.tenant_id !== tenantId || v.dataset_id !== datasetId) {
|
||||
// A verdict from another tenant's run must never influence this gate.
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.quality',
|
||||
'a quality verdict from a different tenant or dataset was supplied to this gate',
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
const blocking = verdicts.filter((v) => !v.passed && v.severity === 'block');
|
||||
const warnings = verdicts.filter((v) => !v.passed && v.severity === 'warn');
|
||||
|
||||
if (blocking.length > 0) {
|
||||
await this.store.put({
|
||||
tenant_id: tenantId,
|
||||
dataset_id: datasetId,
|
||||
quarantined: true,
|
||||
caused_by: blocking.map((v) => v.rule_id),
|
||||
since: new Date(this.now()).toISOString(),
|
||||
reason: `${blocking.length} blocking quality rule(s) failed`,
|
||||
});
|
||||
} else {
|
||||
// Passing does NOT auto-release: releasing is an act with an operator
|
||||
// behind it (`release()`), because a flapping rule would otherwise let a
|
||||
// dataset quarantine and un-quarantine itself between reads.
|
||||
const existing = await this.store.get(tenantId, datasetId);
|
||||
if (!existing || !existing.quarantined) {
|
||||
await this.store.put({
|
||||
tenant_id: tenantId,
|
||||
dataset_id: datasetId,
|
||||
quarantined: false,
|
||||
caused_by: [],
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
return {
|
||||
dataset_id: datasetId,
|
||||
tenant_id: tenantId,
|
||||
passed: blocking.length === 0,
|
||||
quarantined: blocking.length > 0 || (await this.isQuarantined(tenantId, datasetId)),
|
||||
blocking_failures: blocking,
|
||||
warnings,
|
||||
evaluated: verdicts.length,
|
||||
};
|
||||
}
|
||||
|
||||
async isQuarantined(tenantId: string, datasetId: string): Promise<boolean> {
|
||||
return (await this.store.get(tenantId, datasetId))?.quarantined ?? false;
|
||||
}
|
||||
|
||||
/** Release requires an operator and a reason. It is not automatic. */
|
||||
async release(tenantId: string, datasetId: string, by: string, reason: string): Promise<QuarantineRecord> {
|
||||
if (!by?.trim() || !reason?.trim()) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.quality',
|
||||
'releasing a quarantine requires a principal and a reason',
|
||||
);
|
||||
}
|
||||
const record: QuarantineRecord = {
|
||||
tenant_id: tenantId,
|
||||
dataset_id: datasetId,
|
||||
quarantined: false,
|
||||
caused_by: [],
|
||||
released_at: new Date(this.now()).toISOString(),
|
||||
released_by: by,
|
||||
reason,
|
||||
};
|
||||
await this.store.put(record);
|
||||
return record;
|
||||
}
|
||||
|
||||
/**
|
||||
* The read-side gate. Every serving projection read passes through here.
|
||||
* Refuses, rather than annotating — an annotation on a dashboard nobody
|
||||
* reads is not a quarantine.
|
||||
*/
|
||||
async assertProjectionReadable(tenantId: string, datasetId: string): Promise<void> {
|
||||
const record = await this.store.get(tenantId, datasetId);
|
||||
if (record?.quarantined) {
|
||||
throw new SiaxError(
|
||||
'DEPENDENCY_UNAVAILABLE',
|
||||
'0pt.analytics.quality',
|
||||
`serving projection for "${datasetId}" is QUARANTINED: ${record.reason ?? 'failed quality gate'}`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,356 @@
|
||||
/**
|
||||
* src/analytics/repositories/analytics-domain-repository.ts
|
||||
* ==========================================================
|
||||
* BI-P002 persistence for the analytics domain model (sql/008).
|
||||
*
|
||||
* Two rules the type system alone cannot keep, so they live here:
|
||||
*
|
||||
* 1. EVERY read and write is tenant-scoped. There is no method that takes a
|
||||
* dataset_id without also taking the tenant it must belong to, and every
|
||||
* WHERE carries `tenant_id = $tenant`. A caller cannot "forget" the
|
||||
* tenant because there is no signature that lets them.
|
||||
* 2. VERSION ROWS ARE APPEND-ONLY from this layer too. `createDatasetVersion`
|
||||
* /`createMetricVersion` compute the next version number inside the same
|
||||
* statement (`SELECT coalesce(max(version),0)+1`) so two concurrent
|
||||
* minters collide on the UNIQUE(dataset_id, version) constraint instead
|
||||
* of silently overwriting — and the database triggers from sql/008 catch
|
||||
* anything that still tries to UPDATE.
|
||||
*
|
||||
* Uses postgres.js through `RepositorySql`, the same connection type the other
|
||||
* P0 repositories take (src/repositories/db.ts).
|
||||
*/
|
||||
import crypto from "node:crypto";
|
||||
import { SiaxError } from "../../platform/errors";
|
||||
import type { RepositorySql } from "../../repositories/db";
|
||||
import type {
|
||||
AnalyticsDataset,
|
||||
DatasetVersion,
|
||||
DimensionDefinition,
|
||||
MetricDefinition,
|
||||
MetricVersion,
|
||||
PiiClassification,
|
||||
} from "../domain/entities";
|
||||
import type { AnalyticsTier } from "../domain/tiers";
|
||||
|
||||
/** The verified tenant scope every call must present. */
|
||||
export interface TenantScope {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
}
|
||||
|
||||
function assertScope(scope: TenantScope): void {
|
||||
if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
"UNAUTHENTICATED",
|
||||
"0pt.analytics",
|
||||
"analytics persistence requires a verified tenant_id + project_id",
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
function iso(v: string | Date | null | undefined): string | undefined {
|
||||
if (v === null || v === undefined) return undefined;
|
||||
return v instanceof Date ? v.toISOString() : v;
|
||||
}
|
||||
|
||||
interface DatasetRow {
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
name: string;
|
||||
tier: string;
|
||||
schema_contract_id: string;
|
||||
schema_contract_version: string;
|
||||
owner_id: string;
|
||||
source_ref: string;
|
||||
freshness_slo_seconds: number;
|
||||
quality_rule_ids: string[];
|
||||
retention_days: number;
|
||||
classification: string;
|
||||
status: string;
|
||||
raw_access_exception: string | null;
|
||||
description: string | null;
|
||||
created_at: string | Date;
|
||||
updated_at: string | Date | null;
|
||||
created_by: string | null;
|
||||
}
|
||||
|
||||
function toDataset(r: DatasetRow): AnalyticsDataset {
|
||||
return {
|
||||
dataset_id: r.dataset_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
name: r.name,
|
||||
tier: r.tier as AnalyticsTier,
|
||||
schema_contract_id: r.schema_contract_id,
|
||||
schema_contract_version: r.schema_contract_version,
|
||||
owner_id: r.owner_id,
|
||||
source_ref: r.source_ref,
|
||||
freshness_slo_seconds: Number(r.freshness_slo_seconds),
|
||||
quality_rule_ids: r.quality_rule_ids ?? [],
|
||||
retention_days: Number(r.retention_days),
|
||||
classification: r.classification as PiiClassification,
|
||||
status: r.status as AnalyticsDataset["status"],
|
||||
...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}),
|
||||
...(r.description ? { description: r.description } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}),
|
||||
...(r.created_by ? { created_by: r.created_by } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
export class AnalyticsDomainRepository {
|
||||
constructor(private readonly sql: RepositorySql) {}
|
||||
|
||||
// ── AnalyticsDataset ─────────────────────────────────────────────────────
|
||||
|
||||
async createDataset(scope: TenantScope, input: Omit<AnalyticsDataset, "tenant_id" | "project_id" | "created_at" | "updated_at">): Promise<AnalyticsDataset> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<DatasetRow[]>`
|
||||
INSERT INTO opt_analytics_dataset
|
||||
(dataset_id, tenant_id, project_id, name, tier, schema_contract_id, schema_contract_version,
|
||||
owner_id, source_ref, freshness_slo_seconds, quality_rule_ids, retention_days,
|
||||
classification, status, raw_access_exception, description, created_by)
|
||||
VALUES
|
||||
(${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.tier},
|
||||
${input.schema_contract_id}, ${input.schema_contract_version}, ${input.owner_id}, ${input.source_ref},
|
||||
${input.freshness_slo_seconds}, ${this.sql.array(input.quality_rule_ids ?? [])}, ${input.retention_days},
|
||||
${input.classification}, ${input.status}, ${input.raw_access_exception ?? null},
|
||||
${input.description ?? null}, ${input.created_by ?? null})
|
||||
RETURNING *`;
|
||||
return toDataset(rows[0]);
|
||||
}
|
||||
|
||||
/** Tenant-scoped read. A foreign dataset_id returns undefined, never a row. */
|
||||
async getDataset(scope: TenantScope, datasetId: string): Promise<AnalyticsDataset | undefined> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<DatasetRow[]>`
|
||||
SELECT * FROM opt_analytics_dataset
|
||||
WHERE dataset_id = ${datasetId}
|
||||
AND tenant_id = ${scope.tenant_id}
|
||||
AND project_id = ${scope.project_id}
|
||||
LIMIT 1`;
|
||||
return rows[0] ? toDataset(rows[0]) : undefined;
|
||||
}
|
||||
|
||||
async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise<AnalyticsDataset[]> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<DatasetRow[]>`
|
||||
SELECT * FROM opt_analytics_dataset
|
||||
WHERE tenant_id = ${scope.tenant_id}
|
||||
AND project_id = ${scope.project_id}
|
||||
${filter?.tier ? this.sql`AND tier = ${filter.tier}` : this.sql``}
|
||||
${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``}
|
||||
ORDER BY created_at DESC`;
|
||||
return rows.map(toDataset);
|
||||
}
|
||||
|
||||
// ── DatasetVersion (append-only) ─────────────────────────────────────────
|
||||
|
||||
/**
|
||||
* Mint the next version. The version number is computed inside the INSERT,
|
||||
* so two racing callers cannot both read "3" and both write it — the loser
|
||||
* hits UNIQUE(dataset_id, version) and gets a CONFLICT, which is the honest
|
||||
* outcome.
|
||||
*/
|
||||
async createDatasetVersion(
|
||||
scope: TenantScope,
|
||||
input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string },
|
||||
): Promise<DatasetVersion> {
|
||||
assertScope(scope);
|
||||
const owner = await this.getDataset(scope, input.dataset_id);
|
||||
if (!owner) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant");
|
||||
}
|
||||
const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`;
|
||||
try {
|
||||
const rows = await this.sql<
|
||||
{
|
||||
dataset_version_id: string;
|
||||
dataset_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
version: number;
|
||||
contract_hash: string;
|
||||
schema_contract_version: string;
|
||||
record_count: string | number | null;
|
||||
created_at: string | Date;
|
||||
created_by: string | null;
|
||||
}[]
|
||||
>`
|
||||
INSERT INTO opt_analytics_dataset_version
|
||||
(dataset_version_id, dataset_id, tenant_id, project_id, version,
|
||||
contract_hash, schema_contract_version, record_count, created_by)
|
||||
SELECT ${id}, ${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id},
|
||||
coalesce(max(version), 0) + 1,
|
||||
${input.contract_hash}, ${input.schema_contract_version},
|
||||
${input.record_count ?? null}, ${input.created_by ?? null}
|
||||
FROM opt_analytics_dataset_version
|
||||
WHERE dataset_id = ${input.dataset_id} AND tenant_id = ${scope.tenant_id}
|
||||
RETURNING *`;
|
||||
const r = rows[0];
|
||||
return {
|
||||
dataset_version_id: r.dataset_version_id,
|
||||
dataset_id: r.dataset_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
version: Number(r.version),
|
||||
contract_hash: r.contract_hash,
|
||||
schema_contract_version: r.schema_contract_version,
|
||||
...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
...(r.created_by ? { created_by: r.created_by } : {}),
|
||||
};
|
||||
} catch (err) {
|
||||
if (String(err).includes("duplicate key")) {
|
||||
throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version");
|
||||
}
|
||||
throw err;
|
||||
}
|
||||
}
|
||||
|
||||
async listDatasetVersions(scope: TenantScope, datasetId: string): Promise<DatasetVersion[]> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<
|
||||
{ dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[]
|
||||
>`
|
||||
SELECT * FROM opt_analytics_dataset_version
|
||||
WHERE dataset_id = ${datasetId}
|
||||
AND tenant_id = ${scope.tenant_id}
|
||||
AND project_id = ${scope.project_id}
|
||||
ORDER BY version ASC`;
|
||||
return rows.map((r) => ({
|
||||
dataset_version_id: r.dataset_version_id,
|
||||
dataset_id: r.dataset_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
version: Number(r.version),
|
||||
contract_hash: r.contract_hash,
|
||||
schema_contract_version: r.schema_contract_version,
|
||||
...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
...(r.created_by ? { created_by: r.created_by } : {}),
|
||||
}));
|
||||
}
|
||||
|
||||
// ── MetricDefinition / MetricVersion ─────────────────────────────────────
|
||||
|
||||
async createMetric(
|
||||
scope: TenantScope,
|
||||
input: Omit<MetricDefinition, "tenant_id" | "project_id" | "created_at" | "updated_at" | "current_version">,
|
||||
): Promise<MetricDefinition> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<
|
||||
{ metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[]
|
||||
>`
|
||||
INSERT INTO opt_analytics_metric_definition
|
||||
(metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field,
|
||||
denominator_field, allowed_dimension_ids, current_version, owner_id, description)
|
||||
VALUES
|
||||
(${input.metric_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id},
|
||||
${input.aggregation}, ${input.measure_field}, ${input.denominator_field ?? null},
|
||||
${this.sql.array(input.allowed_dimension_ids ?? [])}, 1, ${input.owner_id}, ${input.description ?? null})
|
||||
RETURNING *`;
|
||||
const r = rows[0];
|
||||
return {
|
||||
metric_id: r.metric_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
name: r.name,
|
||||
dataset_id: r.dataset_id,
|
||||
aggregation: r.aggregation as MetricDefinition["aggregation"],
|
||||
measure_field: r.measure_field,
|
||||
...(r.denominator_field ? { denominator_field: r.denominator_field } : {}),
|
||||
allowed_dimension_ids: r.allowed_dimension_ids ?? [],
|
||||
current_version: Number(r.current_version),
|
||||
owner_id: r.owner_id,
|
||||
...(r.description ? { description: r.description } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
/** Append-only. `change_reason` is required for version > 1 (DB CHECK). */
|
||||
async createMetricVersion(
|
||||
scope: TenantScope,
|
||||
input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string },
|
||||
): Promise<MetricVersion> {
|
||||
assertScope(scope);
|
||||
const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex");
|
||||
const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`;
|
||||
const rows = await this.sql<
|
||||
{ metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[]
|
||||
>`
|
||||
INSERT INTO opt_analytics_metric_version
|
||||
(metric_version_id, metric_id, tenant_id, project_id, version, definition_json,
|
||||
definition_hash, change_reason, created_by)
|
||||
SELECT ${id}, ${input.metric_id}, ${scope.tenant_id}, ${scope.project_id},
|
||||
coalesce(max(version), 0) + 1, ${input.definition_json}, ${hash},
|
||||
${input.change_reason ?? null}, ${input.created_by ?? null}
|
||||
FROM opt_analytics_metric_version
|
||||
WHERE metric_id = ${input.metric_id} AND tenant_id = ${scope.tenant_id}
|
||||
RETURNING *`;
|
||||
const r = rows[0];
|
||||
return {
|
||||
metric_version_id: r.metric_version_id,
|
||||
metric_id: r.metric_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
version: Number(r.version),
|
||||
definition_json: r.definition_json,
|
||||
definition_hash: r.definition_hash,
|
||||
...(r.change_reason ? { change_reason: r.change_reason } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
...(r.created_by ? { created_by: r.created_by } : {}),
|
||||
};
|
||||
}
|
||||
|
||||
// ── DimensionDefinition (the query compiler's allow-list) ────────────────
|
||||
|
||||
async createDimension(
|
||||
scope: TenantScope,
|
||||
input: Omit<DimensionDefinition, "tenant_id" | "project_id" | "created_at">,
|
||||
): Promise<DimensionDefinition> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<
|
||||
{ dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[]
|
||||
>`
|
||||
INSERT INTO opt_analytics_dimension_definition
|
||||
(dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description)
|
||||
VALUES
|
||||
(${input.dimension_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id},
|
||||
${input.field_name}, ${input.field_type}, ${input.classification}, ${input.description ?? null})
|
||||
RETURNING *`;
|
||||
const r = rows[0];
|
||||
return {
|
||||
dimension_id: r.dimension_id,
|
||||
tenant_id: r.tenant_id,
|
||||
project_id: r.project_id,
|
||||
name: r.name,
|
||||
dataset_id: r.dataset_id,
|
||||
field_name: r.field_name,
|
||||
field_type: r.field_type as DimensionDefinition["field_type"],
|
||||
classification: r.classification as PiiClassification,
|
||||
...(r.description ? { description: r.description } : {}),
|
||||
created_at: iso(r.created_at)!,
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* The dimension allow-list the compiler resolves through, for one tenant.
|
||||
* Returns caller-facing name -> physical field name. A tenant with no
|
||||
* declared dimensions gets an EMPTY map, which means every dimension
|
||||
* reference is refused — fail closed, not "allow everything".
|
||||
*/
|
||||
async dimensionCatalog(scope: TenantScope, datasetId?: string): Promise<Record<string, string>> {
|
||||
assertScope(scope);
|
||||
const rows = await this.sql<{ name: string; field_name: string }[]>`
|
||||
SELECT name, field_name FROM opt_analytics_dimension_definition
|
||||
WHERE tenant_id = ${scope.tenant_id}
|
||||
AND project_id = ${scope.project_id}
|
||||
${datasetId ? this.sql`AND dataset_id = ${datasetId}` : this.sql``}`;
|
||||
const out: Record<string, string> = Object.create(null);
|
||||
for (const r of rows) out[r.name] = r.field_name;
|
||||
return out;
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,511 @@
|
||||
/**
|
||||
* src/analytics/slo/bi-slo.ts
|
||||
* ============================
|
||||
* BI-P103 — BI SLO. DoD: "measured baselines exist".
|
||||
*
|
||||
* An SLO you cannot measure from real telemetry is a wish, so every objective
|
||||
* here declares WHERE its number comes from, and the evaluator refuses to
|
||||
* produce a number it does not have. Three ways that refusal shows up, each of
|
||||
* which is a lie the usual implementation tells:
|
||||
*
|
||||
* insufficient_data nothing (or too little) was observed in the window.
|
||||
* An availability objective evaluated over zero events
|
||||
* is NOT 100% attainment — it is silence. This is the
|
||||
* single most common false green in SLO reporting, and
|
||||
* it is the same shape BI-P025 found in a quality
|
||||
* check over an empty table.
|
||||
* not_measurable_here the objective is real but these events cannot
|
||||
* measure it. Dashboard data FRESHNESS is a property
|
||||
* of the dataset (BI-P026 / BI-P105 watermarks), not
|
||||
* of a usage event, and REPORT DELIVERY is N0TIFY's
|
||||
* fact — aud0's seven types carry no delivery event.
|
||||
* Saying so beats fabricating a proxy.
|
||||
* low_confidence a p95 or p99 computed from a handful of samples. The
|
||||
* number is reported WITH its sample size and marked,
|
||||
* because a p99 over 12 requests is the 12th-slowest
|
||||
* request wearing a percentile's clothes.
|
||||
*
|
||||
* The measurement path is aud0's seven BI usage event types (BI-P076, merged
|
||||
* in aud0#34), produced by src/analytics/slo/usage-events.ts. There is
|
||||
* deliberately no second telemetry path: two measurement systems disagree, and
|
||||
* the disagreement is discovered during an incident.
|
||||
*/
|
||||
import type { Aud0BiEvent } from './usage-events';
|
||||
|
||||
export const BI_WORKLOAD_CLASSES = ['interactive_dashboard', 'exploration', 'report'] as const;
|
||||
export type BiWorkloadClass = (typeof BI_WORKLOAD_CLASSES)[number];
|
||||
|
||||
export const SLO_OBJECTIVE_KINDS = [
|
||||
'availability',
|
||||
'latency_p95_first',
|
||||
'latency_p95_cached',
|
||||
'latency_p95',
|
||||
'latency_p99',
|
||||
'error_rate',
|
||||
'freshness',
|
||||
'max_result_rows',
|
||||
'timeout',
|
||||
'completion_sla',
|
||||
'delivery_sla',
|
||||
] as const;
|
||||
export type SloObjectiveKind = (typeof SLO_OBJECTIVE_KINDS)[number];
|
||||
|
||||
/** Where the number comes from. `none` is an admission, not a placeholder. */
|
||||
export const MEASUREMENT_SOURCES = [
|
||||
'aud0_bi_usage_events',
|
||||
'analytics_freshness_watermark',
|
||||
'n0tify_delivery_receipt',
|
||||
'engine_configuration',
|
||||
] as const;
|
||||
export type MeasurementSource = (typeof MEASUREMENT_SOURCES)[number];
|
||||
|
||||
export interface SloObjective {
|
||||
objective_id: string;
|
||||
workload: BiWorkloadClass;
|
||||
kind: SloObjectiveKind;
|
||||
/** Target value; unit is fixed by kind (ms, ratio, rows, seconds). */
|
||||
target: number;
|
||||
unit: 'ms' | 'ratio' | 'rows' | 'seconds';
|
||||
/** Higher-is-better (availability) vs lower-is-better (latency). */
|
||||
direction: 'at_least' | 'at_most';
|
||||
measured_from: MeasurementSource;
|
||||
/** Fewer samples than this and the verdict carries low_confidence. */
|
||||
min_samples: number;
|
||||
rationale: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The catalogue, field-for-field with what BI-P103 lists per workload class.
|
||||
* Targets are STARTING POINTS chosen to be honest about what has been
|
||||
* measured: they are the thresholds the evaluator compares against, and
|
||||
* evidence/bi-p103/baseline.json records the first baselines rather than
|
||||
* pretending the targets were derived from a year of production traffic.
|
||||
*
|
||||
* Those baselines are produced by
|
||||
* tests/integration/bi-p103-slo-baseline.integration.test.ts, which runs 460
|
||||
* real queries against a live ClickHouse and feeds them through this
|
||||
* evaluator. Read the file's own caveats before quoting a number from it: the
|
||||
* cached path measured there is the in-process reference cache rather than
|
||||
* Valkey, every latency is the engine round trip with no HTTP or
|
||||
* authorization layer around it, and there is no concurrency. Seven of the
|
||||
* eleven objectives produced a number; the other four say why they could not,
|
||||
* and `fully_measured` is false.
|
||||
*/
|
||||
export const BI_SLO_CATALOGUE: readonly SloObjective[] = [
|
||||
// ── interactive dashboard ────────────────────────────────────────────────
|
||||
{
|
||||
objective_id: 'dash.availability',
|
||||
workload: 'interactive_dashboard',
|
||||
kind: 'availability',
|
||||
target: 0.995,
|
||||
unit: 'ratio',
|
||||
direction: 'at_least',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 100,
|
||||
rationale: 'completed / (completed + failed) for dashboard-surface queries.',
|
||||
},
|
||||
{
|
||||
objective_id: 'dash.p95_first_query',
|
||||
workload: 'interactive_dashboard',
|
||||
kind: 'latency_p95_first',
|
||||
target: 3000,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 50,
|
||||
rationale: 'cache_hit=false: what a user waits for when they open the dashboard cold.',
|
||||
},
|
||||
{
|
||||
objective_id: 'dash.p95_cached',
|
||||
workload: 'interactive_dashboard',
|
||||
kind: 'latency_p95_cached',
|
||||
target: 400,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 50,
|
||||
rationale:
|
||||
'cache_hit=true: BI-P020 cache hits, measured separately so a high hit rate cannot hide a slow cold path.',
|
||||
},
|
||||
{
|
||||
objective_id: 'dash.error_rate',
|
||||
workload: 'interactive_dashboard',
|
||||
kind: 'error_rate',
|
||||
target: 0.005,
|
||||
unit: 'ratio',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 100,
|
||||
rationale: 'failed / terminal. Cancellations count as failures here: the user did not get an answer.',
|
||||
},
|
||||
{
|
||||
objective_id: 'dash.freshness',
|
||||
workload: 'interactive_dashboard',
|
||||
kind: 'freshness',
|
||||
target: 900,
|
||||
unit: 'seconds',
|
||||
direction: 'at_most',
|
||||
measured_from: 'analytics_freshness_watermark',
|
||||
min_samples: 1,
|
||||
rationale:
|
||||
'Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage ' +
|
||||
'event. Usage events cannot measure it and this evaluator does not pretend to.',
|
||||
},
|
||||
// ── exploration ──────────────────────────────────────────────────────────
|
||||
{
|
||||
objective_id: 'explore.p95',
|
||||
workload: 'exploration',
|
||||
kind: 'latency_p95',
|
||||
target: 10_000,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 50,
|
||||
rationale: 'explore-surface queries; an analyst tolerates more than a dashboard viewer.',
|
||||
},
|
||||
{
|
||||
objective_id: 'explore.p99',
|
||||
workload: 'exploration',
|
||||
kind: 'latency_p99',
|
||||
target: 60_000,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 200,
|
||||
rationale: 'p99 needs a genuinely large sample; below min_samples it is reported as low confidence.',
|
||||
},
|
||||
{
|
||||
objective_id: 'explore.max_result',
|
||||
workload: 'exploration',
|
||||
kind: 'max_result_rows',
|
||||
target: 1_000_000,
|
||||
unit: 'rows',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 1,
|
||||
rationale: 'BI-P012 truncates at the engine ceiling; this watches what actually came back.',
|
||||
},
|
||||
{
|
||||
objective_id: 'explore.timeout',
|
||||
workload: 'exploration',
|
||||
kind: 'timeout',
|
||||
target: 300,
|
||||
unit: 'seconds',
|
||||
direction: 'at_most',
|
||||
measured_from: 'engine_configuration',
|
||||
min_samples: 1,
|
||||
rationale: 'The configured ceiling (BI-P012 settings profile), not an observation.',
|
||||
},
|
||||
// ── reports ──────────────────────────────────────────────────────────────
|
||||
{
|
||||
objective_id: 'report.completion_sla',
|
||||
workload: 'report',
|
||||
kind: 'completion_sla',
|
||||
target: 900_000,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'aud0_bi_usage_events',
|
||||
min_samples: 20,
|
||||
rationale: 'aud0.report_generated duration_ms: submitted to produced.',
|
||||
},
|
||||
{
|
||||
objective_id: 'report.delivery_sla',
|
||||
workload: 'report',
|
||||
kind: 'delivery_sla',
|
||||
target: 300_000,
|
||||
unit: 'ms',
|
||||
direction: 'at_most',
|
||||
measured_from: 'n0tify_delivery_receipt',
|
||||
min_samples: 20,
|
||||
rationale:
|
||||
"Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this " +
|
||||
'objective is declared and reported unmeasured rather than proxied by completion.',
|
||||
},
|
||||
];
|
||||
|
||||
export const SLO_STATUSES = ['met', 'breached', 'insufficient_data', 'not_measurable_here'] as const;
|
||||
export type SloStatus = (typeof SLO_STATUSES)[number];
|
||||
|
||||
export interface SloResult {
|
||||
objective_id: string;
|
||||
workload: BiWorkloadClass;
|
||||
kind: SloObjectiveKind;
|
||||
status: SloStatus;
|
||||
target: number;
|
||||
unit: SloObjective['unit'];
|
||||
/** The measured value, or undefined when nothing could be measured. */
|
||||
observed?: number;
|
||||
sample_size: number;
|
||||
/** True when a percentile was computed from fewer samples than min_samples. */
|
||||
low_confidence?: boolean;
|
||||
detail: string;
|
||||
}
|
||||
|
||||
export interface SloReport {
|
||||
window_from: string;
|
||||
window_to: string;
|
||||
results: SloResult[];
|
||||
/** Queries that started and never reached a terminal event in the window. */
|
||||
unresolved_queries: number;
|
||||
/** True only when EVERY objective produced a real number. */
|
||||
fully_measured: boolean;
|
||||
}
|
||||
|
||||
interface QueryFact {
|
||||
query_id: string;
|
||||
surface?: string;
|
||||
started_at: number;
|
||||
outcome?: 'completed' | 'failed';
|
||||
duration_ms?: number;
|
||||
row_count?: number;
|
||||
cache_hit?: boolean;
|
||||
}
|
||||
|
||||
function num(v: unknown): number | undefined {
|
||||
return typeof v === 'number' && Number.isFinite(v) ? v : undefined;
|
||||
}
|
||||
|
||||
function str(v: unknown): string | undefined {
|
||||
return typeof v === 'string' ? v : undefined;
|
||||
}
|
||||
|
||||
/**
|
||||
* Nearest-rank percentile. No interpolation: with small samples an
|
||||
* interpolated p99 invents a value that no request ever experienced.
|
||||
*/
|
||||
export function percentile(sorted: number[], q: number): number | undefined {
|
||||
if (sorted.length === 0) return undefined;
|
||||
const rank = Math.ceil(q * sorted.length);
|
||||
return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
|
||||
}
|
||||
|
||||
/** Fold the seven event types into per-query facts. */
|
||||
function foldQueries(events: Aud0BiEvent[]): Map<string, QueryFact> {
|
||||
const facts = new Map<string, QueryFact>();
|
||||
for (const e of events) {
|
||||
const qid = str(e.payload.query_id);
|
||||
if (!qid) continue;
|
||||
const at = Date.parse(e.occurred_at);
|
||||
const existing = facts.get(qid) ?? { query_id: qid, started_at: Number.isNaN(at) ? 0 : at };
|
||||
if (e.payload.surface !== undefined) existing.surface = str(e.payload.surface);
|
||||
if (e.event_type === 'aud0.analytics_query_started' && !Number.isNaN(at)) existing.started_at = at;
|
||||
if (e.event_type === 'aud0.analytics_query_completed') {
|
||||
existing.outcome = 'completed';
|
||||
existing.duration_ms = num(e.payload.duration_ms);
|
||||
existing.row_count = num(e.payload.row_count);
|
||||
existing.cache_hit = e.payload.cache_hit === true;
|
||||
}
|
||||
if (e.event_type === 'aud0.analytics_query_failed') {
|
||||
existing.outcome = 'failed';
|
||||
existing.duration_ms = num(e.payload.duration_ms);
|
||||
}
|
||||
facts.set(qid, existing);
|
||||
}
|
||||
return facts;
|
||||
}
|
||||
|
||||
function result(
|
||||
o: SloObjective,
|
||||
status: SloStatus,
|
||||
sample_size: number,
|
||||
detail: string,
|
||||
observed?: number,
|
||||
low_confidence?: boolean,
|
||||
): SloResult {
|
||||
const r: SloResult = {
|
||||
objective_id: o.objective_id,
|
||||
workload: o.workload,
|
||||
kind: o.kind,
|
||||
status,
|
||||
target: o.target,
|
||||
unit: o.unit,
|
||||
sample_size,
|
||||
detail,
|
||||
};
|
||||
if (observed !== undefined) r.observed = observed;
|
||||
if (low_confidence) r.low_confidence = true;
|
||||
return r;
|
||||
}
|
||||
|
||||
function compare(o: SloObjective, observed: number): SloStatus {
|
||||
return o.direction === 'at_least'
|
||||
? observed >= o.target
|
||||
? 'met'
|
||||
: 'breached'
|
||||
: observed <= o.target
|
||||
? 'met'
|
||||
: 'breached';
|
||||
}
|
||||
|
||||
/**
|
||||
* Evaluate the catalogue over a window of AUD0 BI usage events.
|
||||
*
|
||||
* Events are DATA: they arrive from aud0's read model, they were written by
|
||||
* other services, and nothing in them is treated as an instruction or as a
|
||||
* schema. Unknown event types and unknown payload keys are ignored, never
|
||||
* spread.
|
||||
*/
|
||||
export function evaluateBiSlos(args: {
|
||||
events: Aud0BiEvent[];
|
||||
from: Date;
|
||||
to: Date;
|
||||
objectives?: readonly SloObjective[];
|
||||
}): SloReport {
|
||||
const objectives = args.objectives ?? BI_SLO_CATALOGUE;
|
||||
const fromMs = args.from.getTime();
|
||||
const toMs = args.to.getTime();
|
||||
const inWindow = args.events.filter((e) => {
|
||||
const t = Date.parse(e.occurred_at);
|
||||
return !Number.isNaN(t) && t >= fromMs && t <= toMs;
|
||||
});
|
||||
|
||||
const queries = [...foldQueries(inWindow).values()];
|
||||
const dashboardQueries = queries.filter((q) => q.surface === 'dashboard');
|
||||
const exploreQueries = queries.filter((q) => q.surface === 'explore');
|
||||
const terminal = (qs: QueryFact[]) => qs.filter((q) => q.outcome !== undefined);
|
||||
const durations = (qs: QueryFact[]) =>
|
||||
qs
|
||||
.map((q) => q.duration_ms)
|
||||
.filter((d): d is number => d !== undefined)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
const reportDurations = inWindow
|
||||
.filter((e) => e.event_type === 'aud0.report_generated')
|
||||
.map((e) => num(e.payload.duration_ms))
|
||||
.filter((d): d is number => d !== undefined)
|
||||
.sort((a, b) => a - b);
|
||||
|
||||
const results = objectives.map((o) => {
|
||||
if (o.measured_from !== 'aud0_bi_usage_events') {
|
||||
return result(
|
||||
o,
|
||||
'not_measurable_here',
|
||||
0,
|
||||
`measured from ${o.measured_from}; BI usage events cannot answer it — ${o.rationale}`,
|
||||
);
|
||||
}
|
||||
|
||||
switch (o.kind) {
|
||||
case 'availability':
|
||||
case 'error_rate': {
|
||||
const t = terminal(dashboardQueries);
|
||||
if (t.length === 0) {
|
||||
return result(o, 'insufficient_data', 0, 'no terminal dashboard query events in the window');
|
||||
}
|
||||
const failed = t.filter((q) => q.outcome === 'failed').length;
|
||||
const observed = o.kind === 'availability' ? (t.length - failed) / t.length : failed / t.length;
|
||||
const status = compare(o, observed);
|
||||
return result(
|
||||
o,
|
||||
t.length < o.min_samples ? 'insufficient_data' : status,
|
||||
t.length,
|
||||
t.length < o.min_samples
|
||||
? `${t.length} terminal queries is below the ${o.min_samples} this objective requires; ` +
|
||||
`the ratio ${observed.toFixed(4)} is reported but not judged`
|
||||
: `${failed} failed of ${t.length} terminal queries`,
|
||||
observed,
|
||||
t.length < o.min_samples,
|
||||
);
|
||||
}
|
||||
case 'latency_p95_first':
|
||||
case 'latency_p95_cached': {
|
||||
const wanted = o.kind === 'latency_p95_cached';
|
||||
const sample = durations(dashboardQueries.filter((q) => (q.cache_hit === true) === wanted));
|
||||
const p95 = percentile(sample, 0.95);
|
||||
if (p95 === undefined) {
|
||||
return result(
|
||||
o,
|
||||
'insufficient_data',
|
||||
0,
|
||||
`no ${wanted ? 'cached' : 'uncached'} dashboard durations`,
|
||||
);
|
||||
}
|
||||
const low = sample.length < o.min_samples;
|
||||
return result(
|
||||
o,
|
||||
low ? 'insufficient_data' : compare(o, p95),
|
||||
sample.length,
|
||||
low
|
||||
? `p95 over ${sample.length} samples (needs ${o.min_samples}) — reported, not judged`
|
||||
: `p95 over ${sample.length} samples`,
|
||||
p95,
|
||||
low,
|
||||
);
|
||||
}
|
||||
case 'latency_p95':
|
||||
case 'latency_p99': {
|
||||
const sample = durations(exploreQueries);
|
||||
const q = o.kind === 'latency_p99' ? 0.99 : 0.95;
|
||||
const value = percentile(sample, q);
|
||||
if (value === undefined) return result(o, 'insufficient_data', 0, 'no exploration durations');
|
||||
const low = sample.length < o.min_samples;
|
||||
return result(
|
||||
o,
|
||||
low ? 'insufficient_data' : compare(o, value),
|
||||
sample.length,
|
||||
low
|
||||
? `p${q * 100} over ${sample.length} samples (needs ${o.min_samples}); with this few samples ` +
|
||||
`it is the ${Math.ceil((1 - q) * sample.length) || 1}-slowest request, not a percentile`
|
||||
: `p${q * 100} over ${sample.length} samples`,
|
||||
value,
|
||||
low,
|
||||
);
|
||||
}
|
||||
case 'max_result_rows': {
|
||||
const rows = exploreQueries
|
||||
.map((q) => q.row_count)
|
||||
.filter((r): r is number => r !== undefined)
|
||||
.sort((a, b) => b - a);
|
||||
if (rows.length === 0) return result(o, 'insufficient_data', 0, 'no exploration row counts');
|
||||
return result(o, compare(o, rows[0]), rows.length, `largest result seen in the window`, rows[0]);
|
||||
}
|
||||
case 'completion_sla': {
|
||||
const p95 = percentile(reportDurations, 0.95);
|
||||
if (p95 === undefined) return result(o, 'insufficient_data', 0, 'no report_generated events');
|
||||
const low = reportDurations.length < o.min_samples;
|
||||
return result(
|
||||
o,
|
||||
low ? 'insufficient_data' : compare(o, p95),
|
||||
reportDurations.length,
|
||||
low ? `p95 over ${reportDurations.length} reports (needs ${o.min_samples})` : 'p95 report duration',
|
||||
p95,
|
||||
low,
|
||||
);
|
||||
}
|
||||
default:
|
||||
return result(o, 'not_measurable_here', 0, `no evaluator for kind "${o.kind}"`);
|
||||
}
|
||||
});
|
||||
|
||||
const unresolved = queries.filter((q) => q.outcome === undefined).length;
|
||||
return {
|
||||
window_from: args.from.toISOString(),
|
||||
window_to: args.to.toISOString(),
|
||||
results,
|
||||
unresolved_queries: unresolved,
|
||||
fully_measured: results.every((r) => r.status === 'met' || r.status === 'breached'),
|
||||
};
|
||||
}
|
||||
|
||||
/**
|
||||
* One line an operator can read: how many objectives are actually being
|
||||
* measured. "9 of 11 met" is meaningless if the other two were never
|
||||
* measured, so the summary counts them separately and never rolls them in.
|
||||
*/
|
||||
export function summariseSloReport(report: SloReport): {
|
||||
met: number;
|
||||
breached: number;
|
||||
insufficient_data: number;
|
||||
not_measurable_here: number;
|
||||
measured_fraction: number;
|
||||
} {
|
||||
const count = (s: SloStatus) => report.results.filter((r) => r.status === s).length;
|
||||
const measured = count('met') + count('breached');
|
||||
return {
|
||||
met: count('met'),
|
||||
breached: count('breached'),
|
||||
insufficient_data: count('insufficient_data'),
|
||||
not_measurable_here: count('not_measurable_here'),
|
||||
measured_fraction: report.results.length === 0 ? 0 : measured / report.results.length,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,209 @@
|
||||
/**
|
||||
* src/analytics/slo/usage-events.ts
|
||||
* ==================================
|
||||
* BI-P103's measurement path: 0pt emits the AUD0 BI usage events (aud0#34,
|
||||
* BI-P076) that the SLO evaluator then reads.
|
||||
*
|
||||
* aud0 says it plainly in docs/BI_ANALYTICS_USAGE.md: "Zero real producers
|
||||
* emit these seven types today." An SLO computed from a second, private
|
||||
* telemetry path would be a number nobody else can audit, so 0pt becomes a
|
||||
* producer instead of inventing one.
|
||||
*
|
||||
* THE PAYLOAD IS AN ALLOWLIST, BUILT FIELD BY NAMED FIELD. AUD0's ingest
|
||||
* refuses result-bearing payloads with a 400 and its schemas are `.strict()`,
|
||||
* so a leak here is a 400 rather than a breach — but a producer that has to be
|
||||
* refused is a producer that is broken. `biUsageEventsForExecution` therefore
|
||||
* never spreads an execution record; it names each field, and
|
||||
* tests/analytics/bi-slo.test.ts feeds it a record carrying SQL text, an error
|
||||
* message, a tenant id and a result sample, then asserts none of it appears in
|
||||
* the output.
|
||||
*
|
||||
* WHAT IS DELIBERATELY NOT SENT: `compiled_query_hash` goes out as
|
||||
* `query_hash` (aud0 requires a 64-hex digest and nothing else), the statement
|
||||
* never leaves 0pt at all, `error_code` travels but `error_message` does not,
|
||||
* and `row_count` travels but no row does.
|
||||
*/
|
||||
|
||||
/** The seven event types, named exactly as aud0 registers them. */
|
||||
export const AUD0_BI_EVENT_TYPES = [
|
||||
'aud0.analytics_query_started',
|
||||
'aud0.analytics_query_completed',
|
||||
'aud0.analytics_query_failed',
|
||||
'aud0.dashboard_viewed',
|
||||
'aud0.report_generated',
|
||||
'aud0.dataset_exported',
|
||||
'aud0.metric_queried',
|
||||
] as const;
|
||||
export type Aud0BiEventType = (typeof AUD0_BI_EVENT_TYPES)[number];
|
||||
|
||||
/** aud0's closed surface list (BI_SURFACES). */
|
||||
export const BI_SURFACES = ['dashboard', 'report', 'explore', 'api', 'scheduled', 'notebook'] as const;
|
||||
export type BiSurface = (typeof BI_SURFACES)[number];
|
||||
|
||||
export interface Aud0BiEvent {
|
||||
event_type: Aud0BiEventType;
|
||||
/** Bounded, allowlisted metadata. Never a result, a statement or a message. */
|
||||
payload: Record<string, string | number | boolean | string[]>;
|
||||
/** Occurrence time, used by the SLO window. */
|
||||
occurred_at: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The 0pt-side view of one execution. Intentionally a SUPERSET of what may be
|
||||
* emitted — it carries the fields 0pt genuinely has, including the ones that
|
||||
* must never leave — so that the mapping below is a real filter rather than a
|
||||
* rename.
|
||||
*/
|
||||
export interface ExecutionTelemetry {
|
||||
query_id: string;
|
||||
/** 64-hex; anything else is dropped rather than reshaped. */
|
||||
compiled_query_hash?: string;
|
||||
/** NEVER emitted. Present here because 0pt has it. */
|
||||
compiled_sql?: string;
|
||||
/** NEVER emitted. */
|
||||
error_message?: string;
|
||||
error_code?: string;
|
||||
/** NEVER emitted — a consumer must not be able to name a tenant, BI-P016. */
|
||||
tenant_id?: string;
|
||||
dataset_id?: string;
|
||||
engine?: string;
|
||||
surface?: BiSurface;
|
||||
state: 'succeeded' | 'failed' | 'cancelled' | 'running';
|
||||
started_at: string;
|
||||
finished_at?: string;
|
||||
duration_ms?: number;
|
||||
row_count?: number;
|
||||
bytes_scanned?: number;
|
||||
cache_hit?: boolean;
|
||||
filter_keys?: string[];
|
||||
/** NEVER emitted: filter VALUES are facts about a subject. */
|
||||
filter_values?: Record<string, string[]>;
|
||||
/** NEVER emitted. */
|
||||
result_sample?: unknown;
|
||||
metric_name?: string;
|
||||
dimension_keys?: string[];
|
||||
granularity?: string;
|
||||
dashboard_id?: string;
|
||||
widget_count?: number;
|
||||
}
|
||||
|
||||
const SHA256 = /^[0-9a-f]{64}$/;
|
||||
const MAX_KEY_NAMES = 50;
|
||||
const MAX_SHORT_TEXT = 120;
|
||||
|
||||
function shortText(v: string | undefined): string | undefined {
|
||||
if (!v) return undefined;
|
||||
return v.slice(0, MAX_SHORT_TEXT);
|
||||
}
|
||||
|
||||
function keyNames(v: string[] | undefined): string[] | undefined {
|
||||
if (!v || v.length === 0) return undefined;
|
||||
return v.slice(0, MAX_KEY_NAMES).map((k) => k.slice(0, MAX_SHORT_TEXT));
|
||||
}
|
||||
|
||||
function put(
|
||||
payload: Record<string, string | number | boolean | string[]>,
|
||||
key: string,
|
||||
value: string | number | boolean | string[] | undefined,
|
||||
): void {
|
||||
if (value === undefined) return;
|
||||
payload[key] = value;
|
||||
}
|
||||
|
||||
/**
|
||||
* Map one execution to its AUD0 usage events.
|
||||
*
|
||||
* A running execution emits only `started`: a query that has not finished is
|
||||
* neither a success nor a failure, and the SLO evaluator counts it as
|
||||
* unresolved rather than quietly as one or the other.
|
||||
*/
|
||||
export function biUsageEventsForExecution(t: ExecutionTelemetry): Aud0BiEvent[] {
|
||||
const base: Record<string, string | number | boolean | string[]> = {};
|
||||
put(base, 'query_id', t.query_id);
|
||||
if (t.compiled_query_hash && SHA256.test(t.compiled_query_hash)) {
|
||||
put(base, 'query_hash', t.compiled_query_hash);
|
||||
}
|
||||
put(base, 'dataset_id', t.dataset_id);
|
||||
put(base, 'engine', shortText(t.engine));
|
||||
put(base, 'surface', t.surface);
|
||||
put(base, 'filter_keys', keyNames(t.filter_keys));
|
||||
|
||||
const events: Aud0BiEvent[] = [
|
||||
{ event_type: 'aud0.analytics_query_started', payload: { ...base }, occurred_at: t.started_at },
|
||||
];
|
||||
|
||||
if (t.state === 'succeeded') {
|
||||
const payload = { ...base };
|
||||
put(payload, 'duration_ms', t.duration_ms);
|
||||
put(payload, 'row_count', t.row_count);
|
||||
put(payload, 'bytes_scanned', t.bytes_scanned);
|
||||
put(payload, 'cache_hit', t.cache_hit);
|
||||
events.push({
|
||||
event_type: 'aud0.analytics_query_completed',
|
||||
payload,
|
||||
occurred_at: t.finished_at ?? t.started_at,
|
||||
});
|
||||
} else if (t.state === 'failed' || t.state === 'cancelled') {
|
||||
const payload = { ...base };
|
||||
put(payload, 'duration_ms', t.duration_ms);
|
||||
// A CODE. `error_message` is never mapped, at any length.
|
||||
put(
|
||||
payload,
|
||||
'error_code',
|
||||
shortText(t.error_code ?? (t.state === 'cancelled' ? 'CANCELLED' : undefined)),
|
||||
);
|
||||
events.push({
|
||||
event_type: 'aud0.analytics_query_failed',
|
||||
payload,
|
||||
occurred_at: t.finished_at ?? t.started_at,
|
||||
});
|
||||
}
|
||||
|
||||
if (t.metric_name) {
|
||||
const payload: Record<string, string | number | boolean | string[]> = {};
|
||||
put(payload, 'metric_name', shortText(t.metric_name));
|
||||
put(payload, 'dataset_id', t.dataset_id);
|
||||
put(payload, 'surface', t.surface);
|
||||
put(payload, 'engine', shortText(t.engine));
|
||||
put(payload, 'dimension_keys', keyNames(t.dimension_keys));
|
||||
put(payload, 'granularity', shortText(t.granularity));
|
||||
put(payload, 'row_count', t.row_count);
|
||||
put(payload, 'duration_ms', t.duration_ms);
|
||||
events.push({
|
||||
event_type: 'aud0.metric_queried',
|
||||
payload,
|
||||
occurred_at: t.finished_at ?? t.started_at,
|
||||
});
|
||||
}
|
||||
|
||||
if (t.dashboard_id) {
|
||||
const payload: Record<string, string | number | boolean | string[]> = {};
|
||||
put(payload, 'dashboard_id', t.dashboard_id);
|
||||
put(payload, 'widget_count', t.widget_count);
|
||||
put(payload, 'duration_ms', t.duration_ms);
|
||||
put(payload, 'surface', t.surface ?? 'dashboard');
|
||||
events.push({ event_type: 'aud0.dashboard_viewed', payload, occurred_at: t.started_at });
|
||||
}
|
||||
|
||||
return events;
|
||||
}
|
||||
|
||||
/** Every payload key this producer can ever emit. Pinned by test. */
|
||||
export const EMITTABLE_PAYLOAD_KEYS = [
|
||||
'query_id',
|
||||
'query_hash',
|
||||
'dataset_id',
|
||||
'engine',
|
||||
'surface',
|
||||
'filter_keys',
|
||||
'duration_ms',
|
||||
'row_count',
|
||||
'bytes_scanned',
|
||||
'cache_hit',
|
||||
'error_code',
|
||||
'metric_name',
|
||||
'dimension_keys',
|
||||
'granularity',
|
||||
'dashboard_id',
|
||||
'widget_count',
|
||||
] as const;
|
||||
@@ -0,0 +1,376 @@
|
||||
/**
|
||||
* src/analytics/sql/analyst-sql-session.ts
|
||||
* =========================================
|
||||
* BI-P017 — ANALYST SQL CONTRACT.
|
||||
*
|
||||
* "For privileged analyst role only. Implement separate AnalyticsSQLSession
|
||||
* using Trino/read-only endpoints. Security: role-gated; query timeout;
|
||||
* row/byte limits; read only; tenant/project catalogs; audit; statement
|
||||
* history."
|
||||
*
|
||||
* SEPARATE is the operative word, and it is why this is its own class rather
|
||||
* than a method on `SiaxAnalyticsEngine`. `executeMetric` has no raw-SQL
|
||||
* surface at all; if analyst SQL lived on the same object, the closed DSL
|
||||
* would be one refactor away from growing one. `ClickHouseAnalyticsEngine
|
||||
* .executeQuery` stays a documented refusal.
|
||||
*
|
||||
* The four layers are described in statement-guard.ts. This class owns layers
|
||||
* 1 (role) and the enforcement of every named limit; the guard is layer 2;
|
||||
* tenant-catalog.ts renders layers 3 and 4.
|
||||
*
|
||||
* AUDIT AND STATEMENT HISTORY — the deliberate split, because BI-P017 requires
|
||||
* statement history and BI-P018 requires NOT storing sensitive raw SQL:
|
||||
*
|
||||
* * the BI-P018 execution record stores a HASH of the compiled statement
|
||||
* and nothing else;
|
||||
* * the analyst statement history stores the NORMALIZED statement (every
|
||||
* literal replaced by `?`) always — that is the searchable, non-PII form;
|
||||
* * the RAW statement is retained only when `retainRawStatements` is on, and
|
||||
* the entry is classified so a retention job can find it. It is on by
|
||||
* default for analyst sessions, because a privileged human executing
|
||||
* unreviewed SQL against tenant data is exactly the case where "who ran
|
||||
* what" must survive, and a masked statement cannot answer it.
|
||||
*
|
||||
* A caller who needs the non-PII form gets `normalized`; nothing forces raw
|
||||
* SQL into a downstream sink.
|
||||
*/
|
||||
import crypto from 'node:crypto';
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { AnalyticsIdentity } from '../engine/siax-analytics-engine';
|
||||
import type { ResourceBudget } from '../engine/resource-governance';
|
||||
import { applyQuota, BUDGET_TIERS, DEFAULT_BUDGETS } from '../engine/resource-governance';
|
||||
import { guardAnalystStatement, type GuardedStatement } from './statement-guard';
|
||||
|
||||
/** The CL0UD role BI-P017 gates on. Verified upstream; never self-asserted. */
|
||||
export const ANALYST_SQL_ROLE = 'analytics.sql.analyst';
|
||||
|
||||
/**
|
||||
* An identity carrying CL0UD-verified roles. Roles are absent by default and
|
||||
* absence is a denial — a session must never open because a field was missing.
|
||||
*/
|
||||
export interface AnalystIdentity extends AnalyticsIdentity {
|
||||
roles?: readonly string[];
|
||||
}
|
||||
|
||||
export interface AnalystStatementRecord {
|
||||
statement_id: string;
|
||||
tenant_id: string;
|
||||
project_id?: string;
|
||||
principal_id: string;
|
||||
/** Literals masked. Always present. */
|
||||
normalized: string;
|
||||
/** Present only when the session retains raw statements. */
|
||||
raw?: string;
|
||||
statement_hash: string;
|
||||
referenced_objects: string[];
|
||||
submitted_at: string;
|
||||
finished_at?: string;
|
||||
duration_ms?: number;
|
||||
rows_returned?: number;
|
||||
bytes_read?: number;
|
||||
rows_read?: number;
|
||||
outcome: 'admitted' | 'refused' | 'succeeded' | 'failed' | 'cancelled';
|
||||
/** SiaxErrorCode on refusal/failure. Never an engine body. */
|
||||
error_code?: string;
|
||||
/** Short operator-facing reason. Never echoes the statement. */
|
||||
refusal_reason?: string;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
/** Where the history goes. AUD0 is the system of record; this is the port. */
|
||||
export interface AnalystAuditSink {
|
||||
record(entry: AnalystStatementRecord): Promise<void> | void;
|
||||
}
|
||||
|
||||
/** The read-only endpoint the session executes against. */
|
||||
export interface ReadOnlySqlEndpoint {
|
||||
/** "clickhouse" | "trino" — recorded on every execution. */
|
||||
readonly engine: string;
|
||||
/**
|
||||
* Execute an ALREADY-GUARDED statement as the tenant's restricted user.
|
||||
* The implementation must not add privileges, and must apply `settings`.
|
||||
*/
|
||||
execute(args: {
|
||||
sql: string;
|
||||
queryId: string;
|
||||
settings: Record<string, string>;
|
||||
signal: AbortSignal;
|
||||
}): Promise<SqlEndpointResult>;
|
||||
/** Stop a running statement server-side. Returns true when it is gone. */
|
||||
cancel(queryId: string): Promise<boolean>;
|
||||
}
|
||||
|
||||
export interface SqlEndpointResult {
|
||||
columns: string[];
|
||||
rows: unknown[][];
|
||||
rows_read?: number;
|
||||
bytes_read?: number;
|
||||
}
|
||||
|
||||
export interface AnalystSqlSessionOptions {
|
||||
identity: AnalystIdentity;
|
||||
endpoint: ReadOnlySqlEndpoint;
|
||||
/** The tenant's own catalog of pre-filtered views (BI-P017 layer 4). */
|
||||
catalog: string;
|
||||
/** Objects inside it the analyst may read. */
|
||||
objects: readonly string[];
|
||||
audit: AnalystAuditSink;
|
||||
/**
|
||||
* Budget tier. ANALYTICAL is the default for a human analyst; the tier is
|
||||
* still only ever TIGHTENED by a CL0UD quota, never loosened (BI-P012).
|
||||
*/
|
||||
tier?: (typeof BUDGET_TIERS)[number];
|
||||
retainRawStatements?: boolean;
|
||||
/** Injected for deterministic tests. */
|
||||
now?: () => number;
|
||||
}
|
||||
|
||||
export interface AnalystQueryOutcome {
|
||||
statement_id: string;
|
||||
query_id: string;
|
||||
columns: string[];
|
||||
rows: unknown[][];
|
||||
duration_ms: number;
|
||||
rows_read?: number;
|
||||
bytes_read?: number;
|
||||
/** Objects the statement was permitted to read, as resolved by the guard. */
|
||||
referenced_objects: string[];
|
||||
truncated: boolean;
|
||||
}
|
||||
|
||||
export class AnalyticsSQLSession {
|
||||
private readonly identity: AnalystIdentity;
|
||||
private readonly endpoint: ReadOnlySqlEndpoint;
|
||||
private readonly catalog: string;
|
||||
private readonly objects: readonly string[];
|
||||
private readonly audit: AnalystAuditSink;
|
||||
private readonly budget: ResourceBudget;
|
||||
private readonly retainRaw: boolean;
|
||||
private readonly now: () => number;
|
||||
private readonly running = new Map<string, AbortController>();
|
||||
private closed = false;
|
||||
|
||||
/**
|
||||
* Opening a session is itself the role gate. A caller without the analyst
|
||||
* role never obtains an object that can run SQL — as opposed to obtaining
|
||||
* one whose methods happen to check a flag.
|
||||
*/
|
||||
constructor(opts: AnalystSqlSessionOptions) {
|
||||
const id = opts.identity;
|
||||
if (!id?.tenant_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.sql',
|
||||
'analyst SQL requires a verified tenant identity',
|
||||
);
|
||||
}
|
||||
if (!id.principal_id?.trim()) {
|
||||
throw new SiaxError(
|
||||
'UNAUTHENTICATED',
|
||||
'0pt.analytics.sql',
|
||||
'analyst SQL requires a verified principal',
|
||||
);
|
||||
}
|
||||
const roles = id.roles ?? [];
|
||||
if (!roles.includes(ANALYST_SQL_ROLE)) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.sql',
|
||||
`analyst SQL requires the ${ANALYST_SQL_ROLE} role`,
|
||||
);
|
||||
}
|
||||
if (opts.objects.length === 0) {
|
||||
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', 'session exposes no readable objects');
|
||||
}
|
||||
this.identity = id;
|
||||
this.endpoint = opts.endpoint;
|
||||
this.catalog = opts.catalog;
|
||||
this.objects = opts.objects;
|
||||
this.audit = opts.audit;
|
||||
this.budget = applyQuota(DEFAULT_BUDGETS[opts.tier ?? 'ANALYTICAL'], id.quota);
|
||||
this.retainRaw = opts.retainRawStatements ?? true;
|
||||
this.now = opts.now ?? Date.now;
|
||||
}
|
||||
|
||||
get resourceBudget(): ResourceBudget {
|
||||
return { ...this.budget };
|
||||
}
|
||||
|
||||
/**
|
||||
* Guard a statement WITHOUT running it. Exposed so a UI can validate as the
|
||||
* analyst types, and so the adversarial test corpus has a pure entry point.
|
||||
*/
|
||||
guard(sql: string): GuardedStatement {
|
||||
return guardAnalystStatement(sql, { allowedCatalog: this.catalog, allowedObjects: this.objects });
|
||||
}
|
||||
|
||||
async query(sql: string): Promise<AnalystQueryOutcome> {
|
||||
if (this.closed) {
|
||||
throw new SiaxError('CONFLICT', '0pt.analytics.sql', 'session is closed');
|
||||
}
|
||||
const statementId = crypto.randomUUID();
|
||||
const submittedAt = new Date(this.now()).toISOString();
|
||||
const base = {
|
||||
statement_id: statementId,
|
||||
tenant_id: this.identity.tenant_id,
|
||||
project_id: this.identity.project_id,
|
||||
principal_id: this.identity.principal_id,
|
||||
correlation_id: this.identity.correlation_id,
|
||||
submitted_at: submittedAt,
|
||||
};
|
||||
|
||||
let guarded: GuardedStatement;
|
||||
try {
|
||||
guarded = this.guard(sql);
|
||||
} catch (e) {
|
||||
const err = e as SiaxError;
|
||||
// A REFUSAL is audited as loudly as an execution. A guard whose
|
||||
// rejections are invisible cannot be reviewed, and an analyst probing
|
||||
// the boundary is exactly what an operator needs to see.
|
||||
await this.audit.record({
|
||||
...base,
|
||||
normalized: safeNormalize(sql),
|
||||
statement_hash: hashStatement(sql),
|
||||
referenced_objects: [],
|
||||
outcome: 'refused',
|
||||
error_code: err.code ?? 'PERMANENT_FAILURE',
|
||||
// `message` is already the sanitised "[CODE] source: detail" form; it
|
||||
// never contains the caller's statement, only the guard's reason.
|
||||
refusal_reason: err.message ?? 'refused',
|
||||
});
|
||||
throw e;
|
||||
}
|
||||
|
||||
const queryId = this.newQueryId();
|
||||
const controller = new AbortController();
|
||||
this.running.set(queryId, controller);
|
||||
const startedMs = this.now();
|
||||
|
||||
const record: AnalystStatementRecord = {
|
||||
...base,
|
||||
normalized: guarded.normalized,
|
||||
...(this.retainRaw ? { raw: guarded.raw } : {}),
|
||||
statement_hash: hashStatement(guarded.raw),
|
||||
referenced_objects: guarded.referencedObjects,
|
||||
outcome: 'admitted',
|
||||
};
|
||||
|
||||
try {
|
||||
const result = await this.endpoint.execute({
|
||||
// The statement is passed THROUGH unchanged. This class does not
|
||||
// rewrite SQL: a rewriter must be a correct parser of the dialect to
|
||||
// be safe, and layers 3+4 do not need it to be one.
|
||||
sql: guarded.raw,
|
||||
queryId,
|
||||
settings: this.executionSettings(),
|
||||
signal: controller.signal,
|
||||
});
|
||||
const duration = this.now() - startedMs;
|
||||
const truncated = result.rows.length >= this.budget.maxResultRows;
|
||||
await this.audit.record({
|
||||
...record,
|
||||
outcome: 'succeeded',
|
||||
finished_at: new Date(this.now()).toISOString(),
|
||||
duration_ms: duration,
|
||||
rows_returned: result.rows.length,
|
||||
rows_read: result.rows_read,
|
||||
bytes_read: result.bytes_read,
|
||||
});
|
||||
return {
|
||||
statement_id: statementId,
|
||||
query_id: queryId,
|
||||
columns: result.columns,
|
||||
rows: result.rows,
|
||||
duration_ms: duration,
|
||||
rows_read: result.rows_read,
|
||||
bytes_read: result.bytes_read,
|
||||
referenced_objects: guarded.referencedObjects,
|
||||
truncated,
|
||||
};
|
||||
} catch (e) {
|
||||
const err = e as SiaxError;
|
||||
await this.audit.record({
|
||||
...record,
|
||||
outcome: controller.signal.aborted ? 'cancelled' : 'failed',
|
||||
finished_at: new Date(this.now()).toISOString(),
|
||||
duration_ms: this.now() - startedMs,
|
||||
error_code: err.code ?? 'PERMANENT_FAILURE',
|
||||
});
|
||||
throw e;
|
||||
} finally {
|
||||
this.running.delete(queryId);
|
||||
}
|
||||
}
|
||||
|
||||
/** Cancel a statement this session started. Never another session's. */
|
||||
async cancel(queryId: string): Promise<boolean> {
|
||||
if (!queryId.startsWith(this.queryIdPrefix())) {
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.sql',
|
||||
'statement does not belong to this session',
|
||||
);
|
||||
}
|
||||
this.running.get(queryId)?.abort();
|
||||
return this.endpoint.cancel(queryId);
|
||||
}
|
||||
|
||||
/** Abort everything in flight. Idempotent. */
|
||||
async close(): Promise<void> {
|
||||
this.closed = true;
|
||||
const ids = [...this.running.keys()];
|
||||
for (const id of ids) {
|
||||
this.running.get(id)?.abort();
|
||||
try {
|
||||
await this.endpoint.cancel(id);
|
||||
} catch {
|
||||
/* best effort; the abort already stopped us waiting */
|
||||
}
|
||||
this.running.delete(id);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* Every limit BI-P017 names, as engine settings. `readonly` is `1`, not `2`:
|
||||
* `readonly = 2` permits SETTINGS changes, which would let a statement raise
|
||||
* its own ceilings.
|
||||
*/
|
||||
private executionSettings(): Record<string, string> {
|
||||
return {
|
||||
readonly: '1',
|
||||
max_execution_time: String(this.budget.maxExecutionTimeSec),
|
||||
timeout_overflow_mode: 'throw',
|
||||
max_result_rows: String(this.budget.maxResultRows),
|
||||
result_overflow_mode: 'throw',
|
||||
max_rows_to_read: String(this.budget.maxRowsToRead),
|
||||
max_bytes_to_read: String(this.budget.maxBytesToRead),
|
||||
read_overflow_mode: 'throw',
|
||||
max_memory_usage: String(this.budget.maxMemoryUsageBytes),
|
||||
};
|
||||
}
|
||||
|
||||
private queryIdPrefix(): string {
|
||||
return `sql_${crypto.createHash('sha256').update(this.identity.tenant_id).digest('hex').slice(0, 12)}_`;
|
||||
}
|
||||
|
||||
private newQueryId(): string {
|
||||
return `${this.queryIdPrefix()}${crypto.randomUUID().replace(/-/g, '')}`;
|
||||
}
|
||||
}
|
||||
|
||||
export function hashStatement(sql: string): string {
|
||||
return crypto.createHash('sha256').update(sql.trim()).digest('hex');
|
||||
}
|
||||
|
||||
/**
|
||||
* Mask literals in a statement that FAILED the guard, so a refusal can be
|
||||
* audited without copying whatever the caller pasted. Best-effort by design:
|
||||
* the statement did not parse cleanly, so neither will this.
|
||||
*/
|
||||
function safeNormalize(sql: string): string {
|
||||
return sql
|
||||
.replace(/'(?:[^'\\]|\\.|'')*'/g, '?')
|
||||
.replace(/\s+/g, ' ')
|
||||
.trim()
|
||||
.slice(0, 2000);
|
||||
}
|
||||
@@ -0,0 +1,516 @@
|
||||
/**
|
||||
* src/analytics/sql/statement-guard.ts
|
||||
* =====================================
|
||||
* BI-P017 — ANALYST SQL CONTRACT, layer 2 (statement admission).
|
||||
*
|
||||
* READ THIS BEFORE TRUSTING IT.
|
||||
*
|
||||
* This module is a GUARD, not a proof. It is the second of four layers, and
|
||||
* it is the WEAKEST of the four, because it reasons about SQL text without
|
||||
* being ClickHouse's parser. A guard that reads a dialect it does not
|
||||
* implement can always be wrong about some string. The layers below it are
|
||||
* the ones that hold when it is:
|
||||
*
|
||||
* 1. ROLE — only a CL0UD-verified analyst role reaches a session
|
||||
* at all (analyst-sql-session.ts).
|
||||
* 2. THIS GUARD — statement admission: one read-only statement, no
|
||||
* banned construct, every table reference resolved
|
||||
* into the tenant's own catalog.
|
||||
* 3. GRANTS — the session authenticates as a per-tenant ClickHouse
|
||||
* user with `readonly = 1` and SELECT granted ONLY on
|
||||
* that tenant's catalog. A statement this guard wrongly
|
||||
* admits still cannot read a table the user cannot see.
|
||||
* 4. VIEW PREDICATE — the tenant catalog contains VIEWS, never base tables,
|
||||
* and each view bakes `WHERE tenant_id = '<tenant>'`
|
||||
* into its own definition. A row policy on the base
|
||||
* tables repeats it server-side.
|
||||
*
|
||||
* tests/integration/analyst-sql-clickhouse.integration.test.ts runs the
|
||||
* adversarial corpus against a live ClickHouse with layer 2 REMOVED, and
|
||||
* requires layers 3 and 4 to refuse it anyway. A defence-in-depth claim that
|
||||
* has never been tested with a layer removed is not evidence — this estate
|
||||
* has shipped several.
|
||||
*
|
||||
* DESIGN NOTES that are load-bearing:
|
||||
*
|
||||
* * Identifiers must resolve to an ALLOW-LIST. The deny-lists below are a
|
||||
* second opinion and a better error message, never the mechanism. Every
|
||||
* table reference is checked positively against the tenant's catalog.
|
||||
* * Outside string literals the statement must be ASCII. Homoglyph and
|
||||
* zero-width tricks against keyword matching stop being expressible.
|
||||
* * The FROM/JOIN scan is CONSERVATIVE where it is unsure, and that costs
|
||||
* something worth naming: ClickHouse's `ARRAY JOIN arr` is a legitimate
|
||||
* construct whose target is a COLUMN, not a table, and this guard refuses
|
||||
* it because the token after JOIN is neither a CTE nor a qualified object.
|
||||
* A false refusal an analyst can see and report is the right side of that
|
||||
* trade; a false ADMISSION is not. Supporting it means teaching the scan
|
||||
* which JOIN flavours take a non-table target, which is the direction that
|
||||
* grows holes, so it stays refused until someone needs it and can add it
|
||||
* WITH a corpus entry proving the narrowing did not open anything else.
|
||||
* * Comments and string literals are removed BEFORE any keyword scan, using
|
||||
* ClickHouse's own lexical rules (`--` and `#` to end of line, `/* *\/`
|
||||
* non-nesting, `'` with backslash and doubled-quote escapes, `"` and
|
||||
* backtick as identifier quotes).
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
|
||||
export interface StatementGuardOptions {
|
||||
/**
|
||||
* The ONLY database name a table reference may carry. This is the tenant's
|
||||
* own catalog of pre-filtered views, not the shared analytics database.
|
||||
*/
|
||||
allowedCatalog: string;
|
||||
/** Object names inside that catalog the analyst may read. */
|
||||
allowedObjects: readonly string[];
|
||||
/** Hard ceiling on statement length; a megabyte of SQL is an attack, not a query. */
|
||||
maxLength?: number;
|
||||
}
|
||||
|
||||
export interface GuardedStatement {
|
||||
/** The statement as submitted, unchanged. */
|
||||
raw: string;
|
||||
/** Comments and literals removed — what the keyword scan actually saw. */
|
||||
stripped: string;
|
||||
/** Literals replaced by `?`, for statement history that is not a PII dump. */
|
||||
normalized: string;
|
||||
/** Fully-qualified objects the statement reads, resolved against the catalog. */
|
||||
referencedObjects: string[];
|
||||
/** Common-table-expression names defined by the statement itself. */
|
||||
cteNames: string[];
|
||||
}
|
||||
|
||||
const DEFAULT_MAX_LENGTH = 20_000;
|
||||
|
||||
/**
|
||||
* Statement-level constructs that make a statement not-a-read. Matched as
|
||||
* whole words against the stripped text.
|
||||
*
|
||||
* `SETTINGS` is here because a trailing `SETTINGS max_rows_to_read = …` is the
|
||||
* obvious way to try to lift the governance ceiling from inside the query.
|
||||
* `FORMAT` is here because the session appends its own; a caller-supplied one
|
||||
* is either a duplicate or an attempt at `FORMAT ...` side effects.
|
||||
*/
|
||||
const BANNED_KEYWORDS = [
|
||||
'insert',
|
||||
'update',
|
||||
'delete',
|
||||
'drop',
|
||||
'alter',
|
||||
'create',
|
||||
'truncate',
|
||||
'rename',
|
||||
'attach',
|
||||
'detach',
|
||||
'replace',
|
||||
'optimize',
|
||||
'grant',
|
||||
'revoke',
|
||||
'kill',
|
||||
'system',
|
||||
'use',
|
||||
'set',
|
||||
'settings',
|
||||
'exchange',
|
||||
'move',
|
||||
'undrop',
|
||||
'backup',
|
||||
'restore',
|
||||
'outfile',
|
||||
'infile',
|
||||
'format',
|
||||
'watch',
|
||||
'check',
|
||||
'explain',
|
||||
'show',
|
||||
'exists',
|
||||
'call',
|
||||
'execute',
|
||||
'declare',
|
||||
'begin',
|
||||
'commit',
|
||||
'rollback',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Functions that read something other than a table the grants control:
|
||||
* the filesystem, the network, another server, a dictionary, an executable.
|
||||
* ClickHouse table functions are the whole reason an analyst SQL surface is
|
||||
* dangerous — `SELECT * FROM url('http://…')` needs no table privilege at all.
|
||||
*/
|
||||
const BANNED_FUNCTIONS = [
|
||||
'file',
|
||||
'url',
|
||||
'urlcluster',
|
||||
's3',
|
||||
's3cluster',
|
||||
'gcs',
|
||||
'azureblobstorage',
|
||||
'remote',
|
||||
'remotesecure',
|
||||
'cluster',
|
||||
'clusterallreplicas',
|
||||
'hdfs',
|
||||
'hdfscluster',
|
||||
'mysql',
|
||||
'postgresql',
|
||||
'mongodb',
|
||||
'jdbc',
|
||||
'odbc',
|
||||
'sqlite',
|
||||
'redis',
|
||||
'deltalake',
|
||||
'hudi',
|
||||
'iceberg',
|
||||
'executable',
|
||||
'input',
|
||||
'generaterandom',
|
||||
'numbers_mt',
|
||||
'merge',
|
||||
'view',
|
||||
'viewifpermitted',
|
||||
'loop',
|
||||
'fuzzjson',
|
||||
'fuzzquery',
|
||||
'dictget',
|
||||
'dictgetordefault',
|
||||
'dictgetornull',
|
||||
'dictgetstring',
|
||||
'dictgetuint64',
|
||||
'dictgethierarchy',
|
||||
'dictisin',
|
||||
'dicthas',
|
||||
'dictgetchildren',
|
||||
'dictgetdescendants',
|
||||
'filesystemavailable',
|
||||
'filesystemfree',
|
||||
'filesystemcapacity',
|
||||
'currentdatabase',
|
||||
'currentuser',
|
||||
'hostname',
|
||||
'fqdn',
|
||||
'uptime',
|
||||
'version',
|
||||
'getsetting',
|
||||
'getmacro',
|
||||
'shardnum',
|
||||
'shardcount',
|
||||
'buildid',
|
||||
'servertimezone',
|
||||
'readwkt',
|
||||
'structurefromjson',
|
||||
's3queue',
|
||||
'timeseries',
|
||||
'mergetreeindex',
|
||||
] as const;
|
||||
|
||||
/**
|
||||
* Constructs that a whole-word keyword ban gets WRONG, so they get a pattern.
|
||||
*
|
||||
* `DESC` is the case that proves the point: banning the word breaks every
|
||||
* `ORDER BY x DESC` an analyst will ever write, and the test corpus caught it
|
||||
* immediately. What is actually dangerous is `DESCRIBE`/`DESC` used as a
|
||||
* STATEMENT — including inside a subquery, where ClickHouse accepts
|
||||
* `SELECT * FROM (DESCRIBE TABLE x)` and the FROM-scan sees only a `(`.
|
||||
*/
|
||||
const BANNED_PATTERNS: Array<{ re: RegExp; reason: string }> = [
|
||||
{ re: /\bdesc(ribe)?\s+(table|\()/i, reason: 'DESCRIBE is not permitted in a read-only analyst session' },
|
||||
{ re: /\bcurrent_user\b/i, reason: 'CURRENT_USER is not permitted in an analyst session' },
|
||||
{ re: /\binto\s+outfile\b/i, reason: 'INTO OUTFILE is not permitted in an analyst session' },
|
||||
];
|
||||
|
||||
/** Names that are catalogs an analyst may never reach, whatever the grants say. */
|
||||
const BANNED_CATALOGS = [
|
||||
'system',
|
||||
'information_schema',
|
||||
'INFORMATION_SCHEMA',
|
||||
'default',
|
||||
'opt_analytics',
|
||||
'opt_scraped',
|
||||
];
|
||||
|
||||
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
|
||||
const QUALIFIED_RE = new RegExp(`^${IDENT}\\.${IDENT}$`);
|
||||
|
||||
function deny(detail: string): never {
|
||||
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', detail);
|
||||
}
|
||||
|
||||
function invalid(detail: string): never {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', detail);
|
||||
}
|
||||
|
||||
interface StripResult {
|
||||
stripped: string;
|
||||
normalized: string;
|
||||
/** Identifier tokens that were quoted, kept so they can still be resolved. */
|
||||
hadQuotedIdentifier: boolean;
|
||||
}
|
||||
|
||||
/**
|
||||
* Remove comments and string literals using ClickHouse's lexical rules.
|
||||
*
|
||||
* Two outputs, deliberately: `stripped` keeps the literal DELIMITERS so the
|
||||
* token stream still parses (`'x'` becomes `''`), and `normalized` replaces
|
||||
* each literal with `?` so a statement history is searchable without being a
|
||||
* copy of the tenant's data.
|
||||
*
|
||||
* An unterminated literal or comment is a REJECTION. Silently treating the
|
||||
* rest of the statement as a string is exactly how a guard and an engine come
|
||||
* to disagree about where the query ends.
|
||||
*/
|
||||
export function stripSqlText(sql: string): StripResult {
|
||||
let stripped = '';
|
||||
let normalized = '';
|
||||
let hadQuotedIdentifier = false;
|
||||
let i = 0;
|
||||
const n = sql.length;
|
||||
|
||||
while (i < n) {
|
||||
const c = sql[i];
|
||||
const next = sql[i + 1];
|
||||
|
||||
// -- line comment
|
||||
if (c === '-' && next === '-') {
|
||||
while (i < n && sql[i] !== '\n') i += 1;
|
||||
stripped += ' ';
|
||||
normalized += ' ';
|
||||
continue;
|
||||
}
|
||||
// # line comment (ClickHouse accepts `#` and `#!`)
|
||||
if (c === '#') {
|
||||
while (i < n && sql[i] !== '\n') i += 1;
|
||||
stripped += ' ';
|
||||
normalized += ' ';
|
||||
continue;
|
||||
}
|
||||
// /* block comment */ — ClickHouse does NOT nest these
|
||||
if (c === '/' && next === '*') {
|
||||
const end = sql.indexOf('*/', i + 2);
|
||||
if (end === -1) invalid('unterminated block comment');
|
||||
i = end + 2;
|
||||
stripped += ' ';
|
||||
normalized += ' ';
|
||||
continue;
|
||||
}
|
||||
// 'string literal'
|
||||
if (c === "'") {
|
||||
i += 1;
|
||||
while (i < n) {
|
||||
if (sql[i] === '\\') {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (sql[i] === "'") {
|
||||
if (sql[i + 1] === "'") {
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
i += 1;
|
||||
}
|
||||
if (i >= n) invalid('unterminated string literal');
|
||||
i += 1; // closing quote
|
||||
stripped += "''";
|
||||
normalized += '?';
|
||||
continue;
|
||||
}
|
||||
// "quoted identifier" / `quoted identifier`
|
||||
if (c === '"' || c === '`') {
|
||||
const quote = c;
|
||||
let ident = '';
|
||||
i += 1;
|
||||
while (i < n) {
|
||||
if (sql[i] === '\\') {
|
||||
ident += sql[i + 1] ?? '';
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
if (sql[i] === quote) {
|
||||
if (sql[i + 1] === quote) {
|
||||
ident += quote;
|
||||
i += 2;
|
||||
continue;
|
||||
}
|
||||
break;
|
||||
}
|
||||
ident += sql[i];
|
||||
i += 1;
|
||||
}
|
||||
if (i >= n) invalid('unterminated quoted identifier');
|
||||
i += 1;
|
||||
hadQuotedIdentifier = true;
|
||||
// The identifier is put back UNQUOTED so the table-reference scan sees
|
||||
// it exactly as it would see a bare one. A quoted identifier that does
|
||||
// not survive that scan is refused like any other.
|
||||
if (!new RegExp(`^${IDENT}$`).test(ident)) {
|
||||
deny(`quoted identifier "${ident.slice(0, 32)}" is not a plain name`);
|
||||
}
|
||||
stripped += ident;
|
||||
normalized += ident;
|
||||
continue;
|
||||
}
|
||||
stripped += c;
|
||||
normalized += c;
|
||||
i += 1;
|
||||
}
|
||||
return { stripped, normalized, hadQuotedIdentifier };
|
||||
}
|
||||
|
||||
/**
|
||||
* Admit a statement, or refuse it.
|
||||
*
|
||||
* Returns the parsed view of what was admitted; the caller (the session) is
|
||||
* responsible for actually running it under the restricted user.
|
||||
*/
|
||||
export function guardAnalystStatement(sql: string, opts: StatementGuardOptions): GuardedStatement {
|
||||
const maxLength = opts.maxLength ?? DEFAULT_MAX_LENGTH;
|
||||
if (typeof sql !== 'string') invalid('statement must be a string');
|
||||
const raw = sql.trim();
|
||||
if (!raw) invalid('statement is empty');
|
||||
if (raw.length > maxLength) invalid(`statement exceeds ${maxLength} characters`);
|
||||
if (!opts.allowedCatalog || !new RegExp(`^${IDENT}$`).test(opts.allowedCatalog)) {
|
||||
// A misconfigured session must not degrade to "any catalog".
|
||||
deny('session has no valid tenant catalog');
|
||||
}
|
||||
if (BANNED_CATALOGS.includes(opts.allowedCatalog)) {
|
||||
deny(`session catalog "${opts.allowedCatalog}" is a reserved catalog`);
|
||||
}
|
||||
if (opts.allowedObjects.length === 0) {
|
||||
deny('session exposes no readable objects');
|
||||
}
|
||||
|
||||
const { stripped, normalized } = stripSqlText(raw);
|
||||
|
||||
// ── ASCII-only outside literals ────────────────────────────────────────
|
||||
// Homoglyphs, zero-width joiners and RTL overrides stop being expressible
|
||||
// against the keyword scan. Values are unaffected: they live in literals,
|
||||
// which have already been removed at this point.
|
||||
// Written as a code-point scan rather than a character class: a regex of
|
||||
// control characters is unreadable, trips `no-control-regex`, and is the
|
||||
// kind of expression a reviewer nods at without checking. Permitted: tab,
|
||||
// LF, CR and printable ASCII. Everything else, NUL included, is refused.
|
||||
for (let i = 0; i < stripped.length; i += 1) {
|
||||
const code = stripped.charCodeAt(i);
|
||||
const printable = code >= 0x20 && code <= 0x7e;
|
||||
const allowedWhitespace = code === 0x09 || code === 0x0a || code === 0x0d;
|
||||
if (!printable && !allowedWhitespace) {
|
||||
deny(
|
||||
`statement contains a disallowed character (U+${code.toString(16).toUpperCase().padStart(4, '0')}) ` +
|
||||
`outside a string literal`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── exactly one statement ──────────────────────────────────────────────
|
||||
const withoutTrailingSemi = stripped.replace(/;\s*$/, '');
|
||||
if (withoutTrailingSemi.includes(';')) {
|
||||
deny('multiple statements are not allowed in an analyst SQL session');
|
||||
}
|
||||
|
||||
const lower = ` ${withoutTrailingSemi.toLowerCase().replace(/\s+/g, ' ').trim()} `;
|
||||
|
||||
// ── must be a read ─────────────────────────────────────────────────────
|
||||
if (!/^\s*(select|with)\b/i.test(withoutTrailingSemi.trim())) {
|
||||
deny('analyst SQL sessions accept SELECT and WITH statements only');
|
||||
}
|
||||
|
||||
for (const kw of BANNED_KEYWORDS) {
|
||||
if (new RegExp(`(^|[^A-Za-z0-9_])${kw}([^A-Za-z0-9_]|$)`).test(lower)) {
|
||||
deny(`"${kw.toUpperCase()}" is not permitted in a read-only analyst session`);
|
||||
}
|
||||
}
|
||||
|
||||
for (const { re, reason } of BANNED_PATTERNS) {
|
||||
if (re.test(lower)) deny(reason);
|
||||
}
|
||||
|
||||
// ── banned functions, matched as `name(` ───────────────────────────────
|
||||
const calls = [...lower.matchAll(new RegExp(`([a-z_][a-z0-9_]*)\\s*\\(`, 'g'))].map((m) => m[1]);
|
||||
for (const call of calls) {
|
||||
if ((BANNED_FUNCTIONS as readonly string[]).includes(call)) {
|
||||
deny(`function "${call}" is not permitted in an analyst session`);
|
||||
}
|
||||
}
|
||||
|
||||
// ── table references: positively resolved against the catalog ──────────
|
||||
const cteNames = collectCteNames(withoutTrailingSemi);
|
||||
const referencedObjects = collectTableReferences(withoutTrailingSemi, cteNames, opts);
|
||||
|
||||
if (referencedObjects.length === 0) {
|
||||
// A SELECT with no FROM (`SELECT 1`) is harmless, but so is refusing it,
|
||||
// and "no table reference" is also what a successful parser evasion looks
|
||||
// like. Refuse: an analyst session exists to read tables.
|
||||
deny('statement references no readable object in the tenant catalog');
|
||||
}
|
||||
|
||||
return { raw, stripped, normalized, referencedObjects, cteNames };
|
||||
}
|
||||
|
||||
/** `WITH x AS (…), y AS (…)` — names the statement defines for itself. */
|
||||
function collectCteNames(stripped: string): string[] {
|
||||
const names: string[] = [];
|
||||
const re = new RegExp(`(?:^|[,\\s(])(${IDENT})\\s+as\\s*\\(`, 'gi');
|
||||
for (const m of stripped.matchAll(re)) names.push(m[1].toLowerCase());
|
||||
return names;
|
||||
}
|
||||
|
||||
/**
|
||||
* Every `FROM` / `JOIN` target, resolved.
|
||||
*
|
||||
* Rules, all fail-closed:
|
||||
* * `(` after FROM/JOIN is a subquery — fine, its own FROM is scanned too.
|
||||
* * `name(` after FROM/JOIN is a TABLE FUNCTION — refused outright, even if
|
||||
* the name is not on the deny-list, because an unknown table function is
|
||||
* precisely the case the deny-list cannot cover.
|
||||
* * a bare `name` must be a CTE this statement defined.
|
||||
* * a `db.name` must have `db === allowedCatalog` and `name` in the
|
||||
* allow-list. Three-part `a.b.c` names are refused.
|
||||
*/
|
||||
function collectTableReferences(stripped: string, cteNames: string[], opts: StatementGuardOptions): string[] {
|
||||
const allowed = new Set(opts.allowedObjects.map((o) => o.toLowerCase()));
|
||||
const ctes = new Set(cteNames);
|
||||
const found = new Set<string>();
|
||||
|
||||
const re = /(^|[^A-Za-z0-9_])(from|join)([^A-Za-z0-9_]|$)/gi;
|
||||
let m: RegExpExecArray | null;
|
||||
while ((m = re.exec(stripped)) !== null) {
|
||||
const after = stripped.slice(m.index + m[0].length - 1);
|
||||
const target = /^\s*([A-Za-z_][A-Za-z0-9_.]*|\()/.exec(after);
|
||||
if (!target) deny('could not resolve a FROM/JOIN target');
|
||||
const token = target[1];
|
||||
if (token === '(') continue; // subquery; its own FROM is scanned by this loop
|
||||
|
||||
const rest = after.slice(target[0].length);
|
||||
if (/^\s*\(/.test(rest)) {
|
||||
deny(`table function "${token}" is not permitted in an analyst session`);
|
||||
}
|
||||
|
||||
const parts = token.split('.');
|
||||
if (parts.length === 1) {
|
||||
if (!ctes.has(parts[0].toLowerCase())) {
|
||||
deny(
|
||||
`unqualified table "${parts[0]}" — an analyst statement must name ` +
|
||||
`${opts.allowedCatalog}.<object>, or a CTE it defined itself`,
|
||||
);
|
||||
}
|
||||
continue;
|
||||
}
|
||||
if (parts.length !== 2 || !QUALIFIED_RE.test(token)) {
|
||||
deny(`"${token}" is not a valid <catalog>.<object> reference`);
|
||||
}
|
||||
const [catalog, object] = parts;
|
||||
if (catalog.toLowerCase() !== opts.allowedCatalog.toLowerCase()) {
|
||||
deny(`catalog "${catalog}" is not readable by this session`);
|
||||
}
|
||||
if (!allowed.has(object.toLowerCase())) {
|
||||
deny(`object "${catalog}.${object}" is not readable by this session`);
|
||||
}
|
||||
found.add(`${catalog}.${object}`);
|
||||
}
|
||||
return [...found];
|
||||
}
|
||||
|
||||
/** Exported for the test corpus and for operator documentation. */
|
||||
export const GUARD_BANNED_KEYWORDS = BANNED_KEYWORDS;
|
||||
export const GUARD_BANNED_FUNCTIONS = BANNED_FUNCTIONS;
|
||||
@@ -0,0 +1,238 @@
|
||||
/**
|
||||
* src/analytics/sql/tenant-catalog.ts
|
||||
* ====================================
|
||||
* BI-P017 — ANALYST SQL CONTRACT, layers 3 and 4.
|
||||
*
|
||||
* The analyst never touches a base table. Each tenant gets its OWN ClickHouse
|
||||
* database containing nothing but VIEWS, and every view has the BI-P011 forced
|
||||
* predicate baked into its own definition:
|
||||
*
|
||||
* CREATE VIEW t_alpha_sql.opt_analytics_event AS
|
||||
* SELECT … FROM opt_analytics.opt_analytics_event
|
||||
* WHERE tenant_id = 't_alpha'
|
||||
*
|
||||
* That is the difference between "the guard rewrites your SQL to add a WHERE"
|
||||
* (fragile — it depends on the guard being a correct SQL parser) and "there is
|
||||
* no unfiltered object in reach" (structural — it depends only on the grants).
|
||||
*
|
||||
* The session's ClickHouse user is created with `readonly = 1` and is granted
|
||||
* SELECT on that database ALONE. `GRANT SELECT ON opt_analytics.*` is never
|
||||
* issued, so a statement the guard wrongly admits still cannot read the base
|
||||
* table: ClickHouse answers "Not enough privileges", not rows.
|
||||
*
|
||||
* The `tenant_id` column is deliberately DROPPED from the view projection.
|
||||
* An analyst cannot filter on, group by, or select a column that does not
|
||||
* exist in the object they can see — which removes the whole class of
|
||||
* "predicate on tenant_id looks legitimate" confusion from the audit trail.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import { assertSafeIdentifier } from '../engine/query-compiler';
|
||||
import type { ResourceBudget } from '../engine/resource-governance';
|
||||
|
||||
export interface TenantSqlCatalogSpec {
|
||||
/** Shared analytics database holding the base tables. */
|
||||
sourceDatabase: string;
|
||||
/** Verified tenant id. Embedded as a literal — validated strictly first. */
|
||||
tenantId: string;
|
||||
/** The tenant's own catalog name, e.g. `t_alpha_sql`. */
|
||||
catalog: string;
|
||||
/** Base tables to expose, with the columns each view projects. */
|
||||
objects: ReadonlyArray<{ table: string; columns: readonly string[] }>;
|
||||
}
|
||||
|
||||
export interface TenantSqlUserSpec extends TenantSqlCatalogSpec {
|
||||
userName: string;
|
||||
/** From a secret ref. Never a literal in code, never logged. */
|
||||
password: string;
|
||||
/**
|
||||
* BI-P012 ceilings. MEASURED CONSTRAINT, not a style choice: a ClickHouse
|
||||
* user whose profile carries `readonly = 1` cannot have ANY setting changed
|
||||
* from the request — the server answers
|
||||
*
|
||||
* Code: 164. Cannot modify 'max_rows_to_read' setting in readonly mode.
|
||||
*
|
||||
* (measured on 24.8.14.39; see evidence/bi-p017/). So the ceilings cannot be
|
||||
* client-supplied per request the way `clickhouseSettings()` supplies them
|
||||
* for the non-readonly engine user. They must be baked into the user's
|
||||
* SETTINGS PROFILE at provisioning time.
|
||||
*
|
||||
* That is strictly BETTER: the ceiling no longer depends on the client
|
||||
* remembering to send it, and the same `readonly = 1` that forbids raising
|
||||
* it also forbids lowering it or turning it off.
|
||||
*/
|
||||
budget?: ResourceBudget;
|
||||
/** Explicit profile name; defaults to `<userName>_profile`. */
|
||||
settingsProfile?: string;
|
||||
}
|
||||
|
||||
const TENANT_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
|
||||
|
||||
function assertTenantId(tenantId: string): string {
|
||||
if (!TENANT_ID_RE.test(tenantId)) {
|
||||
// ClickHouse view definitions take no bound parameters, so validation is
|
||||
// the only protection available here and it is applied strictly.
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', 'tenant id is not catalog-safe');
|
||||
}
|
||||
return tenantId;
|
||||
}
|
||||
|
||||
/**
|
||||
* DDL creating the tenant's read-only catalog of pre-filtered views.
|
||||
* Idempotent; safe to re-run on every deploy.
|
||||
*/
|
||||
export function tenantSqlCatalogDdl(spec: TenantSqlCatalogSpec): string[] {
|
||||
assertSafeIdentifier(spec.sourceDatabase, 'database');
|
||||
assertSafeIdentifier(spec.catalog, 'catalog');
|
||||
assertTenantId(spec.tenantId);
|
||||
if (spec.catalog === spec.sourceDatabase) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.sql',
|
||||
'tenant catalog must not be the source database',
|
||||
);
|
||||
}
|
||||
const stmts: string[] = [`CREATE DATABASE IF NOT EXISTS ${spec.catalog}`];
|
||||
for (const obj of spec.objects) {
|
||||
assertSafeIdentifier(obj.table, 'table');
|
||||
if (obj.columns.length === 0) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', `view ${obj.table} projects no columns`);
|
||||
}
|
||||
for (const col of obj.columns) {
|
||||
assertSafeIdentifier(col, 'column');
|
||||
if (col === 'tenant_id') {
|
||||
// Not an oversight to guard against — a deliberate refusal. Exposing
|
||||
// tenant_id inside a single-tenant view invites a reader to believe
|
||||
// filtering on it means something.
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.sql',
|
||||
'tenant_id must not be projected into a tenant SQL view',
|
||||
);
|
||||
}
|
||||
}
|
||||
stmts.push(
|
||||
`CREATE OR REPLACE VIEW ${spec.catalog}.${obj.table} AS ` +
|
||||
`SELECT ${obj.columns.join(', ')} FROM ${spec.sourceDatabase}.${obj.table} ` +
|
||||
`WHERE tenant_id = '${spec.tenantId}'`,
|
||||
);
|
||||
}
|
||||
return stmts;
|
||||
}
|
||||
|
||||
/**
|
||||
* DDL for the session's ClickHouse user.
|
||||
*
|
||||
* Every line here is a layer-3 control:
|
||||
* * `readonly = 1` — no writes AND no settings changes from inside a query,
|
||||
* which is what stops `SETTINGS max_rows_to_read = …` lifting the ceiling.
|
||||
* * `allow_ddl = 0` — belt to readonly's braces.
|
||||
* * SELECT granted on the tenant catalog ONLY. The base database is never
|
||||
* granted, so a guard bypass hits a privilege error.
|
||||
* * A row policy on the BASE tables as well, so even a mistaken grant, or a
|
||||
* view accidentally created without its predicate, still filters.
|
||||
*/
|
||||
export function tenantSqlUserDdl(spec: TenantSqlUserSpec): string[] {
|
||||
assertSafeIdentifier(spec.userName, 'user');
|
||||
assertSafeIdentifier(spec.catalog, 'catalog');
|
||||
assertSafeIdentifier(spec.sourceDatabase, 'database');
|
||||
assertTenantId(spec.tenantId);
|
||||
if (spec.password.includes("'") || spec.password.includes('\\')) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.sql',
|
||||
'password contains characters that cannot be embedded',
|
||||
);
|
||||
}
|
||||
if (spec.settingsProfile) assertSafeIdentifier(spec.settingsProfile, 'settings profile');
|
||||
|
||||
const profileName = spec.settingsProfile ?? `${spec.userName}_profile`;
|
||||
assertSafeIdentifier(profileName, 'settings profile');
|
||||
|
||||
const stmts: string[] = [];
|
||||
if (spec.budget) {
|
||||
// Every BI-P012 ceiling, plus the overflow modes. `throw` is explicit
|
||||
// because ClickHouse's default for some of these is `break`, which returns
|
||||
// a PARTIAL result that looks complete — the analytics equivalent of a
|
||||
// false green.
|
||||
// `OR REPLACE`, for the same measured reason as the row policy below:
|
||||
// `CREATE SETTINGS PROFILE IF NOT EXISTS` is a silent no-op when the
|
||||
// profile already exists, so a CHANGED budget never lands. Found the hard
|
||||
// way — a re-provisioning run kept a stale 60-row ceiling and starved
|
||||
// every legitimate analyst statement in the integration suite, while the
|
||||
// provisioning DDL reported success on every statement.
|
||||
stmts.push(
|
||||
`CREATE SETTINGS PROFILE OR REPLACE ${profileName} SETTINGS ` +
|
||||
`max_execution_time = ${spec.budget.maxExecutionTimeSec}, ` +
|
||||
`timeout_overflow_mode = 'throw', ` +
|
||||
`max_result_rows = ${spec.budget.maxResultRows}, ` +
|
||||
`result_overflow_mode = 'throw', ` +
|
||||
`max_rows_to_read = ${spec.budget.maxRowsToRead}, ` +
|
||||
`max_bytes_to_read = ${spec.budget.maxBytesToRead}, ` +
|
||||
`read_overflow_mode = 'throw', ` +
|
||||
`max_memory_usage = ${spec.budget.maxMemoryUsageBytes}, ` +
|
||||
`readonly = 1`,
|
||||
);
|
||||
}
|
||||
const settings = spec.budget ? `SETTINGS PROFILE ${profileName}` : `SETTINGS readonly = 1, allow_ddl = 0`;
|
||||
|
||||
stmts.push(
|
||||
`CREATE USER IF NOT EXISTS ${spec.userName} IDENTIFIED WITH sha256_password BY '${spec.password}' ${settings}`,
|
||||
);
|
||||
// Re-applied on every provisioning run so a rotated budget actually lands on
|
||||
// an already-existing user; CREATE USER IF NOT EXISTS would silently skip it.
|
||||
stmts.push(`ALTER USER ${spec.userName} ${settings}`);
|
||||
stmts.push(`GRANT SELECT ON ${spec.catalog}.* TO ${spec.userName}`);
|
||||
for (const obj of spec.objects) {
|
||||
assertSafeIdentifier(obj.table, 'table');
|
||||
// A view reads its source with the DEFINER's rights in ClickHouse only for
|
||||
// parameterised/materialised cases; for a plain view the reader still needs
|
||||
// SELECT on the source. Grant it on the SOURCE TABLE and pin a row policy
|
||||
// to it, so the grant can never be wider than the tenant.
|
||||
stmts.push(`GRANT SELECT ON ${spec.sourceDatabase}.${obj.table} TO ${spec.userName}`);
|
||||
// `OR REPLACE`, never `IF NOT EXISTS`. MEASURED FAILURE, found by a test
|
||||
// that attempted the bypass:
|
||||
//
|
||||
// Dropping and re-creating the user empties the policy's apply-to list
|
||||
// (ClickHouse removes the dropped user from it). On the next
|
||||
// provisioning run `CREATE ROW POLICY IF NOT EXISTS` sees the policy
|
||||
// already exists and does NOTHING — so the policy survives with
|
||||
// `apply_to_list = []`, applying to nobody, while `system.row_policies`
|
||||
// still shows the right `select_filter`. An operator reading that table
|
||||
// sees a policy protecting `opt_analytics.sales` and is wrong: the
|
||||
// restricted user read 100 rows of 100, both tenants.
|
||||
//
|
||||
// This is the "green check that does not gate" shape. `OR REPLACE` makes
|
||||
// provisioning idempotent in the way that was intended, and
|
||||
// `rowPolicyVerificationSql` below turns "the policy is applied" from an
|
||||
// assumption into a query.
|
||||
stmts.push(
|
||||
`CREATE ROW POLICY OR REPLACE ${spec.userName}_${obj.table}_rp ON ${spec.sourceDatabase}.${obj.table} ` +
|
||||
`AS restrictive FOR SELECT USING tenant_id = '${spec.tenantId}' TO ${spec.userName}`,
|
||||
);
|
||||
}
|
||||
return stmts;
|
||||
}
|
||||
|
||||
/**
|
||||
* The query that PROVES the row policies landed, rather than assuming it.
|
||||
*
|
||||
* Returns one row per (table, policy) with `applied_to_user` — which must be
|
||||
* 1 for every object. Run it after provisioning and fail the deploy if any
|
||||
* row reads 0; a policy with an empty apply-to list is indistinguishable from
|
||||
* a working one in `SHOW ROW POLICIES`.
|
||||
*/
|
||||
export function rowPolicyVerificationSql(spec: TenantSqlUserSpec): string {
|
||||
assertSafeIdentifier(spec.userName, 'user');
|
||||
assertSafeIdentifier(spec.sourceDatabase, 'database');
|
||||
assertTenantId(spec.tenantId);
|
||||
return (
|
||||
`SELECT table, short_name, has(apply_to_list, '${spec.userName}') AS applied_to_user, select_filter ` +
|
||||
`FROM system.row_policies WHERE database = '${spec.sourceDatabase}' ` +
|
||||
`AND short_name LIKE '${spec.userName}%' FORMAT JSON`
|
||||
);
|
||||
}
|
||||
|
||||
/** Object names the guard's allow-list is built from. */
|
||||
export function catalogObjectNames(spec: TenantSqlCatalogSpec): string[] {
|
||||
return spec.objects.map((o) => o.table);
|
||||
}
|
||||
@@ -0,0 +1,374 @@
|
||||
/**
|
||||
* src/analytics/transform/dbt-contract.ts
|
||||
* ========================================
|
||||
* BI-P022 — DBT CORE CANONICAL TRANSFORMATION.
|
||||
*
|
||||
* ── SCOPE, STATED FIRST ───────────────────────────────────────────────────
|
||||
* This module defines the CONTRACT. It does not run dbt, and nothing in this
|
||||
* repo does. Execution belongs to N0D (n0d#38), which consumes this contract
|
||||
* from the other side. There is deliberately no `DbtExecutor` implementation
|
||||
* here — only the interface — so that "0pt can run dbt" cannot become true by
|
||||
* accident, and so the two repos have one shared, versioned shape rather than
|
||||
* two drifting ones.
|
||||
*
|
||||
* ── THE VERSION RULE BI-P022 STATES ───────────────────────────────────────
|
||||
* "Use stable dbt Core production line AFTER current license/security
|
||||
* verification. Do NOT use alpha development engine in production unless
|
||||
* separately certified."
|
||||
*
|
||||
* So the pin is recorded together with its certification STATE, and the state
|
||||
* is `unverified` until someone does the review. `assertDbtRuntimeCertified`
|
||||
* refuses a production run against an uncertified or alpha runtime — which
|
||||
* makes the sentence above a gate rather than a paragraph. A version constant
|
||||
* on its own would be exactly the "documented but unenforced" pattern.
|
||||
*
|
||||
* ── THE TIER MAPPING ──────────────────────────────────────────────────────
|
||||
* BI-P022's DoD names raw → staging → curated → serving. BI-P003 (merged)
|
||||
* already owns RAW/NORMALIZED/CURATED/SERVING, so the dbt layers map onto it
|
||||
* rather than inventing a fifth vocabulary:
|
||||
*
|
||||
* sources/ -> RAW (declared, never modelled)
|
||||
* staging/ -> NORMALIZED (rename, cast, deduplicate; no business logic)
|
||||
* intermediate/ -> CURATED (joins, business logic, conformed entities)
|
||||
* marts/ -> SERVING (what a dashboard or a metric reads)
|
||||
*
|
||||
* A model in the wrong directory for its tier is a rejection, not a warning:
|
||||
* the tier is what BI-P003's read rule gates on, and a mart that is really a
|
||||
* staging model is how RAW data reaches a dashboard.
|
||||
*/
|
||||
import { SiaxError } from '../../platform/errors';
|
||||
import type { AnalyticsTier } from '../domain/tiers';
|
||||
|
||||
/** dbt layers, in dependency order. */
|
||||
export const DBT_LAYERS = ['sources', 'staging', 'intermediate', 'marts'] as const;
|
||||
export type DbtLayer = (typeof DBT_LAYERS)[number];
|
||||
|
||||
export const DBT_LAYER_TIERS: Record<DbtLayer, AnalyticsTier> = {
|
||||
sources: 'RAW',
|
||||
staging: 'NORMALIZED',
|
||||
intermediate: 'CURATED',
|
||||
marts: 'SERVING',
|
||||
};
|
||||
|
||||
/** Directories BI-P022 requires under `analytics/dbt/`. */
|
||||
export const REQUIRED_DBT_DIRECTORIES = [
|
||||
'models/sources',
|
||||
'models/staging',
|
||||
'models/intermediate',
|
||||
'models/marts',
|
||||
'tests',
|
||||
'snapshots',
|
||||
] as const;
|
||||
|
||||
// ── runtime certification ──────────────────────────────────────────────────
|
||||
|
||||
export type DbtReleaseLine = 'stable' | 'prerelease' | 'alpha';
|
||||
|
||||
export interface DbtRuntimePin {
|
||||
/** e.g. "1.10.13". Exact, never a range. */
|
||||
version: string;
|
||||
line: DbtReleaseLine;
|
||||
/** Adapter and its pin, e.g. { name: "dbt-clickhouse", version: "1.9.2" }. */
|
||||
adapter: { name: string; version: string };
|
||||
/** Has a licence + supply-chain review been completed FOR THIS PIN? */
|
||||
licenseReviewed: boolean;
|
||||
securityReviewed: boolean;
|
||||
reviewedBy?: string;
|
||||
reviewedAt?: string;
|
||||
/** Required by BI-P022 to run an alpha line in production at all. */
|
||||
separateCertificationRef?: string;
|
||||
}
|
||||
|
||||
/**
|
||||
* The pin this contract is written against.
|
||||
*
|
||||
* BOTH review flags are FALSE. No licence or supply-chain review of dbt Core
|
||||
* has been performed in this repo, and none is claimed. The constant records
|
||||
* an intended pin; `assertDbtRuntimeCertified` is what stops it being treated
|
||||
* as an approval. Flipping these to true without doing the review is the
|
||||
* failure this shape exists to make visible.
|
||||
*/
|
||||
export const DBT_RUNTIME_PIN: DbtRuntimePin = {
|
||||
version: '1.10.13',
|
||||
line: 'stable',
|
||||
adapter: { name: 'dbt-clickhouse', version: '1.9.2' },
|
||||
licenseReviewed: false,
|
||||
securityReviewed: false,
|
||||
};
|
||||
|
||||
export type DbtRunEnvironment = 'development' | 'ci' | 'production';
|
||||
|
||||
/** BI-P022's version rule, as a gate. */
|
||||
export function assertDbtRuntimeCertified(pin: DbtRuntimePin, environment: DbtRunEnvironment): void {
|
||||
if (!/^\d+\.\d+\.\d+$/.test(pin.version)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
'the dbt runtime pin must be an exact version',
|
||||
);
|
||||
}
|
||||
if (environment !== 'production') return;
|
||||
if (pin.line === 'alpha' && !pin.separateCertificationRef) {
|
||||
throw new SiaxError(
|
||||
'ENTITLEMENT_REQUIRED',
|
||||
'0pt.analytics.dbt',
|
||||
'BI-P022: an alpha dbt engine may not run in production without a separate certification reference',
|
||||
);
|
||||
}
|
||||
if (!pin.licenseReviewed || !pin.securityReviewed) {
|
||||
throw new SiaxError(
|
||||
'ENTITLEMENT_REQUIRED',
|
||||
'0pt.analytics.dbt',
|
||||
`BI-P022: dbt ${pin.version} has not completed licence and security verification; ` +
|
||||
`it may not run in production`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
// ── the job contract N0D executes ──────────────────────────────────────────
|
||||
|
||||
export const DBT_COMMANDS = [
|
||||
'build',
|
||||
'run',
|
||||
'test',
|
||||
'snapshot',
|
||||
'seed',
|
||||
'compile',
|
||||
'source_freshness',
|
||||
] as const;
|
||||
export type DbtCommand = (typeof DBT_COMMANDS)[number];
|
||||
|
||||
/**
|
||||
* What 0pt hands N0D. Deliberately narrow: there is no free-form `args` field,
|
||||
* because a passthrough argument list is a shell surface, and a shell surface
|
||||
* on a job that holds warehouse credentials is the whole problem.
|
||||
*/
|
||||
export interface DbtJobRequest {
|
||||
/** Verified, from CL0UD. Never read from a request body. */
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
/** Which analytics project to run — a registered name, not a path. */
|
||||
dbt_project: string;
|
||||
/** Git ref of the analytics repo the models come from. Exact SHA. */
|
||||
revision: string;
|
||||
command: DbtCommand;
|
||||
/** dbt node selectors. Validated; never concatenated into a shell string. */
|
||||
select?: string[];
|
||||
exclude?: string[];
|
||||
/** Target layer, which N0D maps to a credential with matching write scope. */
|
||||
target_layer: DbtLayer;
|
||||
environment: DbtRunEnvironment;
|
||||
runtime: DbtRuntimePin;
|
||||
/** Idempotency key, per the standard N0D job contract. */
|
||||
idempotency_key: string;
|
||||
correlation_id?: string;
|
||||
/** Fail the run if it would write outside this tenant's schema. */
|
||||
expected_target_schema: string;
|
||||
}
|
||||
|
||||
/** The artefacts N0D must return. dbt produces all of them; they are not optional. */
|
||||
export const REQUIRED_DBT_ARTIFACTS = ['manifest.json', 'run_results.json'] as const;
|
||||
|
||||
export interface DbtJobResult {
|
||||
tenant_id: string;
|
||||
dbt_project: string;
|
||||
revision: string;
|
||||
command: DbtCommand;
|
||||
status: 'success' | 'error' | 'partial';
|
||||
started_at: string;
|
||||
finished_at: string;
|
||||
/** ST0RE artifact refs, one per REQUIRED_DBT_ARTIFACTS entry. */
|
||||
artifacts: Record<string, string>;
|
||||
/** Per-node outcomes, the input to BI-P025's DbtQualityProvider. */
|
||||
nodes: DbtNodeResult[];
|
||||
/** dbt's own version, as reported by the runtime that actually ran. */
|
||||
observed_dbt_version: string;
|
||||
}
|
||||
|
||||
export interface DbtNodeResult {
|
||||
unique_id: string;
|
||||
resource_type: 'model' | 'test' | 'snapshot' | 'seed' | 'source';
|
||||
status: 'success' | 'error' | 'skipped' | 'pass' | 'fail' | 'warn';
|
||||
rows_affected?: number;
|
||||
execution_time_s?: number;
|
||||
message?: string;
|
||||
}
|
||||
|
||||
const SELECTOR_RE = /^[A-Za-z0-9_.:*+@ -]{1,200}$/;
|
||||
const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/;
|
||||
const SHA_RE = /^[0-9a-f]{40}$/;
|
||||
|
||||
export function assertValidDbtJobRequest(req: DbtJobRequest): void {
|
||||
if (!req.tenant_id?.trim()) {
|
||||
throw new SiaxError('UNAUTHENTICATED', '0pt.analytics.dbt', 'a dbt job requires a verified tenant');
|
||||
}
|
||||
if (!NAME_RE.test(req.dbt_project)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
'dbt_project must be a registered snake_case name',
|
||||
);
|
||||
}
|
||||
if (!SHA_RE.test(req.revision)) {
|
||||
// A branch name is not reproducible, and BI-P022's DoD is about
|
||||
// reproducibility. "main" today and "main" tomorrow are different projects.
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
'revision must be an exact 40-character commit SHA — a branch name is not reproducible',
|
||||
);
|
||||
}
|
||||
if (!(DBT_COMMANDS as readonly string[]).includes(req.command)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unsupported dbt command "${req.command}"`);
|
||||
}
|
||||
if (!(DBT_LAYERS as readonly string[]).includes(req.target_layer)) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unknown target layer "${req.target_layer}"`);
|
||||
}
|
||||
if (req.target_layer === 'sources') {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
'sources are DECLARED, never built — a job cannot target the sources layer',
|
||||
);
|
||||
}
|
||||
for (const selector of [...(req.select ?? []), ...(req.exclude ?? [])]) {
|
||||
if (!SELECTOR_RE.test(selector)) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
`dbt selector "${selector.slice(0, 40)}" contains characters that are not selector syntax`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (!req.idempotency_key?.trim()) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
'the N0D job contract requires an idempotency key',
|
||||
);
|
||||
}
|
||||
if (!req.expected_target_schema.includes(req.tenant_id)) {
|
||||
// The target schema must be derivable from the tenant. A job that can name
|
||||
// an arbitrary schema is a cross-tenant write waiting to happen.
|
||||
throw new SiaxError(
|
||||
'AUTHORIZATION_DENIED',
|
||||
'0pt.analytics.dbt',
|
||||
"expected_target_schema must be scoped to the job's tenant",
|
||||
);
|
||||
}
|
||||
assertDbtRuntimeCertified(req.runtime, req.environment);
|
||||
}
|
||||
|
||||
/**
|
||||
* Validate what came BACK. A result is untrusted input: it arrives from
|
||||
* another repo's executor, and the estate has been burned by trusting a green
|
||||
* status that meant nothing.
|
||||
*/
|
||||
export function assertValidDbtJobResult(result: DbtJobResult, request: DbtJobRequest): void {
|
||||
if (result.tenant_id !== request.tenant_id) {
|
||||
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.dbt', 'dbt result is for a different tenant');
|
||||
}
|
||||
if (result.revision !== request.revision) {
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.dbt',
|
||||
'dbt result is for a different revision than was requested — the run is not reproducible',
|
||||
);
|
||||
}
|
||||
for (const artifact of REQUIRED_DBT_ARTIFACTS) {
|
||||
if (!result.artifacts[artifact]) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dbt',
|
||||
`dbt result is missing "${artifact}" — without it the run cannot be audited or its tests ingested`,
|
||||
);
|
||||
}
|
||||
}
|
||||
if (result.observed_dbt_version !== request.runtime.version) {
|
||||
// A runtime that is not the pinned one silently changes what the models
|
||||
// mean. Catching it here is the difference between a pin and a wish.
|
||||
throw new SiaxError(
|
||||
'CONFLICT',
|
||||
'0pt.analytics.dbt',
|
||||
`dbt ran as ${result.observed_dbt_version} but ${request.runtime.version} was pinned`,
|
||||
);
|
||||
}
|
||||
if (result.status === 'success' && result.nodes.length === 0) {
|
||||
// A "successful" run that built nothing is the dbt-shaped false green: a
|
||||
// selector that matched no models exits 0.
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dbt',
|
||||
'dbt reported success but executed zero nodes — a selector that matches nothing exits 0',
|
||||
);
|
||||
}
|
||||
const failed = result.nodes.filter((n) => n.status === 'error' || n.status === 'fail');
|
||||
if (result.status === 'success' && failed.length > 0) {
|
||||
throw new SiaxError(
|
||||
'PERMANENT_FAILURE',
|
||||
'0pt.analytics.dbt',
|
||||
`dbt reported success while ${failed.length} node(s) failed`,
|
||||
);
|
||||
}
|
||||
}
|
||||
|
||||
/**
|
||||
* The executor N0D implements. Declared so both repos compile against one
|
||||
* shape; deliberately NOT implemented here.
|
||||
*/
|
||||
export interface DbtExecutor {
|
||||
readonly runsIn: 'n0d';
|
||||
execute(request: DbtJobRequest): Promise<DbtJobResult>;
|
||||
}
|
||||
|
||||
/** Which tier a model path belongs to, or a rejection. */
|
||||
export function tierForModelPath(modelPath: string): AnalyticsTier {
|
||||
const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(modelPath);
|
||||
if (!match) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
`model "${modelPath}" is not under one of ${DBT_LAYERS.join('/')} — BI-P003 cannot assign it a tier`,
|
||||
);
|
||||
}
|
||||
return DBT_LAYER_TIERS[match[1] as DbtLayer];
|
||||
}
|
||||
|
||||
/**
|
||||
* A staging model must not reach past sources, a mart must not read a source
|
||||
* directly, and nothing may skip a layer. Enforced on the dependency graph
|
||||
* from `manifest.json`, because "we follow the convention" is not a check.
|
||||
*/
|
||||
const LAYER_ORDER: DbtLayer[] = ['sources', 'staging', 'intermediate', 'marts'];
|
||||
|
||||
export function assertLayerDependenciesLegal(edges: Array<{ from: string; to: string }>): void {
|
||||
for (const edge of edges) {
|
||||
const fromLayer = layerOf(edge.from);
|
||||
const toLayer = layerOf(edge.to);
|
||||
const fromIdx = LAYER_ORDER.indexOf(fromLayer);
|
||||
const toIdx = LAYER_ORDER.indexOf(toLayer);
|
||||
if (toIdx > fromIdx) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
`"${edge.from}" (${fromLayer}) depends on "${edge.to}" (${toLayer}) — a model may not read a later layer`,
|
||||
);
|
||||
}
|
||||
if (fromIdx - toIdx > 1) {
|
||||
throw new SiaxError(
|
||||
'INVALID_REQUEST',
|
||||
'0pt.analytics.dbt',
|
||||
`"${edge.from}" (${fromLayer}) skips a layer to read "${edge.to}" (${toLayer}) — ` +
|
||||
`raw data reaching a mart directly is what BI-P003's tier rule exists to prevent`,
|
||||
);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function layerOf(path: string): DbtLayer {
|
||||
const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(path);
|
||||
if (!match) {
|
||||
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `model "${path}" is not in a known layer`);
|
||||
}
|
||||
return match[1] as DbtLayer;
|
||||
}
|
||||
Reference in New Issue
Block a user