fix(recovery): restore source removed by production documentation merge

This commit is contained in:
2026-11-10 03:42:10 +01:00
parent 585bdd4b44
commit 3d2fc6f913
957 changed files with 115754 additions and 0 deletions
+237
View File
@@ -0,0 +1,237 @@
/**
* src/analytics/arrow/flatbuffer-builder.ts
* ==========================================
* A minimal FlatBuffers writer, sufficient for the Arrow IPC metadata
* messages this repo emits (BI-P005).
*
* WHY HAND-ROLLED rather than `apache-arrow` from npm: adding a runtime
* dependency to 0pt's blocking-typechecked capability layer means a lockfile
* change, a new supply-chain surface, and a new CVE surface for every
* consumer of this repo — for a format whose METADATA side is a few hundred
* bytes of well-specified binary and whose DATA side is plain columnar memory
* either way.
*
* The correctness bar is NOT "this looks right". The BI-P005 evidence script
* (scripts/bi-p005-arrow-roundtrip.mjs) decodes this writer's output with
* pyarrow AND DuckDB — two independent implementations of the specification —
* so a mistake here surfaces as a failed decode by somebody else's parser,
* not as a green test of my own assumptions.
*
* Layout facts implemented (FlatBuffers binary format, little-endian):
* * the buffer is built BACK TO FRONT; `offset` counts bytes from the end;
* * a uoffset stored at a field site holds `site_offset + 4 - target_offset`
* so a reader resolves it as `position + value`;
* * a table begins with an soffset `table_start - vtable_start`;
* * a vtable is `[vtable_len:i16][table_len:i16][field_offsets:i16...]`,
* with 0 meaning "field absent, use the default".
*/
export type Offset = number;
const SIZEOF_INT = 4;
export class FlatBufferBuilder {
private buf: Uint8Array;
private view: DataView;
/** Bytes written so far, counted from the END of `buf`. */
private used = 0;
private minAlign = 1;
private vtable: number[] = [];
private objectStart = 0;
private inObject = false;
/** Offsets of previously written vtables, for deduplication. */
private vtables: number[] = [];
constructor(initial = 1024) {
this.buf = new Uint8Array(initial);
this.view = new DataView(this.buf.buffer);
}
/** Bytes written so far (FlatBuffers "offset"). */
get offset(): number {
return this.used;
}
private ensure(extra: number): void {
if (this.used + extra <= this.buf.length) return;
let size = this.buf.length;
while (size < this.used + extra) size *= 2;
const next = new Uint8Array(size);
next.set(this.buf.subarray(this.buf.length - this.used), size - this.used);
this.buf = next;
this.view = new DataView(this.buf.buffer);
}
private pad(n: number): void {
if (n <= 0) return;
this.ensure(n);
for (let i = 0; i < n; i++) {
this.used += 1;
this.buf[this.buf.length - this.used] = 0;
}
}
/** FlatBuffers `prep`: pad so that `additional` more bytes land aligned. */
private prep(size: number, additional = 0): void {
if (size > this.minAlign) this.minAlign = size;
const alignSize = (~(this.used + additional) + 1) & (size - 1);
this.pad(alignSize);
}
writeInt8(v: number): void {
this.ensure(1);
this.used += 1;
this.view.setInt8(this.buf.length - this.used, v);
}
writeInt16(v: number): void {
this.ensure(2);
this.used += 2;
this.view.setInt16(this.buf.length - this.used, v, true);
}
writeInt32(v: number): void {
this.ensure(4);
this.used += 4;
this.view.setInt32(this.buf.length - this.used, v, true);
}
writeInt64(v: bigint): void {
this.ensure(8);
this.used += 8;
this.view.setBigInt64(this.buf.length - this.used, v, true);
}
writeBytes(bytes: Uint8Array): void {
this.ensure(bytes.length);
this.used += bytes.length;
this.buf.set(bytes, this.buf.length - this.used);
}
addInt8(slot: number, value: number, def: number): void {
if (value === def) return;
this.prep(1);
this.writeInt8(value);
this.slot(slot);
}
addInt16(slot: number, value: number, def: number): void {
if (value === def) return;
this.prep(2);
this.writeInt16(value);
this.slot(slot);
}
addInt32(slot: number, value: number, def: number): void {
if (value === def) return;
this.prep(4);
this.writeInt32(value);
this.slot(slot);
}
addInt64(slot: number, value: bigint, def: bigint): void {
if (value === def) return;
this.prep(8);
this.writeInt64(value);
this.slot(slot);
}
addOffset(slot: number, value: Offset, def: number): void {
if (value === def) return;
this.prep(SIZEOF_INT);
this.writeInt32(this.used + SIZEOF_INT - value);
this.slot(slot);
}
private slot(slotNum: number): void {
this.vtable[slotNum] = this.used;
}
startObject(numFields: number): void {
if (this.inObject) throw new Error("nested startObject");
this.vtable = new Array<number>(numFields).fill(0);
this.objectStart = this.used;
this.inObject = true;
}
endObject(): Offset {
if (!this.inObject) throw new Error("endObject outside an object");
this.prep(SIZEOF_INT);
this.writeInt32(0); // soffset placeholder, patched below
const objectOffset = this.used;
let trimmed = this.vtable.length;
while (trimmed > 0 && this.vtable[trimmed - 1] === 0) trimmed -= 1;
for (let i = trimmed - 1; i >= 0; i--) {
this.writeInt16(this.vtable[i] !== 0 ? objectOffset - this.vtable[i] : 0);
}
this.writeInt16(objectOffset - this.objectStart); // table byte length
const vtLen = (trimmed + 2) * 2;
this.writeInt16(vtLen); // vtable byte length — lands at the vtable start
const vtStart = this.used;
const vtAbs = this.buf.length - vtStart;
let existing = 0;
for (const candidate of this.vtables) {
const candAbs = this.buf.length - candidate;
if (this.view.getInt16(candAbs, true) !== vtLen) continue;
let same = true;
for (let i = 2; i < vtLen; i += 2) {
if (this.view.getInt16(vtAbs + i, true) !== this.view.getInt16(candAbs + i, true)) {
same = false;
break;
}
}
if (same) {
existing = candidate;
break;
}
}
const objAbs = this.buf.length - objectOffset;
if (existing !== 0) {
this.used -= vtLen; // discard the duplicate vtable just written
this.view.setInt32(objAbs, existing - objectOffset, true);
} else {
this.vtables.push(vtStart);
this.view.setInt32(objAbs, this.used - objectOffset, true);
}
this.inObject = false;
return objectOffset;
}
createString(s: string): Offset {
const bytes = new TextEncoder().encode(s);
this.writeInt8(0); // null terminator (readers rely on it)
this.startVector(1, bytes.length, 1);
this.writeBytes(bytes);
return this.endVector(bytes.length);
}
startVector(elemSize: number, numElems: number, alignment: number): void {
this.prep(SIZEOF_INT, elemSize * numElems);
this.prep(alignment, elemSize * numElems);
}
endVector(numElems: number): Offset {
this.writeInt32(numElems);
return this.used;
}
/** Vector of table offsets. Elements are written in reverse build order. */
createOffsetVector(items: Offset[]): Offset {
this.startVector(SIZEOF_INT, items.length, SIZEOF_INT);
for (let i = items.length - 1; i >= 0; i--) {
this.writeInt32(this.used + SIZEOF_INT - items[i]);
}
return this.endVector(items.length);
}
/** Finish the buffer with `root` as the root table. Returns the bytes. */
finish(root: Offset): Uint8Array {
this.prep(this.minAlign, SIZEOF_INT);
this.prep(SIZEOF_INT);
this.writeInt32(this.used + SIZEOF_INT - root);
return this.buf.slice(this.buf.length - this.used);
}
}
+547
View File
@@ -0,0 +1,547 @@
/**
* src/analytics/arrow/ipc.ts
* ===========================
* BI-P005 — APACHE ARROW INTERCHANGE.
*
* Canonical columnar result encoding ALONGSIDE JSON (JSON is not removed —
* BI-P005 says "alongside"). Two paths, and knowing which is which matters:
*
* PASS-THROUGH (preferred for ClickHouse results): ClickHouse emits real
* Arrow IPC itself via `FORMAT ArrowStream`. Those bytes were produced by
* the Arrow C++ implementation; 0pt streams them out unchanged, so there is
* no encoder of ours in the path at all. `clickhouseArrowStreamFormat()`
* and `arrowStreamSettings()` are that path.
*
* ENCODED (for results 0pt itself assembles — projections, JSON-sourced
* rows, exports): `encodeArrowStream()` writes the IPC stream here.
*
* Both paths are decodable by `decodeArrowStream()`, which exists so 0pt can
* count rows and read the schema of an Arrow payload WITHOUT shelling out to
* another runtime — needed to enforce BI-P012's `max_result_rows` on the
* pass-through path, where the row count is not otherwise visible.
*
* "Supported types" (the DoD's qualifier) is `ARROW_TYPE_SUPPORT` below: the
* exact ClickHouse -> Arrow -> DuckDB/Python mapping, including where
* semantic loss WOULD occur, stated rather than hidden.
*
* IPC stream layout (Arrow format 1.5, metadata V5):
* for each message: 0xFFFFFFFF, int32 metadata length, FlatBuffer Message
* padded so the body starts 8-byte aligned, then the body buffers.
* Terminated by 0xFFFFFFFF 0x00000000.
*/
import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder";
import { SiaxError } from "../../platform/errors";
// ── FlatBuffer enum values from org.apache.arrow.flatbuf ────────────────────
const METADATA_VERSION_V5 = 4;
const MESSAGE_HEADER_SCHEMA = 1;
const MESSAGE_HEADER_RECORD_BATCH = 3;
const ENDIANNESS_LITTLE = 0;
const TYPE_INT = 2;
const TYPE_FLOATING_POINT = 3;
const TYPE_UTF8 = 5;
const TYPE_BOOL = 6;
const TYPE_TIMESTAMP = 10;
const PRECISION_DOUBLE = 2;
// BigInt LITERALS (`0n`) require target >= ES2020; this repo targets ES6, so
// every 64-bit constant is built with BigInt(...) instead. Behaviour is
// identical — this is a compiler-target accommodation, not a semantic choice.
const BIG_ZERO = BigInt(0);
const INT64_MAX = BigInt("9223372036854775807");
const INT64_MIN = BigInt("-9223372036854775808");
const TIME_UNIT_MILLISECOND = 1;
// ── public column model ────────────────────────────────────────────────────
export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms";
export interface ArrowColumn {
name: string;
type: ArrowColumnType;
/** `null` marks a null slot; the validity bitmap is derived from it. */
values: (number | bigint | string | boolean | Date | null)[];
}
/**
* The type table BI-P005's "supported types" qualifier refers to. Every row
* has been exercised end to end by scripts/bi-p005-arrow-roundtrip.mjs.
*/
export const ARROW_TYPE_SUPPORT: {
clickhouse: string;
arrow: ArrowColumnType;
duckdb: string;
python: string;
note: string;
}[] = [
{
clickhouse: "Int64 / UInt32 / Int32",
arrow: "int64",
duckdb: "BIGINT",
python: "int",
note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped",
},
{
clickhouse: "Float64",
arrow: "float64",
duckdb: "DOUBLE",
python: "float",
note: "exact IEEE-754 round trip",
},
{
clickhouse: "String / LowCardinality(String)",
arrow: "utf8",
duckdb: "VARCHAR",
python: "str",
note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not",
},
{
clickhouse: "Bool / UInt8(0|1)",
arrow: "bool",
duckdb: "BOOLEAN",
python: "bool",
note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64",
},
{
clickhouse: "DateTime64(3,'UTC')",
arrow: "timestamp_ms",
duckdb: "TIMESTAMP",
python: "datetime",
note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set",
},
];
// ── ClickHouse pass-through ────────────────────────────────────────────────
/** ClickHouse output format producing an Arrow IPC *stream*. */
export function clickhouseArrowStreamFormat(): string {
return "ArrowStream";
}
/**
* Settings for the pass-through path. `output_format_arrow_string_as_string`
* matters: without it ClickHouse emits String columns as Arrow *Binary*,
* which pyarrow and DuckDB both read as bytes rather than text — semantic
* loss that would slip through a test that only compares row counts.
*/
export function arrowStreamSettings(): Record<string, string> {
return {
output_format_arrow_string_as_string: "1",
output_format_arrow_low_cardinality_as_dictionary: "0",
};
}
// ── encoder ────────────────────────────────────────────────────────────────
function alignTo8(n: number): number {
return (n + 7) & ~7;
}
interface EncodedBuffer {
bytes: Uint8Array;
/** Logical (unpadded) length recorded in the RecordBatch metadata. */
length: number;
}
function validityBuffer(values: readonly unknown[]): { buf: EncodedBuffer; nullCount: number } {
let nullCount = 0;
const byteLen = Math.ceil(values.length / 8);
const bytes = new Uint8Array(alignTo8(byteLen));
for (let i = 0; i < values.length; i++) {
if (values[i] === null || values[i] === undefined) {
nullCount += 1;
} else {
bytes[i >> 3] |= 1 << (i & 7);
}
}
return { buf: { bytes, length: byteLen }, nullCount };
}
function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: number } {
const { buf: validity, nullCount } = validityBuffer(col.values);
const n = col.values.length;
switch (col.type) {
case "int64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v)));
if (b > INT64_MAX || b < INT64_MIN) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`);
}
dv.setBigInt64(i * 8, b, true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "float64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
dv.setFloat64(i * 8, Number(v), true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "timestamp_ms": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v);
dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case "bool": {
const byteLen = Math.ceil(n / 8);
const data = new Uint8Array(alignTo8(byteLen));
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
if (v) data[i >> 3] |= 1 << (i & 7);
});
return { buffers: [validity, { bytes: data, length: byteLen }], nullCount };
}
case "utf8": {
const encoded = col.values.map((v) =>
v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)),
);
const total = encoded.reduce((a, b) => a + b.length, 0);
const offsets = new Uint8Array(alignTo8((n + 1) * 4));
const odv = new DataView(offsets.buffer);
const data = new Uint8Array(alignTo8(total));
let pos = 0;
odv.setInt32(0, 0, true);
encoded.forEach((bytes, i) => {
data.set(bytes, pos);
pos += bytes.length;
odv.setInt32((i + 1) * 4, pos, true);
});
return {
buffers: [validity, { bytes: offsets, length: (n + 1) * 4 }, { bytes: data, length: total }],
nullCount,
};
}
default: {
const never: never = col.type;
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } {
switch (type) {
case "int64": {
b.startObject(2);
b.addInt32(0, 64, 0); // bitWidth
b.addInt8(1, 1, 0); // is_signed
return { typeType: TYPE_INT, offset: b.endObject() };
}
case "float64": {
b.startObject(1);
b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE
return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() };
}
case "utf8": {
b.startObject(0);
return { typeType: TYPE_UTF8, offset: b.endObject() };
}
case "bool": {
b.startObject(0);
return { typeType: TYPE_BOOL, offset: b.endObject() };
}
case "timestamp_ms": {
// Build the timezone string BEFORE startObject — FlatBuffers forbids
// creating a nested object while a table is open.
const tz = b.createString("UTC");
b.startObject(2);
b.addInt16(0, TIME_UNIT_MILLISECOND, 0);
b.addOffset(1, tz, 0);
return { typeType: TYPE_TIMESTAMP, offset: b.endObject() };
}
default: {
const never: never = type;
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
function buildSchemaMessage(columns: readonly ArrowColumn[]): Uint8Array {
const b = new FlatBufferBuilder(2048);
const fieldOffsets: Offset[] = columns.map((col) => {
const nameOffset = b.createString(col.name);
const { typeType, offset: typeOffset } = buildTypeTable(b, col.type);
const children = b.createOffsetVector([]);
b.startObject(7);
b.addOffset(0, nameOffset, 0);
b.addInt8(1, 1, 0); // nullable = true
b.addInt8(2, typeType, 0);
b.addOffset(3, typeOffset, 0);
b.addOffset(5, children, 0);
return b.endObject();
});
const fieldsVector = b.createOffsetVector(fieldOffsets);
b.startObject(4);
b.addInt16(0, ENDIANNESS_LITTLE, 0);
b.addOffset(1, fieldsVector, 0);
const schema = b.endObject();
b.startObject(5);
b.addInt16(0, METADATA_VERSION_V5, 0);
b.addInt8(1, MESSAGE_HEADER_SCHEMA, 0);
b.addOffset(2, schema, 0);
b.addInt64(3, BIG_ZERO, BIG_ZERO); // bodyLength
const message = b.endObject();
return b.finish(message);
}
function buildRecordBatchMessage(
rowCount: number,
nodes: { length: number; nullCount: number }[],
buffers: { offset: number; length: number }[],
bodyLength: number,
): Uint8Array {
const b = new FlatBufferBuilder(2048);
// FieldNode[] — inline structs, written in reverse.
b.startVector(16, nodes.length, 8);
for (let i = nodes.length - 1; i >= 0; i--) {
b.writeInt64(BigInt(nodes[i].nullCount));
b.writeInt64(BigInt(nodes[i].length));
}
const nodesVector = b.endVector(nodes.length);
// Buffer[] — inline structs, written in reverse.
b.startVector(16, buffers.length, 8);
for (let i = buffers.length - 1; i >= 0; i--) {
b.writeInt64(BigInt(buffers[i].length));
b.writeInt64(BigInt(buffers[i].offset));
}
const buffersVector = b.endVector(buffers.length);
b.startObject(5);
b.addInt64(0, BigInt(rowCount), BIG_ZERO);
b.addOffset(1, nodesVector, 0);
b.addOffset(2, buffersVector, 0);
const recordBatch = b.endObject();
b.startObject(5);
b.addInt16(0, METADATA_VERSION_V5, 0);
b.addInt8(1, MESSAGE_HEADER_RECORD_BATCH, 0);
b.addOffset(2, recordBatch, 0);
b.addInt64(3, BigInt(bodyLength), BIG_ZERO);
const message = b.endObject();
return b.finish(message);
}
function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array {
// continuation(4) + length(4) + metadata, padded so the body is 8-aligned.
const prefix = 8;
const padded = alignTo8(prefix + metadata.length) - prefix;
const out = new Uint8Array(prefix + padded + body.length);
const dv = new DataView(out.buffer);
dv.setUint32(0, 0xffffffff, true);
dv.setInt32(4, padded, true);
out.set(metadata, 8);
out.set(body, prefix + padded);
return out;
}
/**
* Encode columns as a complete Arrow IPC *stream* (schema message, one record
* batch, end-of-stream marker).
*/
export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array {
if (columns.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column");
}
const rowCount = columns[0].values.length;
for (const c of columns) {
if (c.values.length !== rowCount) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`column "${c.name}" has ${c.values.length} values, expected ${rowCount}`,
);
}
}
const schemaMsg = buildSchemaMessage(columns);
const nodes: { length: number; nullCount: number }[] = [];
const bufferMeta: { offset: number; length: number }[] = [];
const bodyParts: Uint8Array[] = [];
let bodyOffset = 0;
for (const col of columns) {
const { buffers, nullCount } = encodeColumn(col);
nodes.push({ length: rowCount, nullCount });
for (const buf of buffers) {
bufferMeta.push({ offset: bodyOffset, length: buf.length });
bodyParts.push(buf.bytes);
bodyOffset += buf.bytes.length; // already 8-aligned by encodeColumn
}
}
const body = new Uint8Array(bodyOffset);
let pos = 0;
for (const part of bodyParts) {
body.set(part, pos);
pos += part.length;
}
const batchMsg = buildRecordBatchMessage(rowCount, nodes, bufferMeta, body.length);
const schemaFrame = framedMessage(schemaMsg, new Uint8Array(0));
const batchFrame = framedMessage(batchMsg, body);
const eos = new Uint8Array(8);
new DataView(eos.buffer).setUint32(0, 0xffffffff, true); // length 0 = EOS
const out = new Uint8Array(schemaFrame.length + batchFrame.length + eos.length);
out.set(schemaFrame, 0);
out.set(batchFrame, schemaFrame.length);
out.set(eos, schemaFrame.length + batchFrame.length);
return out;
}
// ── decoder (structural: schema names/types + row counts) ──────────────────
export interface ArrowStreamInfo {
fields: { name: string; typeType: number }[];
/** Total rows across every record batch in the stream. */
rowCount: number;
batchCount: number;
}
/** Minimal FlatBuffers table reader — enough for Message/Schema/RecordBatch. */
class FbTable {
constructor(
readonly dv: DataView,
readonly pos: number,
) {}
fieldPos(slot: number): number {
const vtable = this.pos - this.dv.getInt32(this.pos, true);
const vtLen = this.dv.getInt16(vtable, true);
const idx = 4 + slot * 2;
if (idx >= vtLen) return 0;
const rel = this.dv.getInt16(vtable + idx, true);
return rel === 0 ? 0 : this.pos + rel;
}
readInt8(slot: number, def = 0): number {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getInt8(p);
}
readInt16(slot: number, def = 0): number {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getInt16(p, true);
}
readInt64(slot: number, def: bigint = BIG_ZERO): bigint {
const p = this.fieldPos(slot);
return p === 0 ? def : this.dv.getBigInt64(p, true);
}
/** Follow a uoffset to a nested table. */
readTable(slot: number): FbTable | undefined {
const p = this.fieldPos(slot);
if (p === 0) return undefined;
return new FbTable(this.dv, p + this.dv.getInt32(p, true));
}
vectorLength(slot: number): number {
const p = this.fieldPos(slot);
if (p === 0) return 0;
const vec = p + this.dv.getInt32(p, true);
return this.dv.getInt32(vec, true);
}
vectorElementTable(slot: number, index: number): FbTable {
const p = this.fieldPos(slot);
const vec = p + this.dv.getInt32(p, true);
const elem = vec + 4 + index * 4;
return new FbTable(this.dv, elem + this.dv.getInt32(elem, true));
}
readString(slot: number): string | undefined {
const p = this.fieldPos(slot);
if (p === 0) return undefined;
const str = p + this.dv.getInt32(p, true);
const len = this.dv.getInt32(str, true);
const bytes = new Uint8Array(this.dv.buffer, this.dv.byteOffset + str + 4, len);
return new TextDecoder().decode(bytes);
}
}
/**
* Parse an Arrow IPC stream's structure. Used to enforce BI-P012's result-row
* ceiling on Arrow payloads (including ClickHouse pass-through bytes, where
* the count is otherwise invisible to 0pt) and by the roundtrip tests.
*/
export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo {
const dv = new DataView(bytes.buffer, bytes.byteOffset, bytes.byteLength);
let pos = 0;
let fields: { name: string; typeType: number }[] = [];
let rowCount = 0;
let batchCount = 0;
while (pos + 8 <= bytes.byteLength) {
const continuation = dv.getUint32(pos, true);
if (continuation !== 0xffffffff) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)");
}
const metaLen = dv.getInt32(pos + 4, true);
if (metaLen === 0) break; // end of stream
const metaStart = pos + 8;
const root = metaStart + dv.getInt32(metaStart, true);
const message = new FbTable(dv, root);
const headerType = message.readInt8(1, 0);
const bodyLength = Number(message.readInt64(3, BIG_ZERO));
if (headerType === MESSAGE_HEADER_SCHEMA) {
const schema = message.readTable(2);
if (schema) {
const n = schema.vectorLength(1);
fields = [];
for (let i = 0; i < n; i++) {
const f = schema.vectorElementTable(1, i);
fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) });
}
}
} else if (headerType === MESSAGE_HEADER_RECORD_BATCH) {
const rb = message.readTable(2);
if (rb) {
rowCount += Number(rb.readInt64(0, BIG_ZERO));
batchCount += 1;
}
}
pos = metaStart + metaLen + bodyLength;
}
return { fields, rowCount, batchCount };
}
/**
* BI-P012 on the Arrow path: refuse a payload that exceeds the caller's
* result-row ceiling. Without this, asking for Arrow would be a way to walk
* around `max_result_rows`.
*/
export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: number): ArrowStreamInfo {
const info = decodeArrowStream(bytes);
if (info.rowCount > maxResultRows) {
throw new SiaxError(
"QUOTA_EXCEEDED",
"0pt.analytics",
`Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`,
);
}
return info;
}
+341
View File
@@ -0,0 +1,341 @@
/**
* src/analytics/bench/benchmark-harness.ts
* =========================================
* BI-P104 — BI PERFORMANCE BENCHMARK HARNESS. DoD: "results committed as
* machine-readable evidence."
*
* The harness exists because of a specific way benchmarks lie. Yesterday's
* lane measured the same budget-enforcement overshoot at ~150x on one run and
* ~130x on the next, on the same machine, and correctly reported the SHAPE
* rather than a number. That is the doctrine encoded here:
*
* 1. A SINGLE FIGURE IS NOT A SYSTEM PROPERTY. `summarizeSamples` returns a
* distribution — n, min, p50, p95, p99, max, mean, stdev, CV, IQR — and
* there is no function anywhere in this module that returns one number
* for a measurement. `describeComparison` renders a RANGE and the sample
* size into every sentence it produces.
*
* 2. OVERLAPPING DISTRIBUTIONS ARE INDISTINGUISHABLE, whatever the medians
* say. If the candidate's p95 is not below the baseline's minimum, the
* verdict is `indistinguishable` even when p50 differs by 2x — because a
* 2x median gap inside a 10x spread is noise with a story attached.
*
* 3. TOO FEW SAMPLES IS A REFUSAL. Under MIN_SAMPLES the summary carries
* `stability: 'insufficient_samples'` and comparisons refuse to judge.
* A three-run benchmark is an anecdote.
*
* 4. AN ENGINE THAT WAS NOT RUN IS RECORDED AS NOT RUN. `EngineAvailability`
* carries a reason. The evidence file lists every engine BI-P104 names,
* including the ones nothing measured — an omitted engine reads as "we
* chose not to report it".
*/
/** The six dataset profiles BI-P104 names, with what makes each one distinct. */
export const BENCH_DATASETS = [
{
id: 'small_saas',
label: 'small SaaS',
approx_rows: 100_000,
shape: 'narrow fact table, one tenant, high selectivity',
},
{
id: 'medium_business',
label: 'medium business',
approx_rows: 2_000_000,
shape: 'multi-tenant fact table with a date and a few dimensions',
},
{
id: 'vehiq_market',
label: 'VehIQ market',
approx_rows: 1_000_000,
shape: 'vehicle listings: make/model/year/price/mileage, skewed cardinalities',
},
{
id: 'large_event',
label: 'large event',
approx_rows: 5_000_000,
shape: 'append-only event stream, timestamp-ordered, sparse dimensions',
},
{
id: 'wide_table',
label: 'wide table',
approx_rows: 200_000,
shape: 'many columns, few read per query — the columnar-store best case',
},
{
id: 'geospatial',
label: 'geospatial',
approx_rows: 500_000,
shape: 'lat/lon points with bounding-box and distance predicates',
},
] as const;
export type BenchDatasetId = (typeof BENCH_DATASETS)[number]['id'];
/** Every engine BI-P104 lists. All of them appear in the evidence, run or not. */
export const BENCH_ENGINES = [
'clickhouse',
'duckdb',
'trino',
'semantic_query',
'superset',
'native_charts',
'perspective',
] as const;
export type BenchEngine = (typeof BENCH_ENGINES)[number];
export interface EngineAvailability {
engine: BenchEngine;
available: boolean;
/** Required when unavailable. "Unknown" is not a reason. */
reason?: string;
version?: string;
}
export const STABILITY_VERDICTS = ['stable', 'noisy', 'unstable', 'insufficient_samples'] as const;
export type StabilityVerdict = (typeof STABILITY_VERDICTS)[number];
/** Below this a run is an anecdote, not a measurement. */
export const MIN_SAMPLES = 5;
/** Coefficient of variation thresholds. Wide on purpose: real hosts are noisy. */
const CV_STABLE = 0.15;
const CV_NOISY = 0.5;
export interface Distribution {
n: number;
min: number;
p50: number;
p95: number;
p99: number;
max: number;
mean: number;
stdev: number;
/** stdev / mean. The number that decides whether a comparison is meaningful. */
cv: number;
iqr: number;
/** max / min. Reported because it is what a reader feels. */
spread_ratio: number;
stability: StabilityVerdict;
}
/** Nearest-rank; no interpolation, so every reported value really happened. */
function pct(sorted: number[], q: number): number {
const rank = Math.ceil(q * sorted.length);
return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
}
export function summarizeSamples(samples: readonly number[]): Distribution {
if (samples.length === 0) {
return {
n: 0,
min: 0,
p50: 0,
p95: 0,
p99: 0,
max: 0,
mean: 0,
stdev: 0,
cv: 0,
iqr: 0,
spread_ratio: 0,
stability: 'insufficient_samples',
};
}
const s = [...samples].sort((a, b) => a - b);
const mean = s.reduce((a, b) => a + b, 0) / s.length;
const variance = s.reduce((acc, v) => acc + (v - mean) ** 2, 0) / s.length;
const stdev = Math.sqrt(variance);
const cv = mean === 0 ? 0 : stdev / mean;
const stability: StabilityVerdict =
s.length < MIN_SAMPLES
? 'insufficient_samples'
: cv <= CV_STABLE
? 'stable'
: cv <= CV_NOISY
? 'noisy'
: 'unstable';
return {
n: s.length,
min: s[0],
p50: pct(s, 0.5),
p95: pct(s, 0.95),
p99: pct(s, 0.99),
max: s[s.length - 1],
mean,
stdev,
cv,
iqr: pct(s, 0.75) - pct(s, 0.25),
spread_ratio: s[0] === 0 ? Number.POSITIVE_INFINITY : s[s.length - 1] / s[0],
stability,
};
}
export const COMPARISON_VERDICTS = ['faster', 'slower', 'indistinguishable', 'not_comparable'] as const;
export type ComparisonVerdict = (typeof COMPARISON_VERDICTS)[number];
export interface Comparison {
verdict: ComparisonVerdict;
/** Ratio of medians. Present for information; never the claim on its own. */
p50_ratio: number;
/** Worst case for the candidate: baseline.min / candidate.max. */
conservative_ratio: number;
/** Best case for the candidate: baseline.max / candidate.min. */
optimistic_ratio: number;
/** True when the observed ranges overlap at all. */
ranges_overlap: boolean;
baseline: Distribution;
candidate: Distribution;
}
/**
* Compare two measured distributions.
*
* `faster` requires clean separation — the candidate's p95 below the
* baseline's minimum. Anything less is `indistinguishable`, however flattering
* the medians look, because that is exactly the claim a noisy host will
* manufacture for you.
*/
export function compareDistributions(baseline: Distribution, candidate: Distribution): Comparison {
const ratio = (a: number, b: number) => (b === 0 ? Number.POSITIVE_INFINITY : a / b);
const base: Omit<Comparison, 'verdict'> = {
p50_ratio: ratio(baseline.p50, candidate.p50),
conservative_ratio: ratio(baseline.min, candidate.max),
optimistic_ratio: ratio(baseline.max, candidate.min),
ranges_overlap: !(candidate.max < baseline.min || baseline.max < candidate.min),
baseline,
candidate,
};
if (baseline.stability === 'insufficient_samples' || candidate.stability === 'insufficient_samples') {
return { ...base, verdict: 'not_comparable' };
}
if (candidate.p95 < baseline.min) return { ...base, verdict: 'faster' };
if (baseline.p95 < candidate.min) return { ...base, verdict: 'slower' };
return { ...base, verdict: 'indistinguishable' };
}
/**
* The sentence a human is allowed to quote. It ALWAYS carries the range, the
* sample size and the stability verdict, so it cannot be trimmed down to "3.4x
* faster" without visibly removing the caveats.
*/
export function describeComparison(c: Comparison, label = 'candidate'): string {
const r = (v: number) => (Number.isFinite(v) ? v.toFixed(1) : '∞');
if (c.verdict === 'not_comparable') {
return (
`${label}: NOT COMPARABLE — ${c.baseline.n} baseline and ${c.candidate.n} candidate samples, ` +
`fewer than the ${MIN_SAMPLES} required. A run this short is an anecdote.`
);
}
if (c.verdict === 'indistinguishable') {
return (
`${label}: INDISTINGUISHABLE at this sample size. Medians differ by ${r(c.p50_ratio)}x but the ` +
`observed ranges overlap (baseline ${c.baseline.min.toFixed(1)}-${c.baseline.max.toFixed(1)}ms, ` +
`candidate ${c.candidate.min.toFixed(1)}-${c.candidate.max.toFixed(1)}ms; n=${c.baseline.n}/${c.candidate.n}, ` +
`CV ${c.baseline.cv.toFixed(2)}/${c.candidate.cv.toFixed(2)}).`
);
}
return (
`${label}: ${c.verdict.toUpperCase()} — ${r(c.conservative_ratio)}x to ${r(c.optimistic_ratio)}x ` +
`(median ${r(c.p50_ratio)}x) over n=${c.baseline.n}/${c.candidate.n}, ` +
`stability ${c.baseline.stability}/${c.candidate.stability}. Warm cache, one host, this query shape.`
);
}
export interface BenchmarkRun {
engine: BenchEngine;
dataset: BenchDatasetId;
query_label: string;
/** Discarded from the statistics; recorded so the discard is visible. */
warmup_ms: number[];
samples_ms: number[];
distribution: Distribution;
rows_returned?: number;
}
/**
* Run one measurement. Warmups are executed and RECORDED but excluded from the
* distribution: a first run that pays for cold marks and page cache is a
* different measurement, and silently folding it in is how a p95 becomes a
* story about disk.
*/
export async function measure(args: {
engine: BenchEngine;
dataset: BenchDatasetId;
query_label: string;
run: () => Promise<{ rows?: number }>;
warmups?: number;
repetitions?: number;
now?: () => number;
}): Promise<BenchmarkRun> {
const now = args.now ?? (() => performance.now());
const warmups = args.warmups ?? 2;
const repetitions = args.repetitions ?? 9;
const warmup_ms: number[] = [];
const samples_ms: number[] = [];
let rows: number | undefined;
for (let i = 0; i < warmups; i += 1) {
const t0 = now();
await args.run();
warmup_ms.push(now() - t0);
}
for (let i = 0; i < repetitions; i += 1) {
const t0 = now();
const out = await args.run();
samples_ms.push(now() - t0);
rows = out.rows ?? rows;
}
return {
engine: args.engine,
dataset: args.dataset,
query_label: args.query_label,
warmup_ms,
samples_ms,
distribution: summarizeSamples(samples_ms),
...(rows !== undefined ? { rows_returned: rows } : {}),
};
}
export interface BenchmarkEvidence {
recorded_at: string;
host_note: string;
datasets: typeof BENCH_DATASETS;
engines: EngineAvailability[];
runs: BenchmarkRun[];
comparisons: Array<{ dataset: BenchDatasetId; query_label: string; description: string } & Comparison>;
/** Engines BI-P104 names that produced no measurement, each with a reason. */
not_exercised: EngineAvailability[];
}
/**
* Assemble the evidence document. Every engine BI-P104 names appears, and the
* ones with no runs are listed under `not_exercised` with their reason — an
* engine quietly missing from a benchmark table reads as a choice.
*/
export function buildEvidence(args: {
host_note: string;
engines: EngineAvailability[];
runs: BenchmarkRun[];
comparisons?: BenchmarkEvidence['comparisons'];
now?: Date;
}): BenchmarkEvidence {
const measured = new Set(args.runs.map((r) => r.engine));
const declared = new Map(args.engines.map((e) => [e.engine, e]));
return {
recorded_at: (args.now ?? new Date()).toISOString(),
host_note: args.host_note,
datasets: BENCH_DATASETS,
engines: BENCH_ENGINES.map(
(engine) => declared.get(engine) ?? { engine, available: false, reason: 'not declared by this run' },
),
runs: args.runs,
comparisons: args.comparisons ?? [],
not_exercised: BENCH_ENGINES.filter((e) => !measured.has(e)).map((engine) => {
const d = declared.get(engine);
if (!d) return { engine, available: false, reason: 'not present in this environment' };
// An engine declared AVAILABLE that produced no measurement is the
// dangerous case: it reads as "we could have, and chose not to say".
// It gets a reason too, generated here rather than left blank.
return d.reason ? d : { ...d, reason: 'declared available but produced no measurement in this run' };
}),
};
}
+329
View File
@@ -0,0 +1,329 @@
/**
* src/analytics/cache/query-cache.ts
* ===================================
* BI-P020 — QUERY CACHE.
*
* "Cache key includes: tenant, semantic model version, metric definitions,
* filters, query, data snapshot/version. Never share tenant result cache."
*
* All six are REQUIRED, and the key builder refuses to produce a key when one
* is missing rather than defaulting it. A defaulted component is how a cache
* serves a result computed under a metric definition that no longer exists —
* the answer is not stale, it is WRONG, and it looks current.
*
* "Never share tenant result cache" is enforced structurally, not by
* discipline:
*
* * the key is PREFIXED with the tenant id, and the prefix is derived from
* the verified identity inside this module — a caller cannot pass one;
* * every read re-derives the prefix from the identity and REFUSES a stored
* entry whose recorded tenant does not match, so a prefix collision or a
* shared backing store still cannot serve a foreign result;
* * the stored entry carries its own tenant id, so the check is against the
* DATA, not against the key that found it.
*
* The second and third points exist because the first is a naming convention,
* and naming conventions are what ST0RE was relying on when it turned out to
* have four cross-tenant paths rather than the one reported.
*
* Provider-neutral: `AnalyticsResultCache` is the port. `InMemoryResultCache`
* is the reference implementation and the test double; `ValkeyResultCache`
* adapts any Valkey/Redis client that can GET/SET/DEL with a TTL, taking it as
* an injected interface so this module adds no dependency.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
/**
* Everything that must be true for a cached result to still be the right
* answer. Missing any one of these makes the cache unsound, so the type has no
* optional fields.
*/
export interface CacheKeyComponents {
/** From the verified identity. Not a caller-supplied value. */
tenant_id: string;
project_id?: string;
dataset_id: string;
/** BI-P030's semantic model version. Bumping it invalidates everything under it. */
semantic_model_version: number;
/**
* metric_id -> version, for EVERY metric in the query. A metric whose
* definition changed must not serve a result computed under the old one.
*/
metric_versions: Record<string, number>;
/** The compiled statement hash (BI-P018). Covers filters, grain and limit. */
compiled_query_hash: string;
/**
* The data snapshot the result was computed from. Whatever identifies "the
* data as of now" for this dataset: a DatasetVersion number, an Iceberg
* snapshot id, a max ingested watermark. Advancing it invalidates.
*/
data_snapshot: string;
/** Result encoding, since a JSON and an Arrow result are different bytes. */
format: string;
}
export interface CachedResult<T = unknown> {
/** Recorded so the read check is against the DATA, not against the key. */
tenant_id: string;
key: string;
value: T;
stored_at: string;
expires_at: string;
row_count: number;
/** Kept for targeted invalidation. */
dataset_id: string;
metric_ids: string[];
semantic_model_version: number;
data_snapshot: string;
}
export interface AnalyticsResultCache {
readonly provider: string;
get<T>(components: CacheKeyComponents): Promise<CachedResult<T> | undefined>;
set<T>(
components: CacheKeyComponents,
value: T,
opts: { ttlSeconds: number; rowCount: number },
): Promise<void>;
/** Drop every entry for a dataset — the blunt, always-correct invalidation. */
invalidateDataset(tenantId: string, datasetId: string): Promise<number>;
/** Drop every entry that used a given metric. */
invalidateMetric(tenantId: string, metricId: string): Promise<number>;
/** Drop everything for a tenant (offboarding, incident response). */
invalidateTenant(tenantId: string): Promise<number>;
}
function required(value: unknown, name: string): void {
if (
value === undefined ||
value === null ||
value === '' ||
(typeof value === 'number' && !Number.isFinite(value))
) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.cache',
`cache key component "${name}" is required — a defaulted component serves results computed under a definition that no longer exists`,
);
}
}
/**
* Build the key. Deterministic, tenant-prefixed, and total over the six
* components BI-P020 lists.
*
* The tenant prefix is in PLAINTEXT at the front rather than folded into the
* hash, so `invalidateTenant` is a prefix scan and an operator reading the
* store can see which tenant an entry belongs to without a lookup table.
*/
export function buildCacheKey(c: CacheKeyComponents): string {
required(c.tenant_id, 'tenant_id');
required(c.dataset_id, 'dataset_id');
required(c.semantic_model_version, 'semantic_model_version');
required(c.compiled_query_hash, 'compiled_query_hash');
required(c.data_snapshot, 'data_snapshot');
required(c.format, 'format');
if (!c.metric_versions || Object.keys(c.metric_versions).length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.cache',
'cache key component "metric_versions" is required and must name every metric in the query',
);
}
for (const [id, version] of Object.entries(c.metric_versions)) {
required(version, `metric_versions[${id}]`);
}
if (!/^[A-Za-z0-9_-]{1,64}$/.test(c.tenant_id)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'tenant id is not key-safe');
}
const material = JSON.stringify([
c.tenant_id,
c.project_id ?? '',
c.dataset_id,
c.semantic_model_version,
Object.keys(c.metric_versions)
.sort()
.map((k) => [k, c.metric_versions[k]]),
c.compiled_query_hash,
c.data_snapshot,
c.format,
]);
const digest = crypto.createHash('sha256').update(material).digest('hex');
return `optq:${c.tenant_id}:${digest}`;
}
/** The prefix `invalidateTenant` scans. Derived, never supplied. */
export function tenantKeyPrefix(tenantId: string): string {
return `optq:${tenantId}:`;
}
abstract class BaseResultCache implements AnalyticsResultCache {
abstract readonly provider: string;
protected abstract read(key: string): Promise<CachedResult | undefined>;
protected abstract write(entry: CachedResult, ttlSeconds: number): Promise<void>;
protected abstract entries(tenantId: string): Promise<CachedResult[]>;
protected abstract remove(keys: string[]): Promise<void>;
protected readonly now: () => number;
constructor(now: () => number = Date.now) {
this.now = now;
}
async get<T>(components: CacheKeyComponents): Promise<CachedResult<T> | undefined> {
const key = buildCacheKey(components);
const entry = await this.read(key);
if (!entry) return undefined;
// The check that survives a prefix collision or a shared backing store:
// the ENTRY says which tenant it belongs to, and it must be this one.
if (entry.tenant_id !== components.tenant_id) {
// Not a miss — a cross-tenant hit is a security event, and returning
// `undefined` would hide it. Drop the poisoned entry and refuse loudly.
await this.remove([key]);
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.cache',
'cache entry belongs to a different tenant; entry evicted',
);
}
if (Date.parse(entry.expires_at) <= this.now()) {
await this.remove([key]);
return undefined;
}
return entry as CachedResult<T>;
}
async set<T>(
components: CacheKeyComponents,
value: T,
opts: { ttlSeconds: number; rowCount: number },
): Promise<void> {
if (!Number.isFinite(opts.ttlSeconds) || opts.ttlSeconds <= 0) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.cache', 'ttlSeconds must be positive');
}
const key = buildCacheKey(components);
const nowMs = this.now();
await this.write(
{
tenant_id: components.tenant_id,
key,
value,
stored_at: new Date(nowMs).toISOString(),
expires_at: new Date(nowMs + opts.ttlSeconds * 1000).toISOString(),
row_count: opts.rowCount,
dataset_id: components.dataset_id,
metric_ids: Object.keys(components.metric_versions),
semantic_model_version: components.semantic_model_version,
data_snapshot: components.data_snapshot,
},
opts.ttlSeconds,
);
}
async invalidateDataset(tenantId: string, datasetId: string): Promise<number> {
const victims = (await this.entries(tenantId)).filter((e) => e.dataset_id === datasetId);
await this.remove(victims.map((v) => v.key));
return victims.length;
}
async invalidateMetric(tenantId: string, metricId: string): Promise<number> {
const victims = (await this.entries(tenantId)).filter((e) => e.metric_ids.includes(metricId));
await this.remove(victims.map((v) => v.key));
return victims.length;
}
async invalidateTenant(tenantId: string): Promise<number> {
const victims = await this.entries(tenantId);
await this.remove(victims.map((v) => v.key));
return victims.length;
}
}
export class InMemoryResultCache extends BaseResultCache {
readonly provider = 'memory';
private readonly store = new Map<string, CachedResult>();
protected async read(key: string): Promise<CachedResult | undefined> {
return this.store.get(key);
}
protected async write(entry: CachedResult): Promise<void> {
this.store.set(entry.key, entry);
}
protected async entries(tenantId: string): Promise<CachedResult[]> {
const prefix = tenantKeyPrefix(tenantId);
return [...this.store.values()].filter((e) => e.key.startsWith(prefix));
}
protected async remove(keys: string[]): Promise<void> {
for (const k of keys) this.store.delete(k);
}
/** Test/observability affordance. Never used to bypass the tenant check. */
get size(): number {
return this.store.size;
}
}
/**
* The minimum a Valkey/Redis client must offer. Taken as an interface so this
* module adds no dependency and no CVE surface — the same reason BI-P005's
* Arrow writer is hand-written.
*/
export interface ValkeyLikeClient {
get(key: string): Promise<string | null>;
set(key: string, value: string, opts: { EX: number }): Promise<unknown>;
del(keys: string[]): Promise<unknown>;
/** Cursor-based scan. A blocking KEYS on a shared Valkey is an outage. */
scan(cursor: string, opts: { MATCH: string; COUNT: number }): Promise<{ cursor: string; keys: string[] }>;
}
export class ValkeyResultCache extends BaseResultCache {
readonly provider = 'valkey';
constructor(
private readonly client: ValkeyLikeClient,
now: () => number = Date.now,
) {
super(now);
}
protected async read(key: string): Promise<CachedResult | undefined> {
const raw = await this.client.get(key);
if (raw === null) return undefined;
try {
return JSON.parse(raw) as CachedResult;
} catch {
// A corrupt entry is a miss, not a crash — but it is also removed, so a
// permanently unparseable key cannot become a permanent cache miss that
// nobody notices.
await this.client.del([key]);
return undefined;
}
}
protected async write(entry: CachedResult, ttlSeconds: number): Promise<void> {
// The TTL is set on the STORE as well as recorded in the entry. Relying on
// the recorded timestamp alone leaves expired results occupying memory
// forever on a busy tenant.
await this.client.set(entry.key, JSON.stringify(entry), { EX: Math.ceil(ttlSeconds) });
}
protected async entries(tenantId: string): Promise<CachedResult[]> {
const prefix = tenantKeyPrefix(tenantId);
const out: CachedResult[] = [];
let cursor = '0';
do {
const page = await this.client.scan(cursor, { MATCH: `${prefix}*`, COUNT: 500 });
cursor = page.cursor;
for (const key of page.keys) {
const entry = await this.read(key);
// Defence in depth: SCAN MATCH is a glob, and a glob is a naming
// convention. Confirm against the entry's own tenant id.
if (entry && entry.tenant_id === tenantId) out.push(entry);
}
} while (cursor !== '0');
return out;
}
protected async remove(keys: string[]): Promise<void> {
if (keys.length) await this.client.del(keys);
}
}
+233
View File
@@ -0,0 +1,233 @@
/**
* src/analytics/clickhouse-analytics.ts
* ======================================
* ClickHouse-backed Analytics capability (Master-DoD §10 Analytics:
* ClickHouse, tenant isolation, dataset/event ingest, query limits, metric
* query capability, retention).
*
* LEGAL BOUNDARY (absolute): this module is a GENERIC analytics adapter for
* 0pt capability events/dataset metrics. It has NO connection to, and MUST
* NEVER be pointed at, the b0ts ClickHouse pipelines (Ratsit/person-PII —
* separate legal case) or any quarantined market='sverige'/'norge' market
* data (written DPIA prohibition). Nothing here reads services/scrapers/*.
*
* Tenant isolation model:
* - every table carries a `tenant_id` column;
* - EVERY query built by this service injects `tenant_id = {tenant}` as a
* parameterized predicate — callers cannot omit it;
* - callers NEVER pass raw SQL: the query surface is a closed metric-query
* DSL (metric + optional group-by + time range) compiled here, which is
* what makes the injection guarantee enforceable.
*
* Query limits (§10): every SELECT runs with readonly=1,
* max_execution_time, max_result_rows — enforced via ClickHouse settings on
* the HTTP interface.
*
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
*/
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
export interface ClickHouseAnalyticsOptions {
baseUrl: string;
/** "user:password" — from env/secret ref. */
basicAuth?: string;
database?: string;
timeoutMs?: number;
/** SELECT limits (§10 query limits). */
maxExecutionTimeSec?: number;
maxResultRows?: number;
}
export interface AnalyticsEvent {
tenant_id: string;
dataset_id: string;
metric: string;
value: number;
/** ISO timestamp; defaults to now. */
event_time?: string;
dimensions?: Record<string, string>;
}
export interface MetricQuery {
tenant_id: string;
dataset_id?: string;
metric: string;
aggregate: "sum" | "avg" | "min" | "max" | "count";
/** Group by one dimension key (validated identifier). */
group_by?: string;
from_time?: string;
to_time?: string;
/** Row limit; clamped to maxResultRows. */
limit?: number;
}
export interface MetricQueryRow {
group?: string;
value: number;
}
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
export class ClickHouseAnalytics {
private readonly baseUrl: string;
private readonly authHeader?: string;
private readonly database: string;
private readonly timeoutMs: number;
private readonly maxExecutionTimeSec: number;
private readonly maxResultRows: number;
constructor(opts: ClickHouseAnalyticsOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
this.database = opts.database ?? "opt_analytics";
this.timeoutMs = opts.timeoutMs ?? 30_000;
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
this.maxResultRows = opts.maxResultRows ?? 10_000;
}
/** Create the events table (idempotent) with retention TTL (§10 retention). */
async ensureSchema(retentionDays = 90): Promise<void> {
await this.exec(`CREATE DATABASE IF NOT EXISTS ${this.database}`);
await this.exec(
`CREATE TABLE IF NOT EXISTS ${this.database}.opt_metric_events (
tenant_id LowCardinality(String),
dataset_id String,
metric LowCardinality(String),
value Float64,
event_time DateTime64(3, 'UTC'),
dimensions Map(String, String)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, metric, dataset_id, event_time)
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
);
}
/** Ingest events (JSONEachRow). tenant_id is required per event. */
async ingest(events: AnalyticsEvent[]): Promise<{ ingested: number }> {
if (events.length === 0) return { ingested: 0 };
for (const e of events) {
if (!e.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
}
if (!Number.isFinite(e.value)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
}
}
const rows = events
.map((e) =>
JSON.stringify({
tenant_id: e.tenant_id,
dataset_id: e.dataset_id,
metric: e.metric,
value: e.value,
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
dimensions: e.dimensions ?? {},
})
)
.join("\n");
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
return { ingested: events.length };
}
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
if (!q.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
}
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
}
const agg =
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
// Parameterized query — values travel as HTTP params, never spliced.
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
if (q.dataset_id) {
conditions.push("dataset_id = {dataset:String}");
params.param_dataset = q.dataset_id;
}
if (q.from_time) {
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
params.param_fromTime = q.from_time;
}
if (q.to_time) {
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
params.param_toTime = q.to_time;
}
const sql =
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
`FROM ${this.database}.opt_metric_events ` +
`WHERE ${conditions.join(" AND ")} ` +
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
`LIMIT ${limit} FORMAT JSON`;
const json = (await this.exec(sql, undefined, {
readonly: "1",
max_execution_time: String(this.maxExecutionTimeSec),
max_result_rows: String(this.maxResultRows),
...params,
})) as { data?: { grp?: string; value: number | string }[] };
return (json?.data ?? []).map((r) => ({
...(r.grp !== undefined ? { group: r.grp } : {}),
value: Number(r.value),
}));
}
async health(): Promise<boolean> {
try {
await this.exec("SELECT 1 FORMAT JSON");
return true;
} catch {
return false;
}
}
// -----------------------------------------------------------------------
private async exec(
sql: string,
body?: string,
settings?: Record<string, string>
): Promise<unknown> {
const url = new URL(this.baseUrl);
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
const headers: Record<string, string> = {};
if (this.authHeader) headers["Authorization"] = this.authHeader;
let res: Response;
try {
res = await fetch(url.toString(), {
method: "POST",
headers,
body: body ?? "",
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
}
const text = await res.text();
if (!res.ok) {
// ClickHouse error text can include query fragments — keep only the code.
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
"clickhouse",
`query failed (CH code ${code})`
);
}
if (!text) return undefined;
try {
return JSON.parse(text);
} catch {
return text;
}
}
}
@@ -0,0 +1,277 @@
/**
* src/analytics/contract/compile-clickhouse.ts
* =============================================
* BI-P016, compile target 1 of 2: the canonical `AnalyticsQueryRequest`
* becomes tenant-forced ClickHouse SQL.
*
* The BI-P011 guarantee is repeated here VERBATIM and by construction:
*
* * `FORCED_TENANT_PREDICATE` is condition[0] with no branch that can skip
* it, and the tenant value comes from the verified identity handed in by
* the engine — this function has no way to read one from a request,
* because `AnalyticsQueryRequest` has no tenant field.
* * every caller VALUE is a `param_*` binding; `sql` never contains one.
* * every caller NAME has already been resolved through the catalog
* (`resolveAnalyticsQuery`), and the resolved physical name is validated
* as an identifier a second time before it reaches the string.
*
* tests/analytics/query-request-tenant-forcing.test.ts asserts the predicate
* survives every request shape this compiler accepts, by ENUMERATING the
* shapes rather than spot-checking two of them.
*/
import { SiaxError } from '../../platform/errors';
import { assertSafeIdentifier, FORCED_TENANT_PREDICATE, type CompiledQuery } from '../engine/query-compiler';
import type { AnalyticsTable } from '../engine/schema';
import type { ResolvedQuery, TimeGrain } from './query-request';
import type { FilterOperator } from './query-request';
/** Grain -> ClickHouse truncation function. A closed map, never caller text. */
const GRAIN_FN: Record<TimeGrain, string> = {
hour: 'toStartOfHour',
day: 'toStartOfDay',
week: 'toStartOfWeek',
month: 'toStartOfMonth',
quarter: 'toStartOfQuarter',
year: 'toStartOfYear',
};
/**
* Aggregation -> conditional-aggregate template. `count_distinct` uses
* `uniqExactIf` so a multi-metric request stays one scan instead of N.
*
* The argument is the WHOLE per-metric predicate, not just the metric-key
* comparison, because a metric definition can carry its own filters (BI-P028
* `MetricSpec.filters`) and those belong INSIDE this metric's `If` — one
* request may carry two metrics that exclude different rows.
*/
const AGG_IF: Record<string, (predicate: string) => string> = {
sum: (p) => `sumIf(value, ${p})`,
avg: (p) => `avgIf(value, ${p})`,
min: (p) => `minIf(value, ${p})`,
max: (p) => `maxIf(value, ${p})`,
count: (p) => `countIf(${p})`,
count_distinct: (p) => `uniqExactIf(event_id, ${p})`,
};
export interface ClickHouseCompileOptions {
/** From the VERIFIED CL0UD identity. Never from a request body. */
tenant_id: string;
project_id?: string;
database: string;
table?: AnalyticsTable;
/** BI-P012 ceiling; `limit` is clamped to it, never raised by it. */
maxResultRows: number;
}
export interface CompiledAnalyticsQuery extends CompiledQuery {
/** Column alias -> caller-facing metric name, so the caller never sees `m0`. */
metricAliases: Record<string, string>;
/** Column alias -> caller-facing dimension name. */
dimensionAliases: Record<string, string>;
effectiveLimit: number;
}
export function compileRequestToClickHouse(
resolved: ResolvedQuery,
opts: ClickHouseCompileOptions,
): CompiledAnalyticsQuery {
if (!opts.tenant_id || !opts.tenant_id.trim()) {
// Fail closed. A blank tenant must never compile to "no predicate".
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query');
}
assertSafeIdentifier(opts.database, 'database');
const table: AnalyticsTable = opts.table ?? 'opt_analytics_event';
assertSafeIdentifier(table, 'table');
const params: Record<string, string> = { param_tenant: opts.tenant_id };
// ── the forced predicate. First condition, always, no branch. ─────────
const conditions: string[] = [FORCED_TENANT_PREDICATE];
if (opts.project_id) {
conditions.push('project_id = {project:String}');
params.param_project = opts.project_id;
}
conditions.push('dataset_id = {dataset:String}');
params.param_dataset = resolved.dataset_id;
// ── metrics: bound keys, compiler-generated aliases ───────────────────
const selectParts: string[] = [];
const groupParts: string[] = [];
const dimensionAliases: Record<string, string> = {};
const metricAliases: Record<string, string> = {};
const metricKeyParams: string[] = [];
resolved.dimensions.forEach((d, i) => {
assertSafeIdentifier(d.field_name, 'dimension');
const key = `d${i}`;
const alias = `dim_${i}`;
selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS ${alias}`);
groupParts.push(alias);
params[`param_${key}`] = d.field_name;
dimensionAliases[alias] = d.name;
});
// ── time grain: a GROUP BY bucket, from the closed GRAIN_FN map ───────
const grain = resolved.request.timeRange?.grain;
if (grain) {
const timeCol = resolved.timeDimension ?? 'event_time';
assertSafeIdentifier(timeCol, 'time dimension');
const alias = 'time_bucket';
selectParts.push(`${GRAIN_FN[grain]}(${timeCol}) AS ${alias}`);
groupParts.push(alias);
dimensionAliases[alias] = resolved.request.timeRange?.dimension ?? '__time';
}
resolved.metrics.forEach((m, i) => {
const key = `m${i}`;
const alias = `metric_${i}`;
const template = AGG_IF[m.aggregation];
if (!template) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported aggregation "${m.aggregation}"`);
}
// The metric's own definition filters (BI-P028) join the metric-key
// comparison inside THIS metric's `If`. Bound, never spliced, and scoped
// to this metric — putting them in the shared WHERE would apply one
// metric's meaning to every other metric in the same request.
const metricConditions = [`metric = {${key}:String}`];
(m.definition_filters ?? []).forEach((f, j) => {
assertSafeIdentifier(f.field_name, 'metric definition filter dimension');
const nameKey = `mf${i}_${j}`;
params[`param_${nameKey}`] = f.field_name;
const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`;
metricConditions.push(renderFilter(lhs, nameKey, f.operator, [...f.values], `${i}_${j}m`, params));
});
selectParts.push(`${template(metricConditions.join(' AND '))} AS ${alias}`);
params[`param_${key}`] = m.event_metric_key;
metricKeyParams.push(`{${key}:String}`);
metricAliases[alias] = m.name;
});
// Restrict the SCAN to the requested metric keys — a correctness AND a cost
// measure: without it every metric row in the dataset is read to be thrown
// away by the `If` conditions.
conditions.push(`metric IN (${metricKeyParams.join(', ')})`);
// ── time range: bound instants, half-open ─────────────────────────────
const tr = resolved.request.timeRange;
if (tr?.from || tr?.to) {
const timeCol = resolved.timeDimension ?? 'event_time';
assertSafeIdentifier(timeCol, 'time dimension');
if (tr.from) {
conditions.push(`${timeCol} >= parseDateTime64BestEffort({fromTime:String})`);
params.param_fromTime = tr.from;
}
if (tr.to) {
conditions.push(`${timeCol} < parseDateTime64BestEffort({toTime:String})`);
params.param_toTime = tr.to;
}
}
// ── filters: closed operator set, values always bound ─────────────────
resolved.filters.forEach((entry, i) => {
assertSafeIdentifier(entry.dimension.field_name, 'dimension');
const nameKey = `fn${i}`;
params[`param_${nameKey}`] = entry.dimension.field_name;
const lhs = `dim_values[indexOf(dim_names, {${nameKey}:String})]`;
conditions.push(
renderFilter(lhs, nameKey, entry.filter.operator, entry.filter.values ?? [], String(i), params),
);
});
// ── order by: resolved back onto compiler-generated aliases only ──────
const aliasForName = new Map<string, string>();
for (const [alias, name] of Object.entries(dimensionAliases)) aliasForName.set(name, alias);
for (const [alias, name] of Object.entries(metricAliases)) aliasForName.set(name, alias);
const orderParts: string[] = [];
for (const o of resolved.request.orderBy ?? []) {
const alias = aliasForName.get(o.ref);
if (!alias) {
// Cannot happen after parseAnalyticsQueryRequest, which already refuses
// an orderBy that is not selected. Kept as a hard stop rather than a
// silent skip: a guard that quietly does nothing is how a bypass hides.
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `orderBy "${o.ref}" is not selected`);
}
orderParts.push(`${alias} ${o.direction === 'asc' ? 'ASC' : 'DESC'}`);
}
const requested = resolved.request.limit ?? 1000;
const effectiveLimit = Math.max(1, Math.min(Math.floor(requested), opts.maxResultRows));
const sql =
`SELECT ${selectParts.join(', ')} ` +
`FROM ${opts.database}.${table} ` +
`WHERE ${conditions.join(' AND ')}` +
(groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') +
(orderParts.length ? ` ORDER BY ${orderParts.join(', ')}` : '') +
` LIMIT ${effectiveLimit} FORMAT JSON`;
return {
sql,
params,
forcedTenantPredicate: FORCED_TENANT_PREDICATE,
table,
metricAliases,
dimensionAliases,
effectiveLimit,
};
}
function renderFilter(
lhs: string,
nameKey: string,
operator: FilterOperator,
values: string[],
/**
* Uniquifies this filter's bound-value parameter names. A STRING, not a
* number, because request filters and per-metric definition filters share
* one parameter map: `0` and `0_0m` must not collide, and two collided
* bindings would silently make one filter win over the other.
*/
index: string,
params: Record<string, string>,
): string {
const bind = (v: string, suffix: string): string => {
const key = `fv${index}${suffix}`;
params[`param_${key}`] = v;
return `{${key}:String}`;
};
switch (operator) {
case 'eq':
return `${lhs} = ${bind(values[0], '')}`;
case 'ne':
return `${lhs} != ${bind(values[0], '')}`;
case 'lt':
return `${lhs} < ${bind(values[0], '')}`;
case 'lte':
return `${lhs} <= ${bind(values[0], '')}`;
case 'gt':
return `${lhs} > ${bind(values[0], '')}`;
case 'gte':
return `${lhs} >= ${bind(values[0], '')}`;
case 'contains':
// `position(haystack, needle) > 0` keeps the needle a BOUND VALUE.
// A LIKE with a concatenated pattern would make it SQL text.
return `position(${lhs}, ${bind(values[0], '')}) > 0`;
case 'in':
case 'not_in': {
const bound = values.map((v, i) => bind(v, `_${i}`));
return `${lhs} ${operator === 'in' ? 'IN' : 'NOT IN'} (${bound.join(', ')})`;
}
case 'is_null':
// Dimension values live in a paired-array map; an absent key yields ''
// rather than NULL, so "is null" means "the key is not present". The
// key itself is the already-bound `param_fn<i>` — no new binding, and
// certainly no spliced identifier.
return `NOT has(dim_names, {${nameKey}:String})`;
case 'is_not_null':
return `has(dim_names, {${nameKey}:String})`;
default: {
const _never: never = operator;
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`unsupported filter operator ${String(_never)}`,
);
}
}
}
+169
View File
@@ -0,0 +1,169 @@
/**
* src/analytics/contract/compile-semantic.ts
* ===========================================
* BI-P016, compile target 2 of 2 — and the thing that makes BI-P016's DoD
* ("same request can compile to ClickHouse/semantic engine without consumer
* rewrite") a testable claim rather than a design intention.
*
* A `SemanticQuerySpec` is the engine-neutral form a semantic layer consumes.
* It is deliberately shaped like MetricFlow's query interface (metrics /
* group-by / where / order / limit / time-constraint) so BI-P029's
* `MetricFlowProvider` can execute it without an adapter, but it names
* nothing MetricFlow-specific: swapping MetricFlow for Cube or a hand-rolled
* compiler changes this file's CONSUMER, not this file, and never the
* product API.
*
* The `where` clauses here are STRUCTURED, not SQL strings. A semantic layer
* that accepts SQL predicate text (MetricFlow's `--where` does) would put a
* raw-SQL hole behind the closed DSL — so the spec carries operator+values
* and the provider renders them itself under its own escaping rules.
*/
import type { FilterOperator, ResolvedQuery, TimeGrain } from './query-request';
export interface SemanticMetricRef {
/** Canonical SIAX metric id — stable across semantic-engine replacement. */
metric_id: string;
name: string;
/**
* The metric DEFINITION's own filters, carried through so the semantic
* target computes the same number as the ClickHouse target. Dropping them
* on one side and not the other is how one canonical metric id comes to
* mean two different things — the divergence BI-P029's DoD is written to
* catch. Structured, like every other filter here: never predicate text.
*/
definition_filters?: readonly SemanticWhere[];
}
export interface SemanticGroupBy {
dimension_id: string;
name: string;
/** Set when this group-by is a time bucket rather than a categorical one. */
grain?: TimeGrain;
}
export interface SemanticWhere {
dimension_id: string;
name: string;
operator: FilterOperator;
values: string[];
}
export interface SemanticOrderBy {
ref: string;
direction: 'asc' | 'desc';
}
export interface SemanticTimeConstraint {
/** Inclusive lower bound. */
start?: string;
/** EXCLUSIVE upper bound — matched to the ClickHouse target exactly. */
end?: string;
dimension?: string;
}
export interface SemanticQuerySpec {
dataset_id: string;
semantic_model_version?: number;
metrics: SemanticMetricRef[];
groupBy: SemanticGroupBy[];
where: SemanticWhere[];
orderBy: SemanticOrderBy[];
limit: number;
timeConstraint?: SemanticTimeConstraint;
format: 'json' | 'arrow_ipc';
}
export function compileRequestToSemantic(resolved: ResolvedQuery): SemanticQuerySpec {
const groupBy: SemanticGroupBy[] = resolved.dimensions.map((d) => ({
dimension_id: d.dimension_id,
name: d.name,
}));
const grain = resolved.request.timeRange?.grain;
if (grain) {
groupBy.push({
dimension_id: resolved.request.timeRange?.dimension ?? '__time',
name: resolved.request.timeRange?.dimension ?? '__time',
grain,
});
}
return {
dataset_id: resolved.dataset_id,
semantic_model_version: resolved.semantic_model_version,
metrics: resolved.metrics.map((m) => ({
metric_id: m.metric_id,
name: m.name,
...(m.definition_filters?.length
? {
definition_filters: m.definition_filters.map((f) => ({
// Already resolved onto the physical key; the dimension id is
// the same value here because the definition names a field, not
// a caller-supplied alias.
dimension_id: f.field_name,
name: f.field_name,
operator: f.operator,
values: [...f.values],
})),
}
: {}),
})),
groupBy,
where: resolved.filters.map((f) => ({
dimension_id: f.dimension.dimension_id,
name: f.dimension.name,
operator: f.filter.operator,
values: f.filter.values ?? [],
})),
orderBy: (resolved.request.orderBy ?? []).map((o) => ({ ref: o.ref, direction: o.direction })),
limit: resolved.request.limit ?? 1000,
timeConstraint:
resolved.request.timeRange && (resolved.request.timeRange.from || resolved.request.timeRange.to)
? {
start: resolved.request.timeRange.from,
end: resolved.request.timeRange.to,
dimension: resolved.request.timeRange.dimension,
}
: undefined,
format: resolved.request.format ?? 'json',
};
}
/**
* The comparable SHAPE of a compiled query, target-independent.
*
* BI-P016's DoD is about semantic equivalence, and the honest way to test
* that is to project both targets onto the same tuple and demand equality —
* not to eyeball two strings. `tests/analytics/query-request-contract.test.ts`
* builds this from the ClickHouse SQL by PARSING the compiled artefact, so a
* compiler that quietly drops a filter fails the comparison.
*/
export interface QueryShape {
dataset_id: string;
metrics: string[];
dimensions: string[];
filters: Array<{ dimension: string; operator: FilterOperator; values: string[] }>;
order: string[];
limit: number;
timeFrom?: string;
timeTo?: string;
grain?: TimeGrain;
}
export function shapeOfSemanticSpec(spec: SemanticQuerySpec): QueryShape {
const timeBucket = spec.groupBy.find((g) => g.grain);
return {
dataset_id: spec.dataset_id,
metrics: spec.metrics.map((m) => m.name).sort(),
dimensions: spec.groupBy
.filter((g) => !g.grain)
.map((g) => g.name)
.sort(),
filters: spec.where
.map((w) => ({ dimension: w.name, operator: w.operator, values: [...w.values] }))
.sort((a, b) => (a.dimension + a.operator).localeCompare(b.dimension + b.operator)),
order: spec.orderBy.map((o) => `${o.ref}:${o.direction}`),
limit: spec.limit,
timeFrom: spec.timeConstraint?.start,
timeTo: spec.timeConstraint?.end,
grain: timeBucket?.grain,
};
}
+518
View File
@@ -0,0 +1,518 @@
/**
* src/analytics/contract/query-request.ts
* ========================================
* BI-P016 — QUERY REQUEST CONTRACT.
*
* The canonical shape every SIAX product/application API speaks when it wants
* analytics:
*
* AnalyticsQueryRequest { dataset metrics[] dimensions[] filters[]
* timeRange orderBy limit format }
*
* BI-P016's hard rule: "The default product/application API must NOT accept
* arbitrary SQL." That is enforced structurally, not by a review convention —
* there is no field in this type that can carry SQL, and
* `parseAnalyticsQueryRequest` REJECTS unknown keys rather than ignoring them.
* A caller who adds `sql`, `where`, `having` or `expression` gets
* INVALID_REQUEST, not silent truncation; silent truncation is how an
* "unsupported" field becomes a supported one after the next refactor.
*
* Everything in this request is a NAME, resolved through a declared catalog
* (`QueryResolution`) before it can reach a compiler. Caller text therefore
* never becomes a SQL identifier and never becomes a SQL literal — the two
* ways injection happens.
*
* WHY THIS IS SEPARATE FROM `MetricQueryRequest` (engine/query-compiler.ts):
* that type is the ENGINE's internal, already-resolved, single-metric input,
* and it carries `tenant_id` because the engine sets it. This one is the
* CONSUMER-facing contract: it has no tenant field at all, because a consumer
* must never be able to name a tenant. INF0's Wave-2 hole was exactly a tenant
* read out of a request body; the fix that sticks is a request type in which
* the field does not exist.
*/
import { SiaxError } from '../../platform/errors';
// ── vocabulary ─────────────────────────────────────────────────────────────
/**
* Filter operators. A CLOSED list: every one compiles to a fixed SQL shape
* with bound parameters. There is no "raw" or "expression" operator, and
* adding one would have to survive `filter-operators-are-closed` in
* tests/analytics/query-request-contract.test.ts.
*/
export const FILTER_OPERATORS = [
'eq',
'ne',
'in',
'not_in',
'lt',
'lte',
'gt',
'gte',
'contains',
'is_null',
'is_not_null',
] as const;
export type FilterOperator = (typeof FILTER_OPERATORS)[number];
/** Operators that take exactly one value. */
const SINGLE_VALUE_OPERATORS: readonly FilterOperator[] = ['eq', 'ne', 'lt', 'lte', 'gt', 'gte', 'contains'];
/** Operators that take one-or-more values. */
const MULTI_VALUE_OPERATORS: readonly FilterOperator[] = ['in', 'not_in'];
/** Operators that take NO value. Supplying one is a rejection, not a shrug. */
const NO_VALUE_OPERATORS: readonly FilterOperator[] = ['is_null', 'is_not_null'];
export const TIME_GRAINS = ['hour', 'day', 'week', 'month', 'quarter', 'year'] as const;
export type TimeGrain = (typeof TIME_GRAINS)[number];
export const ORDER_DIRECTIONS = ['asc', 'desc'] as const;
export type OrderDirection = (typeof ORDER_DIRECTIONS)[number];
/** Result encodings, matching `QueryResult.encoding` (BI-P002/BI-P005). */
export const REQUEST_FORMATS = ['json', 'arrow_ipc'] as const;
export type RequestFormat = (typeof REQUEST_FORMATS)[number];
// ── the request ────────────────────────────────────────────────────────────
export interface AnalyticsFilter {
/** Caller-facing dimension NAME. Resolved through the catalog, never spliced. */
dimension: string;
operator: FilterOperator;
/** Literal values. They travel as bound parameters; never as SQL text. */
values?: string[];
}
export interface AnalyticsTimeRange {
/**
* Caller-facing time dimension name. Omitted means "the dataset's declared
* primary time dimension", which the resolution supplies — a consumer never
* has to know the physical column.
*/
dimension?: string;
/** ISO-8601 inclusive lower bound. */
from?: string;
/** ISO-8601 EXCLUSIVE upper bound (half-open, so adjacent ranges never double count). */
to?: string;
grain?: TimeGrain;
}
export interface AnalyticsOrderBy {
/**
* A metric name or a dimension name that ALREADY APPEARS in this request.
* Ordering by something not selected is refused: it is both a correctness
* trap and the obvious place to smuggle an expression.
*/
ref: string;
direction: OrderDirection;
}
/** BI-P016's canonical request, field-for-field. */
export interface AnalyticsQueryRequest {
dataset: string;
metrics: string[];
dimensions: string[];
filters: AnalyticsFilter[];
timeRange?: AnalyticsTimeRange;
orderBy?: AnalyticsOrderBy[];
limit?: number;
format?: RequestFormat;
}
/** The exact keys the contract accepts at the top level. */
export const ANALYTICS_QUERY_REQUEST_KEYS = [
'dataset',
'metrics',
'dimensions',
'filters',
'timeRange',
'orderBy',
'limit',
'format',
] as const;
/**
* Keys a caller might reach for when they want to smuggle SQL. Listed
* explicitly so the rejection MESSAGE names the rule rather than saying
* "unknown key" — an integrator who tries `sql` should learn why, not guess.
* The unknown-key rejection below covers everything, listed or not.
*/
const SQL_SMELLING_KEYS = new Set([
'sql',
'query',
'rawSql',
'raw_sql',
'raw',
'where',
'having',
'expression',
'expr',
'select',
'from',
'groupBy',
'group_by',
'sqlFragment',
'unsafe',
'tenant',
'tenant_id',
'tenantId',
'project_id',
'database',
'catalog',
'table',
]);
const MAX_METRICS = 16;
const MAX_DIMENSIONS = 8;
const MAX_FILTERS = 32;
const MAX_FILTER_VALUES = 500;
const MAX_ORDER_BY = 8;
/** A name is a catalog key, not free text; keep it short and boring. */
const MAX_NAME_LENGTH = 128;
/** A filter VALUE is data (it is bound, never spliced) but still bounded. */
const MAX_VALUE_LENGTH = 4096;
export const DEFAULT_REQUEST_LIMIT = 1000;
function invalid(detail: string): never {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', detail);
}
function asObject(raw: unknown, what: string): Record<string, unknown> {
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`);
return raw as Record<string, unknown>;
}
function asName(raw: unknown, what: string): string {
if (typeof raw !== 'string') invalid(`${what} must be a string`);
const v = raw.trim();
if (!v) invalid(`${what} must not be empty`);
if (v.length > MAX_NAME_LENGTH) invalid(`${what} exceeds ${MAX_NAME_LENGTH} characters`);
return v;
}
function rejectUnknownKeys(obj: Record<string, unknown>, allowed: readonly string[], what: string): void {
for (const key of Object.keys(obj)) {
if (allowed.includes(key)) continue;
if (SQL_SMELLING_KEYS.has(key)) {
invalid(
`"${key}" is not part of the analytics query contract — the product API does not accept SQL, ` +
`catalogs, or tenant selection (BI-P016)`,
);
}
invalid(`unknown field "${key}" in ${what}`);
}
}
/**
* Parse and VALIDATE an untrusted request body into the canonical contract.
*
* Fail-closed at every branch: an unparseable field is a rejection, never a
* default. The only defaulting this function does is `filters: []`,
* `dimensions: []` and `limit`, and each of those defaults is strictly
* narrower than any value a caller could have supplied.
*/
export function parseAnalyticsQueryRequest(raw: unknown): AnalyticsQueryRequest {
const obj = asObject(raw, 'analytics query request');
rejectUnknownKeys(obj, ANALYTICS_QUERY_REQUEST_KEYS, 'analytics query request');
const dataset = asName(obj.dataset, 'dataset');
if (!Array.isArray(obj.metrics)) invalid('metrics must be an array');
if (obj.metrics.length === 0) invalid('at least one metric is required');
if (obj.metrics.length > MAX_METRICS) invalid(`at most ${MAX_METRICS} metrics per request`);
const metrics = obj.metrics.map((m, i) => asName(m, `metrics[${i}]`));
if (new Set(metrics).size !== metrics.length) invalid('metrics contains duplicates');
const rawDimensions = obj.dimensions ?? [];
if (!Array.isArray(rawDimensions)) invalid('dimensions must be an array');
if (rawDimensions.length > MAX_DIMENSIONS) invalid(`at most ${MAX_DIMENSIONS} dimensions per request`);
const dimensions = rawDimensions.map((d, i) => asName(d, `dimensions[${i}]`));
if (new Set(dimensions).size !== dimensions.length) invalid('dimensions contains duplicates');
const rawFilters = obj.filters ?? [];
if (!Array.isArray(rawFilters)) invalid('filters must be an array');
if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters per request`);
const filters = rawFilters.map((f, i) => parseFilter(f, i));
const timeRange = obj.timeRange === undefined ? undefined : parseTimeRange(obj.timeRange);
const rawOrderBy = obj.orderBy ?? [];
if (!Array.isArray(rawOrderBy)) invalid('orderBy must be an array');
if (rawOrderBy.length > MAX_ORDER_BY) invalid(`at most ${MAX_ORDER_BY} orderBy entries`);
const orderBy = rawOrderBy.map((o, i) => parseOrderBy(o, i));
const selectable = new Set<string>([...metrics, ...dimensions]);
for (const o of orderBy) {
if (!selectable.has(o.ref)) {
invalid(`orderBy "${o.ref}" is neither a requested metric nor a requested dimension`);
}
}
let limit = DEFAULT_REQUEST_LIMIT;
if (obj.limit !== undefined) {
if (typeof obj.limit !== 'number' || !Number.isFinite(obj.limit))
invalid('limit must be a finite number');
if (!Number.isInteger(obj.limit)) invalid('limit must be an integer');
if (obj.limit < 1) invalid('limit must be >= 1');
limit = obj.limit;
}
let format: RequestFormat = 'json';
if (obj.format !== undefined) {
if (typeof obj.format !== 'string' || !(REQUEST_FORMATS as readonly string[]).includes(obj.format)) {
invalid(`format must be one of ${REQUEST_FORMATS.join(', ')}`);
}
format = obj.format as RequestFormat;
}
return { dataset, metrics, dimensions, filters, timeRange, orderBy, limit, format };
}
function parseFilter(raw: unknown, index: number): AnalyticsFilter {
const obj = asObject(raw, `filters[${index}]`);
rejectUnknownKeys(obj, ['dimension', 'operator', 'values'], `filters[${index}]`);
const dimension = asName(obj.dimension, `filters[${index}].dimension`);
if (typeof obj.operator !== 'string' || !(FILTER_OPERATORS as readonly string[]).includes(obj.operator)) {
invalid(`filters[${index}].operator must be one of ${FILTER_OPERATORS.join(', ')}`);
}
const operator = obj.operator as FilterOperator;
if (NO_VALUE_OPERATORS.includes(operator)) {
if (obj.values !== undefined) invalid(`filters[${index}].operator "${operator}" takes no values`);
return { dimension, operator };
}
if (!Array.isArray(obj.values)) invalid(`filters[${index}].values must be an array`);
if (obj.values.length === 0) invalid(`filters[${index}].values must not be empty`);
if (obj.values.length > MAX_FILTER_VALUES) {
invalid(`filters[${index}].values exceeds ${MAX_FILTER_VALUES} entries`);
}
if (SINGLE_VALUE_OPERATORS.includes(operator) && obj.values.length !== 1) {
invalid(`filters[${index}].operator "${operator}" takes exactly one value`);
}
if (!MULTI_VALUE_OPERATORS.includes(operator) && !SINGLE_VALUE_OPERATORS.includes(operator)) {
invalid(`filters[${index}].operator "${operator}" is not a value operator`);
}
const values = obj.values.map((v, i) => {
if (typeof v !== 'string') invalid(`filters[${index}].values[${i}] must be a string`);
if (v.length > MAX_VALUE_LENGTH)
invalid(`filters[${index}].values[${i}] exceeds ${MAX_VALUE_LENGTH} characters`);
return v;
});
return { dimension, operator, values };
}
function parseTimeRange(raw: unknown): AnalyticsTimeRange {
const obj = asObject(raw, 'timeRange');
rejectUnknownKeys(obj, ['dimension', 'from', 'to', 'grain'], 'timeRange');
const out: AnalyticsTimeRange = {};
if (obj.dimension !== undefined) out.dimension = asName(obj.dimension, 'timeRange.dimension');
if (obj.from !== undefined) out.from = asIsoInstant(obj.from, 'timeRange.from');
if (obj.to !== undefined) out.to = asIsoInstant(obj.to, 'timeRange.to');
if (obj.grain !== undefined) {
if (typeof obj.grain !== 'string' || !(TIME_GRAINS as readonly string[]).includes(obj.grain)) {
invalid(`timeRange.grain must be one of ${TIME_GRAINS.join(', ')}`);
}
out.grain = obj.grain as TimeGrain;
}
if (out.from && out.to && Date.parse(out.from) >= Date.parse(out.to)) {
invalid('timeRange.from must be strictly before timeRange.to');
}
return out;
}
/**
* A timestamp is validated as a real instant, not merely as "a string that
* gets bound". It is bound as a parameter either way, so this is not the
* injection defence — it is the defence against a nonsense range silently
* producing an empty answer that a dashboard renders as "zero sales".
*/
function asIsoInstant(raw: unknown, what: string): string {
if (typeof raw !== 'string') invalid(`${what} must be an ISO-8601 string`);
const v = raw.trim();
if (!/^\d{4}-\d{2}-\d{2}([T ]\d{2}:\d{2}(:\d{2}(\.\d{1,9})?)?(Z|[+-]\d{2}:?\d{2})?)?$/.test(v)) {
invalid(`${what} is not an ISO-8601 timestamp`);
}
if (Number.isNaN(Date.parse(v))) invalid(`${what} is not a valid timestamp`);
return v;
}
function parseOrderBy(raw: unknown, index: number): AnalyticsOrderBy {
const obj = asObject(raw, `orderBy[${index}]`);
rejectUnknownKeys(obj, ['ref', 'direction'], `orderBy[${index}]`);
const ref = asName(obj.ref, `orderBy[${index}].ref`);
if (typeof obj.direction !== 'string' || !(ORDER_DIRECTIONS as readonly string[]).includes(obj.direction)) {
invalid(`orderBy[${index}].direction must be one of ${ORDER_DIRECTIONS.join(', ')}`);
}
return { ref, direction: obj.direction as OrderDirection };
}
// ── resolution: names -> declared catalog entries ──────────────────────────
/**
* What a metric NAME resolves to. Supplied from `MetricDefinition` rows
* (BI-P002/BI-P028); the compilers never invent one.
*/
/**
* A filter that belongs to the METRIC DEFINITION rather than to the request
* (BI-P028 `MetricSpec.filters`) — "net revenue excludes internal transfers"
* is part of what the metric MEANS, not something a caller asks for.
*
* It is carried in ALREADY-RESOLVED form: `field_name` is the physical key
* inside `dim_names`, mapped from the definition's dimension name by whoever
* builds the resolution. The compilers stay dumb — they bind and render, they
* never look a dimension up — and `assertSafeIdentifier` still applies.
*
* These filters are NOT optional decoration. A compile path that drops them
* answers a different question under the same canonical metric id, which is
* exactly the divergence BI-P029's DoD exists to catch; see
* tests/integration/semantic-parity.integration.test.ts.
*/
export interface ResolvedMetricFilter {
field_name: string;
operator: FilterOperator;
values: readonly string[];
}
export interface ResolvedMetric {
name: string;
metric_id: string;
/** The `metric` key stored in the analytics event table. */
event_metric_key: string;
aggregation: 'sum' | 'avg' | 'min' | 'max' | 'count' | 'count_distinct';
/** Dimension names this metric may legitimately be sliced by. */
allowed_dimensions: readonly string[];
unit?: string;
/**
* The metric definition's own filters. Applied PER METRIC — folded into that
* metric's conditional aggregate — never into the shared WHERE clause: a
* request may ask for two metrics whose definitions exclude different rows,
* and a shared predicate would silently apply one metric's meaning to the
* other.
*/
definition_filters?: readonly ResolvedMetricFilter[];
}
/** What a dimension NAME resolves to. Supplied from `DimensionDefinition` rows. */
export interface ResolvedDimension {
name: string;
dimension_id: string;
/** Physical key inside `dim_names`. Validated as an identifier downstream. */
field_name: string;
}
/**
* The declared surface for ONE dataset, for ONE tenant. A compiler is handed
* this; it cannot reach past it. An unresolvable name is a rejection.
*/
export interface QueryResolution {
dataset_id: string;
/** Semantic model version this resolution came from — part of the cache key (BI-P020). */
semantic_model_version?: number;
metric(name: string): ResolvedMetric | undefined;
dimension(name: string): ResolvedDimension | undefined;
/** Physical time dimension used when `timeRange.dimension` is omitted. */
defaultTimeDimension?: string;
}
export function queryResolutionFrom(args: {
dataset_id: string;
metrics: readonly ResolvedMetric[];
dimensions: readonly ResolvedDimension[];
semantic_model_version?: number;
defaultTimeDimension?: string;
}): QueryResolution {
const metricMap = new Map(args.metrics.map((m) => [m.name, m]));
const dimMap = new Map(args.dimensions.map((d) => [d.name, d]));
return {
dataset_id: args.dataset_id,
semantic_model_version: args.semantic_model_version,
defaultTimeDimension: args.defaultTimeDimension,
// Map lookups: "__proto__" / "constructor" resolve to undefined, as they must.
metric: (name) => metricMap.get(name),
dimension: (name) => dimMap.get(name),
};
}
/**
* Resolve every name in a request, or refuse the request.
*
* This is the single place where "a caller said X" becomes "the catalog
* declares Y". Both compile targets (ClickHouse and semantic) consume the
* RESOLVED form, which is why BI-P016's DoD holds: the consumer writes one
* request and neither target sees consumer text.
*/
export interface ResolvedQuery {
request: AnalyticsQueryRequest;
dataset_id: string;
metrics: ResolvedMetric[];
dimensions: ResolvedDimension[];
filters: Array<{ filter: AnalyticsFilter; dimension: ResolvedDimension }>;
timeDimension?: string;
semantic_model_version?: number;
}
export function resolveAnalyticsQuery(
request: AnalyticsQueryRequest,
resolution: QueryResolution,
): ResolvedQuery {
if (request.dataset !== resolution.dataset_id) {
// The resolution is built FOR a dataset the caller was authorised to read
// (BI-P006 grants). A mismatch means the caller is asking about something
// else entirely, and answering from this resolution would be a leak.
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics',
'requested dataset does not match the authorised resolution',
);
}
const metrics = request.metrics.map((name) => {
const m = resolution.metric(name);
if (!m) invalid(`unknown metric "${name}" for this dataset`);
return m;
});
const dimensions = request.dimensions.map((name) => {
const d = resolution.dimension(name);
if (!d) invalid(`unknown dimension "${name}" for this dataset`);
return d;
});
// Every requested dimension must be legitimate for EVERY requested metric.
// Slicing a metric by a dimension its definition does not allow is how a
// "harmless" grouping turns into an unintended disclosure.
for (const m of metrics) {
for (const d of dimensions) {
if (!m.allowed_dimensions.includes(d.name)) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics',
`metric "${m.name}" may not be sliced by dimension "${d.name}"`,
);
}
}
}
const filters = request.filters.map((f) => {
const d = resolution.dimension(f.dimension);
if (!d) invalid(`unknown filter dimension "${f.dimension}"`);
return { filter: f, dimension: d };
});
let timeDimension: string | undefined;
if (request.timeRange) {
if (request.timeRange.dimension) {
const d = resolution.dimension(request.timeRange.dimension);
if (!d) invalid(`unknown time dimension "${request.timeRange.dimension}"`);
timeDimension = d.field_name;
} else {
timeDimension = resolution.defaultTimeDimension;
if ((request.timeRange.from || request.timeRange.to) && !timeDimension) {
invalid('dataset declares no default time dimension; timeRange.dimension is required');
}
}
}
return {
request,
dataset_id: resolution.dataset_id,
metrics,
dimensions,
filters,
timeDimension,
semantic_model_version: resolution.semantic_model_version,
};
}
+415
View File
@@ -0,0 +1,415 @@
/**
* src/analytics/cost/query-budget.ts
* ===================================
* BI-P021 — QUERY COST / BUDGET.
*
* ── SAY THE HONEST THING FIRST ────────────────────────────────────────────
* A budget checked once before execution and never again bounds NOTHING. The
* estimate is a planner's guess; a query that scans ten times its estimate
* runs to completion regardless, and the budget discovers this after the money
* is spent. Every "cost control" that consists only of a pre-flight check is
* an accounting feature wearing a safety feature's name.
*
* ClickHouse's own `max_rows_to_read` (BI-P012) DOES stop a query mid-flight —
* but it bounds ONE query. It cannot express "this tenant may scan 10 billion
* rows per hour across all its queries", because it has no idea what the
* tenant's other queries did. That window is what a budget is for, and it is
* exactly the thing a per-query setting cannot enforce.
*
* So this module has three parts, and only the second one is a control:
*
* 1. ADMISSION — estimate vs. remaining window. Rejects the obviously
* unaffordable before it starts. Cheap, and insufficient.
* 2. IN-FLIGHT — polls the engine's live `read_rows`/`read_bytes` counters
* for the running query and CANCELS it (via BI-P019, with
* verification) the moment window spend would be exceeded.
* THIS is what makes the budget real.
* 3. ACCOUNTING — settles the actual cost into a ledger the control plane
* reads: CL0UD quotas, AUD0 usage events, B00K accounting.
*
* If part 2 is disabled (`enforceInFlight: false`), `describeEnforcement()`
* says so in plain words, and the admission-only posture is reported as
* "advisory". A control that does not control should say which one it is.
*
* ── THE CAVEAT THAT THE LIVE RUN MEASURED ────────────────────────────────
* In-flight enforcement is granular to the poll interval, so a query can
* OVERSHOOT its budget between polls — and on a columnar engine the overshoot
* is large. Measured against ClickHouse 24.8.14.39 with a 1,000,000-row window
* and a 100ms poll: the query was cancelled at 153,318,696 rows read, 55ms in.
* A 153x overshoot. See evidence/bi-p019-p021/cancellation-and-budget.json,
* which is regenerated by the live suite — the figure is machine-dependent and
* moves between runs (an earlier run on the same box recorded 131M / 130x).
* Do not quote a specific multiple; quote the SHAPE.
*
* That shape is still a bound — the same query, unwatched, read 4,227,318,261
* rows before it was stopped in the BI-P019 case — but it is a bound of the
* form "within roughly one poll interval of engine throughput", not "at
* exactly the budget". Anyone sizing a budget must size it for that, and
* anyone quoting this as a hard cap is wrong. Shortening the interval narrows
* the overshoot and costs a `system.processes` read per poll per running
* query.
*/
import { SiaxError } from '../../platform/errors';
import {
cancelAndVerify,
type CancellationOutcome,
type EngineProcessControl,
} from '../execution/cancellation';
/** The cost axes BI-P021 names. */
export interface QueryCost {
rows_scanned: number;
bytes_scanned: number;
execution_ms: number;
engine: string;
/** Bytes read from object storage, when the engine distinguishes it. */
storage_read_bytes?: number;
}
/**
* Weights turning raw counters into the single `cost_units` figure BI-P018
* records and B00K bills.
*
* The defaults are deliberately dominated by BYTES, not rows: on a columnar
* engine a billion-row scan of one narrow column is cheap and a
* hundred-million-row scan of a wide one is not, so a rows-weighted unit would
* bill the wrong query.
*/
export interface CostModel {
unitsPerBillionRows: number;
unitsPerGigabyteScanned: number;
unitsPerSecond: number;
unitsPerGigabyteStorageRead: number;
}
export const DEFAULT_COST_MODEL: CostModel = {
unitsPerBillionRows: 1,
unitsPerGigabyteScanned: 10,
unitsPerSecond: 2,
unitsPerGigabyteStorageRead: 15,
};
export function costUnits(cost: QueryCost, model: CostModel = DEFAULT_COST_MODEL): number {
const units =
(cost.rows_scanned / 1e9) * model.unitsPerBillionRows +
(cost.bytes_scanned / 2 ** 30) * model.unitsPerGigabyteScanned +
(cost.execution_ms / 1000) * model.unitsPerSecond +
((cost.storage_read_bytes ?? 0) / 2 ** 30) * model.unitsPerGigabyteStorageRead;
// Rounded to six places so the same cost always serialises identically;
// float noise in a billing figure produces support tickets.
return Math.round(units * 1e6) / 1e6;
}
/** A tenant's allowance over a rolling window. Issued by CL0UD entitlements. */
export interface QueryBudget {
windowMs: number;
maxRowsScanned: number;
maxBytesScanned: number;
maxCostUnits: number;
/** What to do when the window is exhausted. */
onExhausted: 'reject' | 'throttle';
/** For `throttle`: queries are delayed by this much instead of refused. */
throttleDelayMs?: number;
}
export const DEFAULT_QUERY_BUDGET: QueryBudget = {
windowMs: 60 * 60 * 1000,
maxRowsScanned: 50_000_000_000,
maxBytesScanned: 2 * 2 ** 40,
maxCostUnits: 10_000,
onExhausted: 'reject',
};
interface LedgerEntry {
at: number;
rows: number;
bytes: number;
units: number;
}
/** Where settled cost goes. AUD0/B00K/CL0UD sit behind this. */
export interface UsageLedgerSink {
record(entry: {
tenant_id: string;
project_id?: string;
principal_id: string;
query_id: string;
cost: QueryCost;
cost_units: number;
/** True when the query was stopped by the budget rather than by the user. */
terminated_by_budget: boolean;
at: string;
}): Promise<void> | void;
}
export interface WindowSpend {
rows_scanned: number;
bytes_scanned: number;
cost_units: number;
queries: number;
window_started_at: string;
}
/**
* In-memory rolling window per tenant.
*
* Explicitly NOT the durable ledger — this is one process's view, and a
* multi-replica deployment needs the shared counter (Valkey or the CL0UD quota
* service) behind `SharedWindowStore` for the budget to bind across replicas.
* Stated rather than implied: with N replicas and no shared store, a tenant
* can spend N times its budget. `describeEnforcement()` reports this.
*/
export interface SharedWindowStore {
/** Atomically add to the tenant's window and return the new totals. */
add(
tenantId: string,
entry: { rows: number; bytes: number; units: number; windowMs: number },
): Promise<WindowSpend>;
get(tenantId: string): Promise<WindowSpend>;
}
export class InMemoryWindowStore implements SharedWindowStore {
private readonly entries = new Map<string, LedgerEntry[]>();
constructor(private readonly now: () => number = Date.now) {}
private prune(tenantId: string, windowMs: number): LedgerEntry[] {
const cutoff = this.now() - windowMs;
const kept = (this.entries.get(tenantId) ?? []).filter((e) => e.at >= cutoff);
this.entries.set(tenantId, kept);
return kept;
}
async add(
tenantId: string,
entry: { rows: number; bytes: number; units: number; windowMs: number },
): Promise<WindowSpend> {
const kept = this.prune(tenantId, entry.windowMs);
kept.push({ at: this.now(), rows: entry.rows, bytes: entry.bytes, units: entry.units });
this.entries.set(tenantId, kept);
return this.totals(kept, entry.windowMs);
}
async get(tenantId: string, windowMs = DEFAULT_QUERY_BUDGET.windowMs): Promise<WindowSpend> {
return this.totals(this.prune(tenantId, windowMs), windowMs);
}
private totals(entries: LedgerEntry[], windowMs: number): WindowSpend {
return {
rows_scanned: entries.reduce((a, e) => a + e.rows, 0),
bytes_scanned: entries.reduce((a, e) => a + e.bytes, 0),
cost_units: entries.reduce((a, e) => a + e.units, 0),
queries: entries.length,
window_started_at: new Date(this.now() - windowMs).toISOString(),
};
}
}
export interface BudgetEnforcerOptions {
budget: QueryBudget;
store: SharedWindowStore;
ledger: UsageLedgerSink;
control?: EngineProcessControl;
costModel?: CostModel;
/** The control. Off means admission-only, and `describeEnforcement` says so. */
enforceInFlight?: boolean;
pollIntervalMs?: number;
/** True when `store` is shared across replicas. Affects the honest description. */
storeIsShared?: boolean;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
export interface AdmissionDecision {
admitted: boolean;
reason?: string;
/** For `throttle`: how long the caller must wait first. */
delay_ms?: number;
remaining: { rows: number; bytes: number; units: number };
}
export interface InFlightOutcome {
/** True when the enforcer stopped the query. */
terminated: boolean;
reason?: 'rows' | 'bytes' | 'units';
/** Present when it terminated: the verified cancellation. */
cancellation?: CancellationOutcome;
observed: { rows_read: number; bytes_read: number; elapsed_ms: number };
polls: number;
}
export class QueryBudgetEnforcer {
private readonly opts: Required<
Pick<BudgetEnforcerOptions, 'enforceInFlight' | 'pollIntervalMs' | 'storeIsShared'>
> &
BudgetEnforcerOptions;
private readonly now: () => number;
private readonly sleep: (ms: number) => Promise<void>;
constructor(options: BudgetEnforcerOptions) {
this.opts = {
enforceInFlight: options.enforceInFlight ?? true,
pollIntervalMs: options.pollIntervalMs ?? 200,
storeIsShared: options.storeIsShared ?? false,
...options,
};
this.now = options.now ?? Date.now;
this.sleep = options.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
if (this.opts.enforceInFlight && !this.opts.control) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.cost',
'in-flight budget enforcement requires an EngineProcessControl; ' +
'construct with enforceInFlight:false to run admission-only and be honest about it',
);
}
}
/**
* Exactly what this enforcer does and does not bound, in words an operator
* can paste into an incident review. Returned by the analytics health
* endpoint so the posture is visible without reading this file.
*/
describeEnforcement(): {
admission: string;
in_flight: string;
across_replicas: string;
binding: boolean;
} {
return {
admission: 'estimate is compared to the remaining window before the query starts',
in_flight: this.opts.enforceInFlight
? `live engine counters are polled every ${this.opts.pollIntervalMs}ms and the query is cancelled ` +
'(with verification) when window spend would be exceeded'
: 'NOT ENFORCED — this budget is advisory. A query that exceeds its estimate runs to completion.',
across_replicas: this.opts.storeIsShared
? 'window spend is held in a shared store, so the budget binds across replicas'
: 'window spend is per-process — with N replicas a tenant can spend up to N times this budget',
binding: this.opts.enforceInFlight && this.opts.storeIsShared,
};
}
async admit(
tenantId: string,
estimate: { rows_to_read: number; bytes_to_read: number },
): Promise<AdmissionDecision> {
const spend = await this.opts.store.get(tenantId);
const b = this.opts.budget;
const remaining = {
rows: Math.max(0, b.maxRowsScanned - spend.rows_scanned),
bytes: Math.max(0, b.maxBytesScanned - spend.bytes_scanned),
units: Math.max(0, b.maxCostUnits - spend.cost_units),
};
const over =
estimate.rows_to_read > remaining.rows
? 'rows'
: estimate.bytes_to_read > remaining.bytes
? 'bytes'
: undefined;
if (!over) return { admitted: true, remaining };
if (b.onExhausted === 'throttle') {
return {
admitted: true,
delay_ms: b.throttleDelayMs ?? 1000,
reason: `tenant window ${over} budget exhausted; throttled`,
remaining,
};
}
return { admitted: false, reason: `tenant window ${over} budget exhausted`, remaining };
}
/**
* Watch a RUNNING query and stop it if it breaches the window.
*
* Runs concurrently with the query itself; the caller awaits both. Returns
* as soon as the query leaves the engine's process list, so a well-behaved
* query costs one extra poll.
*/
async enforceInFlight(tenantId: string, queryId: string, signal?: AbortSignal): Promise<InFlightOutcome> {
const observed = { rows_read: 0, bytes_read: 0, elapsed_ms: 0 };
if (!this.opts.enforceInFlight || !this.opts.control?.progress) {
return { terminated: false, observed, polls: 0 };
}
const control = this.opts.control;
const spendBefore = await this.opts.store.get(tenantId);
const b = this.opts.budget;
const remainingRows = Math.max(0, b.maxRowsScanned - spendBefore.rows_scanned);
const remainingBytes = Math.max(0, b.maxBytesScanned - spendBefore.bytes_scanned);
const remainingUnits = Math.max(0, b.maxCostUnits - spendBefore.cost_units);
let polls = 0;
for (;;) {
if (signal?.aborted) return { terminated: false, observed, polls };
let progress;
try {
progress = await control.progress?.(queryId);
} catch {
progress = undefined;
}
polls += 1;
if (!progress) {
// Gone from the process list: finished, or never started. Either way
// there is nothing left to bound.
return { terminated: false, observed, polls };
}
observed.rows_read = progress.rows_read;
observed.bytes_read = progress.bytes_read;
observed.elapsed_ms = progress.elapsed_ms;
const liveUnits = costUnits(
{
rows_scanned: progress.rows_read,
bytes_scanned: progress.bytes_read,
execution_ms: progress.elapsed_ms,
engine: control.engine,
},
this.opts.costModel,
);
const breach =
progress.rows_read > remainingRows
? 'rows'
: progress.bytes_read > remainingBytes
? 'bytes'
: liveUnits > remainingUnits
? 'units'
: undefined;
if (breach) {
const cancellation = await cancelAndVerify(control, queryId, { timeoutMs: 5_000 });
return { terminated: true, reason: breach, cancellation, observed, polls };
}
await this.sleep(this.opts.pollIntervalMs);
}
}
/** Settle the actual cost. Called for EVERY outcome, including failures. */
async settle(args: {
tenant_id: string;
project_id?: string;
principal_id: string;
query_id: string;
cost: QueryCost;
terminated_by_budget?: boolean;
}): Promise<{ cost_units: number; window: WindowSpend }> {
const units = costUnits(args.cost, this.opts.costModel);
const window = await this.opts.store.add(args.tenant_id, {
rows: args.cost.rows_scanned,
bytes: args.cost.bytes_scanned,
units,
windowMs: this.opts.budget.windowMs,
});
await this.opts.ledger.record({
tenant_id: args.tenant_id,
project_id: args.project_id,
principal_id: args.principal_id,
query_id: args.query_id,
cost: args.cost,
cost_units: units,
// A query the budget killed still COST what it burned before it died.
// Not charging for it is how a tenant discovers that the cheapest query
// is one that gets cancelled.
terminated_by_budget: args.terminated_by_budget ?? false,
at: new Date(this.now()).toISOString(),
});
return { cost_units: units, window };
}
}
@@ -0,0 +1,837 @@
/**
* src/analytics/dashboards/dashboard-document.ts
* ===============================================
* BI-P060 — ANALYTICS DASHBOARD MODEL.
*
* BI-P002 already named `DashboardDefinition` and `DashboardVersion` and gave
* them storage (sql/008). What it did not give them is a BODY: `layout_json`
* is a string, and a string can hold anything — including a Superset export
* that only Superset can read back. This module is that body, and its whole
* point is the DoD:
*
* "A native SIAX dashboard can be reconstructed without Superset."
*
* Reconstruction is not a claim you can make by writing a type. It is provable
* exactly once you can take a document and produce, for every widget, a
* runnable `AnalyticsQueryRequest` (BI-P016) plus a rendering spec, with no
* vendor identifier anywhere in the plan. `planDashboard()` is that function
* and `tests/analytics/dashboard-model.test.ts` runs it over a document
* carrying Superset ids, asserting the plan is complete and vendor-free.
*
* THE VENDOR RULE, stated once: a Superset/Metabase/Cube identifier may be
* carried as `vendor_ref` for round-tripping, and it is
* - never a widget's source of truth (a widget with only a vendor_ref is
* REFUSED — it is precisely the document that cannot be reconstructed),
* - never part of the canonical hash (so re-exporting to a new Superset
* instance does not fabricate a new dashboard version).
*
* NO SQL LIVES HERE either. Widget sources are metric/dimension NAMES that
* resolve through the catalog, and query widgets are validated by
* `parseAnalyticsQueryRequest`, which rejects `sql`, `where`, `tenant_id` and
* every other smuggling key structurally. A dashboard is therefore not a new
* way into the engine; it is a saved set of contract requests.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import {
parseAnalyticsQueryRequest,
TIME_GRAINS,
type AnalyticsFilter,
type AnalyticsQueryRequest,
type FilterOperator,
type TimeGrain,
} from '../contract/query-request';
// ── vocabulary ─────────────────────────────────────────────────────────────
/**
* Visualization types. A CLOSED list, deliberately engine-neutral: these are
* SIAX chart kinds (DES1GN BI-P051..P057 renders them), not Superset viz_type
* strings. A document naming `echarts_timeseries` is a Superset document.
*/
export const VISUALIZATION_TYPES = [
'big_number',
'line',
'area',
'bar',
'stacked_bar',
'pie',
'table',
'pivot_table',
'scatter',
'heatmap',
'funnel',
'map',
] as const;
export type VisualizationType = (typeof VISUALIZATION_TYPES)[number];
export const REFRESH_MODES = ['manual', 'interval', 'scheduled'] as const;
export type RefreshMode = (typeof REFRESH_MODES)[number];
/**
* Who may open the dashboard. `roles` are cl0ud role names (BI-P033 decides,
* this only declares). There is deliberately NO tenant field: a dashboard is
* stored inside a tenant, and a document that could name one would be a way to
* ask for another tenant's dashboard — the INF0 Wave-2 shape.
*/
export const DASHBOARD_VISIBILITIES = ['private', 'tenant', 'roles'] as const;
export type DashboardVisibility = (typeof DASHBOARD_VISIBILITIES)[number];
export const PARAMETER_TYPES = ['string', 'number', 'date', 'enum'] as const;
export type ParameterType = (typeof PARAMETER_TYPES)[number];
// ── the document ───────────────────────────────────────────────────────────
/** Grid placement. A 12-column grid, matching the portal layout (BI-P062). */
export interface WidgetLayout {
x: number;
y: number;
w: number;
h: number;
}
/** A saved dashboard filter. Its VALUES are data; its dimension is a name. */
export interface DashboardFilter {
filter_id: string;
label: string;
/** Caller-facing dimension name, resolved through the catalog downstream. */
dimension: string;
operator: FilterOperator;
/** Default values, or `{{param}}` references to a declared parameter. */
default_values?: string[];
/** May a viewer change it? A locked filter is how row scoping is saved. */
locked?: boolean;
}
/** A declared parameter. Widgets and filters may reference it as `{{name}}`. */
export interface DashboardParameter {
name: string;
type: ParameterType;
required: boolean;
default_value?: string;
/** Required for `enum`; a closed list is the only way a value can be trusted. */
allowed_values?: string[];
}
/** A widget backed by ONE metric — the common case, and the cheapest to plan. */
export interface MetricWidgetSource {
kind: 'metric';
dataset: string;
metric: string;
dimensions?: string[];
grain?: TimeGrain;
limit?: number;
}
/** A widget backed by a saved contract request (still no SQL, by construction). */
export interface QueryWidgetSource {
kind: 'query';
/** Parsed with `parseAnalyticsQueryRequest` at validation time, not at render time. */
request: AnalyticsQueryRequest;
}
export type WidgetSource = MetricWidgetSource | QueryWidgetSource;
export interface VisualizationSpec {
type: VisualizationType;
/** Field roles by NAME. `x`/`y` name metrics or dimensions in the source. */
encoding: {
x?: string;
y?: string[];
series?: string;
value?: string;
};
/** Presentation-only knobs. Bounded scalars; never code, never a template. */
options?: Record<string, string | number | boolean>;
}
export interface DashboardWidget {
widget_id: string;
title: string;
source: WidgetSource;
visualization: VisualizationSpec;
layout: WidgetLayout;
/** filter_ids this widget honours. Empty means "all dashboard filters". */
filter_bindings?: string[];
/** Advisory only: excluded from the canonical hash, never a source of truth. */
vendor_ref?: VendorRef;
}
/** A foreign system's id for the same object. Round-tripping aid, nothing more. */
export interface VendorRef {
system: string;
id: string;
}
export interface DashboardRefresh {
mode: RefreshMode;
/** Required for `interval`. */
interval_seconds?: number;
/** Required for `scheduled`. */
cron?: string;
/**
* The staleness the OWNER accepts. BI-P026 computes actual freshness; this
* is the number that makes a freshness verdict actionable rather than
* decorative, and BI-P105 reads it when deciding whether a pre-aggregate may
* serve this dashboard.
*/
max_staleness_seconds: number;
}
export interface DashboardPermissions {
visibility: DashboardVisibility;
/** Required (non-empty) when visibility is `roles`. */
roles?: string[];
/** Optional cl0ud entitlement key (BI-P079). */
required_entitlement?: string;
}
/** BI-P060's canonical document, field-for-field with the program's list. */
export interface DashboardDocument {
id: string;
title: string;
description?: string;
layout: { grid_columns: number; row_height_px: number };
filters: DashboardFilter[];
widgets: DashboardWidget[];
parameters: DashboardParameter[];
refresh: DashboardRefresh;
permissions: DashboardPermissions;
/** Document schema version — bumped when THIS shape changes, not per edit. */
version: number;
vendor_ref?: VendorRef;
}
export const DASHBOARD_DOCUMENT_KEYS = [
'id',
'title',
'description',
'layout',
'filters',
'widgets',
'parameters',
'refresh',
'permissions',
'version',
'vendor_ref',
] as const;
/**
* Keys that mean "this document is really a Superset export". Named
* explicitly so the refusal explains the architecture instead of saying
* "unknown key"; the unknown-key rule below catches everything else anyway.
*/
const VENDOR_NATIVE_KEYS = new Set([
'slice_id',
'slice_name',
'viz_type',
'datasource',
'datasource_id',
'datasource_type',
'form_data',
'query_context',
'chart_id',
'dashboard_id_superset',
'position_json',
'json_metadata',
'card_id',
'cube',
]);
/** Keys that would turn a dashboard into an SQL surface. */
const SQL_SMELLING_KEYS = new Set([
'sql',
'query_text',
'statement',
'expression',
'where',
'having',
'raw',
'tenant',
'tenant_id',
'project_id',
'database',
'table',
]);
const MAX_WIDGETS = 60;
const MAX_FILTERS = 32;
const MAX_PARAMETERS = 24;
const MAX_NAME_LENGTH = 128;
const MAX_TITLE_LENGTH = 200;
const MAX_GRID_COLUMNS = 24;
const MIN_REFRESH_INTERVAL_SECONDS = 10;
const MAX_STALENESS_SECONDS = 30 * 24 * 3600;
/** `{{param}}` — the only templating the document has, and it binds a VALUE. */
const PARAM_REFERENCE = /^\{\{\s*([A-Za-z][A-Za-z0-9_]{0,63})\s*\}\}$/;
const IDENTIFIER = /^[A-Za-z][A-Za-z0-9_.-]{0,63}$/;
function invalid(detail: string): never {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dashboard', detail);
}
function asObject(raw: unknown, what: string): Record<string, unknown> {
if (raw === null || typeof raw !== 'object' || Array.isArray(raw)) invalid(`${what} must be an object`);
return raw as Record<string, unknown>;
}
function rejectUnknownKeys(obj: Record<string, unknown>, allowed: readonly string[], what: string): void {
for (const key of Object.keys(obj)) {
if (allowed.includes(key)) continue;
if (VENDOR_NATIVE_KEYS.has(key)) {
invalid(
`"${key}" in ${what} is a vendor-native field — a SIAX dashboard is defined by metrics, ` +
`queries and a visualization spec, not by a BI tool's internal ids (BI-P060). ` +
`Carry the id in "vendor_ref" if you need to round-trip it.`,
);
}
if (SQL_SMELLING_KEYS.has(key)) {
invalid(`"${key}" in ${what} is not part of the dashboard model — dashboards do not carry SQL`);
}
invalid(`unknown field "${key}" in ${what}`);
}
}
function asString(raw: unknown, what: string, max = MAX_NAME_LENGTH): string {
if (typeof raw !== 'string') invalid(`${what} must be a string`);
const v = raw.trim();
if (!v) invalid(`${what} must not be empty`);
if (v.length > max) invalid(`${what} exceeds ${max} characters`);
return v;
}
function asIdentifier(raw: unknown, what: string): string {
const v = asString(raw, what);
if (!IDENTIFIER.test(v)) invalid(`${what} is not a valid identifier`);
return v;
}
function asInteger(raw: unknown, what: string, min: number, max: number): number {
if (typeof raw !== 'number' || !Number.isFinite(raw)) invalid(`${what} must be a finite number`);
if (!Number.isInteger(raw)) invalid(`${what} must be an integer`);
if (raw < min || raw > max) invalid(`${what} must be between ${min} and ${max}`);
return raw;
}
function parseVendorRef(raw: unknown, what: string): VendorRef {
const obj = asObject(raw, what);
rejectUnknownKeys(obj, ['system', 'id'], what);
return { system: asString(obj.system, `${what}.system`), id: asString(obj.id, `${what}.id`, 256) };
}
// ── parsing ────────────────────────────────────────────────────────────────
function parseParameter(raw: unknown, i: number): DashboardParameter {
const what = `parameters[${i}]`;
const obj = asObject(raw, what);
rejectUnknownKeys(obj, ['name', 'type', 'required', 'default_value', 'allowed_values'], what);
const name = asIdentifier(obj.name, `${what}.name`);
const type = asString(obj.type, `${what}.type`);
if (!(PARAMETER_TYPES as readonly string[]).includes(type)) invalid(`${what}.type is not a parameter type`);
if (typeof obj.required !== 'boolean') invalid(`${what}.required must be a boolean`);
const p: DashboardParameter = { name, type: type as ParameterType, required: obj.required };
if (obj.default_value !== undefined)
p.default_value = asString(obj.default_value, `${what}.default_value`, 4096);
if (obj.allowed_values !== undefined) {
if (!Array.isArray(obj.allowed_values) || obj.allowed_values.length === 0)
invalid(`${what}.allowed_values must be a non-empty array`);
p.allowed_values = obj.allowed_values.map((v, j) => asString(v, `${what}.allowed_values[${j}]`, 4096));
}
if (p.type === 'enum' && !p.allowed_values) {
invalid(`${what} is an enum parameter with no allowed_values — an unbounded enum is a free-text field`);
}
if (p.required && p.default_value !== undefined) {
invalid(`${what} is required AND has a default — a default makes "required" unenforceable`);
}
return p;
}
function parseFilter(raw: unknown, i: number, parameterNames: Set<string>): DashboardFilter {
const what = `filters[${i}]`;
const obj = asObject(raw, what);
rejectUnknownKeys(obj, ['filter_id', 'label', 'dimension', 'operator', 'default_values', 'locked'], what);
const filter_id = asIdentifier(obj.filter_id, `${what}.filter_id`);
const label = asString(obj.label, `${what}.label`, MAX_TITLE_LENGTH);
const dimension = asString(obj.dimension, `${what}.dimension`);
const operator = asString(obj.operator, `${what}.operator`) as FilterOperator;
const f: DashboardFilter = { filter_id, label, dimension, operator };
if (obj.default_values !== undefined) {
if (!Array.isArray(obj.default_values)) invalid(`${what}.default_values must be an array`);
f.default_values = obj.default_values.map((v, j) => {
const s = asString(v, `${what}.default_values[${j}]`, 4096);
const m = PARAM_REFERENCE.exec(s);
if (m && !parameterNames.has(m[1])) {
invalid(`${what}.default_values[${j}] references undeclared parameter "${m[1]}"`);
}
return s;
});
}
if (obj.locked !== undefined) {
if (typeof obj.locked !== 'boolean') invalid(`${what}.locked must be a boolean`);
f.locked = obj.locked;
}
return f;
}
function parseVisualization(raw: unknown, what: string): VisualizationSpec {
const obj = asObject(raw, what);
rejectUnknownKeys(obj, ['type', 'encoding', 'options'], what);
const type = asString(obj.type, `${what}.type`);
if (!(VISUALIZATION_TYPES as readonly string[]).includes(type)) {
invalid(
`${what}.type "${type}" is not a SIAX visualization type — the canonical list is engine-neutral ` +
`so a chart survives a BI engine swap (BI-P060)`,
);
}
const encodingRaw = asObject(obj.encoding ?? {}, `${what}.encoding`);
rejectUnknownKeys(encodingRaw, ['x', 'y', 'series', 'value'], `${what}.encoding`);
const encoding: VisualizationSpec['encoding'] = {};
if (encodingRaw.x !== undefined) encoding.x = asString(encodingRaw.x, `${what}.encoding.x`);
if (encodingRaw.series !== undefined)
encoding.series = asString(encodingRaw.series, `${what}.encoding.series`);
if (encodingRaw.value !== undefined) encoding.value = asString(encodingRaw.value, `${what}.encoding.value`);
if (encodingRaw.y !== undefined) {
if (!Array.isArray(encodingRaw.y)) invalid(`${what}.encoding.y must be an array`);
encoding.y = encodingRaw.y.map((v, j) => asString(v, `${what}.encoding.y[${j}]`));
}
const spec: VisualizationSpec = { type: type as VisualizationType, encoding };
if (obj.options !== undefined) {
const options = asObject(obj.options, `${what}.options`);
const out: Record<string, string | number | boolean> = {};
for (const [k, v] of Object.entries(options)) {
if (!IDENTIFIER.test(k)) invalid(`${what}.options key "${k}" is not an identifier`);
if (typeof v === 'string') {
if (v.length > 256) invalid(`${what}.options.${k} exceeds 256 characters`);
out[k] = v;
} else if (typeof v === 'number') {
if (!Number.isFinite(v)) invalid(`${what}.options.${k} must be finite`);
out[k] = v;
} else if (typeof v === 'boolean') {
out[k] = v;
} else {
invalid(`${what}.options.${k} must be a string, number or boolean — options are not a payload`);
}
}
spec.options = out;
}
return spec;
}
function parseSource(raw: unknown, what: string): WidgetSource {
const obj = asObject(raw, what);
const kind = asString(obj.kind, `${what}.kind`);
if (kind === 'metric') {
rejectUnknownKeys(obj, ['kind', 'dataset', 'metric', 'dimensions', 'grain', 'limit'], what);
const source: MetricWidgetSource = {
kind: 'metric',
dataset: asString(obj.dataset, `${what}.dataset`),
metric: asString(obj.metric, `${what}.metric`),
};
if (obj.dimensions !== undefined) {
if (!Array.isArray(obj.dimensions)) invalid(`${what}.dimensions must be an array`);
source.dimensions = obj.dimensions.map((d, j) => asString(d, `${what}.dimensions[${j}]`));
}
if (obj.grain !== undefined) {
const grain = asString(obj.grain, `${what}.grain`);
if (!(TIME_GRAINS as readonly string[]).includes(grain)) invalid(`${what}.grain is not a time grain`);
source.grain = grain as TimeGrain;
}
if (obj.limit !== undefined) source.limit = asInteger(obj.limit, `${what}.limit`, 1, 100_000);
return source;
}
if (kind === 'query') {
rejectUnknownKeys(obj, ['kind', 'request'], what);
// parseAnalyticsQueryRequest is the SQL boundary: it is why a query widget
// cannot become an SQL widget by adding a field.
return { kind: 'query', request: parseAnalyticsQueryRequest(obj.request) };
}
invalid(
`${what}.kind must be "metric" or "query" — a widget with no reconstructable source cannot be ` +
`rendered without the tool that exported it (BI-P060 DoD)`,
);
}
function parseWidget(raw: unknown, i: number, gridColumns: number): DashboardWidget {
const what = `widgets[${i}]`;
const obj = asObject(raw, what);
rejectUnknownKeys(
obj,
['widget_id', 'title', 'source', 'visualization', 'layout', 'filter_bindings', 'vendor_ref'],
what,
);
const widget_id = asIdentifier(obj.widget_id, `${what}.widget_id`);
const title = asString(obj.title, `${what}.title`, MAX_TITLE_LENGTH);
const source = parseSource(obj.source, `${what}.source`);
const visualization = parseVisualization(obj.visualization, `${what}.visualization`);
const layoutObj = asObject(obj.layout, `${what}.layout`);
rejectUnknownKeys(layoutObj, ['x', 'y', 'w', 'h'], `${what}.layout`);
const layout: WidgetLayout = {
x: asInteger(layoutObj.x, `${what}.layout.x`, 0, gridColumns - 1),
y: asInteger(layoutObj.y, `${what}.layout.y`, 0, 10_000),
w: asInteger(layoutObj.w, `${what}.layout.w`, 1, gridColumns),
h: asInteger(layoutObj.h, `${what}.layout.h`, 1, 200),
};
if (layout.x + layout.w > gridColumns) {
invalid(`${what}.layout overflows the ${gridColumns}-column grid (x=${layout.x}, w=${layout.w})`);
}
const widget: DashboardWidget = { widget_id, title, source, visualization, layout };
if (obj.filter_bindings !== undefined) {
if (!Array.isArray(obj.filter_bindings)) invalid(`${what}.filter_bindings must be an array`);
widget.filter_bindings = obj.filter_bindings.map((b, j) =>
asIdentifier(b, `${what}.filter_bindings[${j}]`),
);
}
if (obj.vendor_ref !== undefined) widget.vendor_ref = parseVendorRef(obj.vendor_ref, `${what}.vendor_ref`);
return widget;
}
/**
* Parse an untrusted dashboard document. Fail-closed everywhere: a field this
* function does not understand is a refusal, because "ignored for now" is how
* a Superset-only document ends up stored as canonical truth.
*/
export function parseDashboardDocument(raw: unknown): DashboardDocument {
const obj = asObject(raw, 'dashboard document');
rejectUnknownKeys(obj, DASHBOARD_DOCUMENT_KEYS, 'dashboard document');
const id = asIdentifier(obj.id, 'id');
const title = asString(obj.title, 'title', MAX_TITLE_LENGTH);
const version = asInteger(obj.version, 'version', 1, 1_000_000);
const layoutObj = asObject(obj.layout, 'layout');
rejectUnknownKeys(layoutObj, ['grid_columns', 'row_height_px'], 'layout');
const layout = {
grid_columns: asInteger(layoutObj.grid_columns, 'layout.grid_columns', 1, MAX_GRID_COLUMNS),
row_height_px: asInteger(layoutObj.row_height_px, 'layout.row_height_px', 8, 400),
};
const rawParameters = obj.parameters ?? [];
if (!Array.isArray(rawParameters)) invalid('parameters must be an array');
if (rawParameters.length > MAX_PARAMETERS) invalid(`at most ${MAX_PARAMETERS} parameters`);
const parameters = rawParameters.map((p, i) => parseParameter(p, i));
const parameterNames = new Set(parameters.map((p) => p.name));
if (parameterNames.size !== parameters.length) invalid('parameters contains duplicate names');
const rawFilters = obj.filters ?? [];
if (!Array.isArray(rawFilters)) invalid('filters must be an array');
if (rawFilters.length > MAX_FILTERS) invalid(`at most ${MAX_FILTERS} filters`);
const filters = rawFilters.map((f, i) => parseFilter(f, i, parameterNames));
const filterIds = new Set(filters.map((f) => f.filter_id));
if (filterIds.size !== filters.length) invalid('filters contains duplicate filter_id');
if (!Array.isArray(obj.widgets)) invalid('widgets must be an array');
if (obj.widgets.length === 0) invalid('a dashboard with no widgets has nothing to reconstruct');
if (obj.widgets.length > MAX_WIDGETS) invalid(`at most ${MAX_WIDGETS} widgets`);
const widgets = obj.widgets.map((w, i) => parseWidget(w, i, layout.grid_columns));
if (new Set(widgets.map((w) => w.widget_id)).size !== widgets.length) {
invalid('widgets contains duplicate widget_id');
}
for (const w of widgets) {
for (const b of w.filter_bindings ?? []) {
if (!filterIds.has(b)) invalid(`widget "${w.widget_id}" binds undeclared filter "${b}"`);
}
}
const refreshObj = asObject(obj.refresh, 'refresh');
rejectUnknownKeys(refreshObj, ['mode', 'interval_seconds', 'cron', 'max_staleness_seconds'], 'refresh');
const mode = asString(refreshObj.mode, 'refresh.mode');
if (!(REFRESH_MODES as readonly string[]).includes(mode)) invalid('refresh.mode is not a refresh mode');
const refresh: DashboardRefresh = {
mode: mode as RefreshMode,
max_staleness_seconds: asInteger(
refreshObj.max_staleness_seconds,
'refresh.max_staleness_seconds',
1,
MAX_STALENESS_SECONDS,
),
};
if (refresh.mode === 'interval') {
refresh.interval_seconds = asInteger(
refreshObj.interval_seconds,
'refresh.interval_seconds',
MIN_REFRESH_INTERVAL_SECONDS,
86_400,
);
if (refreshObj.cron !== undefined) invalid('refresh.cron is meaningless when mode is "interval"');
} else if (refresh.mode === 'scheduled') {
refresh.cron = asString(refreshObj.cron, 'refresh.cron', 128);
if (refreshObj.interval_seconds !== undefined) {
invalid('refresh.interval_seconds is meaningless when mode is "scheduled"');
}
} else if (refreshObj.interval_seconds !== undefined || refreshObj.cron !== undefined) {
invalid('refresh.mode is "manual" but a schedule was supplied');
}
const permObj = asObject(obj.permissions, 'permissions');
rejectUnknownKeys(permObj, ['visibility', 'roles', 'required_entitlement'], 'permissions');
const visibility = asString(permObj.visibility, 'permissions.visibility');
if (!(DASHBOARD_VISIBILITIES as readonly string[]).includes(visibility)) {
invalid('permissions.visibility is not a visibility');
}
const permissions: DashboardPermissions = { visibility: visibility as DashboardVisibility };
if (permObj.roles !== undefined) {
if (!Array.isArray(permObj.roles)) invalid('permissions.roles must be an array');
permissions.roles = permObj.roles.map((r, i) => asIdentifier(r, `permissions.roles[${i}]`));
}
if (permissions.visibility === 'roles' && (permissions.roles ?? []).length === 0) {
invalid(
'permissions.visibility is "roles" with no roles — an empty role list reads as "deny all" to one ' +
'implementation and "allow all" to the next; say which',
);
}
if (permObj.required_entitlement !== undefined) {
permissions.required_entitlement = asIdentifier(
permObj.required_entitlement,
'permissions.required_entitlement',
);
}
const doc: DashboardDocument = {
id,
title,
layout,
filters,
widgets,
parameters,
refresh,
permissions,
version,
};
if (obj.description !== undefined) doc.description = asString(obj.description, 'description', 2000);
if (obj.vendor_ref !== undefined) doc.vendor_ref = parseVendorRef(obj.vendor_ref, 'vendor_ref');
return doc;
}
// ── canonical hash ─────────────────────────────────────────────────────────
/**
* Stable ordering of every field that MEANS something, with `vendor_ref`
* omitted at both levels. Two documents that render identically hash
* identically even if their JSON key order differs or one of them has been
* round-tripped through a BI tool; a document whose meaning changed hashes
* differently. That is what makes `DashboardVersion.layout_hash` a usable
* identity for BI-P061 rollback.
*/
function canonicalize(doc: DashboardDocument): unknown {
return [
doc.id,
doc.title,
doc.description ?? '',
doc.version,
[doc.layout.grid_columns, doc.layout.row_height_px],
doc.parameters
.map((p) => [p.name, p.type, p.required, p.default_value ?? '', [...(p.allowed_values ?? [])].sort()])
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
doc.filters
.map((f) => [f.filter_id, f.label, f.dimension, f.operator, f.default_values ?? [], f.locked ?? false])
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
doc.widgets
.map((w) => [
w.widget_id,
w.title,
w.source.kind === 'metric'
? [
'metric',
w.source.dataset,
w.source.metric,
w.source.dimensions ?? [],
w.source.grain ?? '',
w.source.limit ?? 0,
]
: ['query', canonicalRequest(w.source.request)],
[
w.visualization.type,
w.visualization.encoding.x ?? '',
w.visualization.encoding.y ?? [],
w.visualization.encoding.series ?? '',
w.visualization.encoding.value ?? '',
Object.keys(w.visualization.options ?? {})
.sort()
.map((k) => [k, (w.visualization.options ?? {})[k]]),
],
[w.layout.x, w.layout.y, w.layout.w, w.layout.h],
[...(w.filter_bindings ?? [])].sort(),
])
.sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
[
doc.refresh.mode,
doc.refresh.interval_seconds ?? 0,
doc.refresh.cron ?? '',
doc.refresh.max_staleness_seconds,
],
[
doc.permissions.visibility,
[...(doc.permissions.roles ?? [])].sort(),
doc.permissions.required_entitlement ?? '',
],
];
}
function canonicalRequest(r: AnalyticsQueryRequest): unknown {
return [
r.dataset,
[...r.metrics],
[...r.dimensions],
r.filters.map((f) => [f.dimension, f.operator, f.values ?? []]),
r.timeRange
? [r.timeRange.dimension ?? '', r.timeRange.from ?? '', r.timeRange.to ?? '', r.timeRange.grain ?? '']
: [],
(r.orderBy ?? []).map((o) => [o.ref, o.direction]),
r.limit ?? 0,
r.format ?? 'json',
];
}
/** sha256 of the canonical form. Fills `DashboardVersion.layout_hash`. */
export function dashboardDocumentHash(doc: DashboardDocument): string {
return crypto
.createHash('sha256')
.update(JSON.stringify(canonicalize(doc)))
.digest('hex');
}
// ── reconstruction (the DoD) ───────────────────────────────────────────────
export interface PlannedWidget {
widget_id: string;
title: string;
/** Runnable, contract-valid, engine-neutral. This is the DoD in one field. */
request: AnalyticsQueryRequest;
visualization: VisualizationSpec;
layout: WidgetLayout;
}
export interface DashboardPlan {
dashboard_id: string;
title: string;
document_hash: string;
grid_columns: number;
widgets: PlannedWidget[];
/** Parameter values actually bound, after defaults. Reported, not implied. */
bound_parameters: Record<string, string>;
}
export interface PlanOptions {
/** Caller-supplied parameter values. Validated against the declarations. */
parameters?: Record<string, string>;
/** Filter ids a viewer switched off. `locked` filters cannot be switched off. */
disabled_filter_ids?: string[];
}
function bindParameters(doc: DashboardDocument, supplied: Record<string, string>): Record<string, string> {
const bound: Record<string, string> = {};
const declared = new Set(doc.parameters.map((p) => p.name));
for (const name of Object.keys(supplied)) {
if (!declared.has(name)) invalid(`parameter "${name}" is not declared by this dashboard`);
}
for (const p of doc.parameters) {
const value = supplied[p.name] ?? p.default_value;
if (value === undefined) {
if (p.required) invalid(`required parameter "${p.name}" was not supplied`);
continue;
}
if (p.allowed_values && !p.allowed_values.includes(value)) {
invalid(`parameter "${p.name}" value is not in its allowed_values`);
}
if (p.type === 'number' && !Number.isFinite(Number(value))) {
invalid(`parameter "${p.name}" is declared number but "${value}" is not numeric`);
}
if (p.type === 'date' && Number.isNaN(Date.parse(value))) {
invalid(`parameter "${p.name}" is declared date but "${value}" is not an ISO date`);
}
bound[p.name] = value;
}
return bound;
}
function resolveFilterValues(f: DashboardFilter, bound: Record<string, string>): string[] | undefined {
if (!f.default_values) return undefined;
return f.default_values.map((v) => {
const m = PARAM_REFERENCE.exec(v);
if (!m) return v;
const value = bound[m[1]];
if (value === undefined) {
invalid(`filter "${f.filter_id}" references parameter "${m[1]}", which has no value`);
}
return value;
});
}
/**
* Turn a dashboard document into runnable analytics requests.
*
* This IS BI-P060's DoD. If this function returns a plan, the dashboard can be
* rendered by anything that can execute `AnalyticsQueryRequest` and draw the
* twelve visualization types — Superset not required, not consulted, and not
* present anywhere in the output.
*/
export function planDashboard(doc: DashboardDocument, options: PlanOptions = {}): DashboardPlan {
const bound = bindParameters(doc, options.parameters ?? {});
const disabled = new Set(options.disabled_filter_ids ?? []);
for (const f of doc.filters) {
if (disabled.has(f.filter_id) && f.locked) {
invalid(
`filter "${f.filter_id}" is locked and cannot be disabled — locked filters are how a saved ` +
`dashboard pins its scope, and a viewer removing one silently widens the result set`,
);
}
}
const widgets = doc.widgets.map((w) => {
const applicable = doc.filters.filter(
(f) =>
!disabled.has(f.filter_id) &&
(w.filter_bindings === undefined || w.filter_bindings.length === 0
? true
: w.filter_bindings.includes(f.filter_id)),
);
const dashboardFilters: AnalyticsFilter[] = applicable.map((f) => {
const values = resolveFilterValues(f, bound);
const af: AnalyticsFilter = { dimension: f.dimension, operator: f.operator };
if (values !== undefined) af.values = values;
return af;
});
const base =
w.source.kind === 'metric'
? {
dataset: w.source.dataset,
metrics: [w.source.metric],
dimensions: w.source.dimensions ?? [],
filters: [] as AnalyticsFilter[],
...(w.source.grain ? { timeRange: { grain: w.source.grain } } : {}),
...(w.source.limit ? { limit: w.source.limit } : {}),
}
: {
...w.source.request,
timeRange: w.source.request.timeRange,
};
// Re-parse the merged request rather than trusting the merge: the filters
// we just added come from the document, and a document is untrusted input
// even after it has been stored.
const request = parseAnalyticsQueryRequest({
...base,
filters: [...(w.source.kind === 'query' ? w.source.request.filters : []), ...dashboardFilters],
});
return {
widget_id: w.widget_id,
title: w.title,
request,
visualization: w.visualization,
layout: w.layout,
};
});
return {
dashboard_id: doc.id,
title: doc.title,
document_hash: dashboardDocumentHash(doc),
grid_columns: doc.layout.grid_columns,
widgets,
bound_parameters: bound,
};
}
@@ -0,0 +1,449 @@
/**
* src/analytics/dashboards/dashboard-release.ts
* ==============================================
* BI-P061 — DASHBOARD VERSIONING.
*
* DRAFT -> PUBLISHED -> SUPERSEDED, and the DoD: "rollback to previous
* dashboard definition".
*
* Three decisions worth stating, because each one is a bug someone would
* otherwise write:
*
* 1. ROLLBACK MINTS A NEW VERSION. It does not resurrect the old release row
* and it does not edit the published one. "We went back to v3" is then a
* fact with a timestamp (v5 = rollback_of_version 3) instead of a hole in
* the history where v4 used to be. `verifyRollback()` proves the restore is
* faithful by comparing BI-P060 canonical hashes, so a rollback that
* silently restored a different document fails loudly.
*
* 2. THE DOCUMENT IS FROZEN, THE RELEASE MOVES. sql/008 makes
* DashboardVersion rows immutable at the database. The release ledger
* (sql/010) carries the state, and its trigger allows exactly two
* transitions and freezes every other column. Un-publishing is refused:
* it is how a rollback gets faked without leaving a trace.
*
* 3. THE APPLICATION CHECK IS NOT THE GUARANTEE. `publish()` checks for an
* existing published release, and that check is a read-then-write race.
* The partial unique index in sql/010 is what actually holds under
* concurrency; the realinfra test drops it, runs the race and shows two
* published releases for one dashboard, so nobody has to take that on
* faith. This module maps the resulting constraint violation onto a
* CONFLICT rather than a 500.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import { dashboardDocumentHash, parseDashboardDocument, type DashboardDocument } from './dashboard-document';
export const DASHBOARD_RELEASE_STATES = ['draft', 'published', 'superseded'] as const;
export type DashboardReleaseState = (typeof DASHBOARD_RELEASE_STATES)[number];
/** The only transitions that exist. Everything else is a refusal, not a no-op. */
const LEGAL_TRANSITIONS: Readonly<Record<DashboardReleaseState, readonly DashboardReleaseState[]>> = {
draft: ['published'],
published: ['superseded'],
superseded: [],
};
export function isLegalTransition(from: DashboardReleaseState, to: DashboardReleaseState): boolean {
return LEGAL_TRANSITIONS[from].includes(to);
}
export interface DashboardRelease {
release_id: string;
dashboard_id: string;
dashboard_version_id: string;
tenant_id: string;
project_id: string;
version: number;
state: DashboardReleaseState;
/** BI-P060 canonical hash of the released document. */
document_hash: string;
/** Set when this release restores an earlier version. */
rollback_of_version?: number;
created_by?: string;
created_at: string;
published_at?: string;
superseded_at?: string;
superseded_by_release_id?: string;
}
/** The immutable frozen document behind a release (sql/008's DashboardVersion). */
export interface DashboardVersionRecord {
dashboard_version_id: string;
dashboard_id: string;
tenant_id: string;
project_id: string;
version: number;
layout_json: string;
layout_hash: string;
created_at: string;
created_by?: string;
}
export interface TenantScope {
tenant_id: string;
project_id: string;
}
/**
* The persistence port. Deliberately narrow, and every method is tenant-scoped
* — there is no signature that lets a caller forget the tenant.
*/
export interface DashboardReleaseStore {
/** Highest version number ever minted for this dashboard, or 0. */
maxVersion(scope: TenantScope, dashboardId: string): Promise<number>;
insertVersionAndRelease(version: DashboardVersionRecord, release: DashboardRelease): Promise<void>;
getRelease(scope: TenantScope, releaseId: string): Promise<DashboardRelease | undefined>;
getVersion(scope: TenantScope, dashboardVersionId: string): Promise<DashboardVersionRecord | undefined>;
findVersionByNumber(
scope: TenantScope,
dashboardId: string,
version: number,
): Promise<DashboardVersionRecord | undefined>;
findByState(
scope: TenantScope,
dashboardId: string,
state: DashboardReleaseState,
): Promise<DashboardRelease | undefined>;
listReleases(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]>;
/**
* Publish `releaseId` and supersede `supersedes` in ONE transaction. Both or
* neither: a supersede that lands without its publish leaves a dashboard
* with no live version, and a publish that lands without its supersede
* leaves two.
*/
publishAtomically(args: {
scope: TenantScope;
releaseId: string;
publishedAt: string;
supersedes?: { release_id: string; superseded_at: string };
}): Promise<void>;
}
function conflict(detail: string): never {
throw new SiaxError('CONFLICT', '0pt.analytics.dashboard', detail);
}
function notFound(detail: string): never {
throw new SiaxError('NOT_FOUND', '0pt.analytics.dashboard', detail);
}
function assertScope(scope: TenantScope): void {
if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.dashboard',
'dashboard releases require a verified tenant_id + project_id',
);
}
}
export interface DraftResult {
release: DashboardRelease;
version: DashboardVersionRecord;
}
/**
* The lifecycle. Stateless apart from its store and its clock; the clock is
* injectable so a test can pin timestamps instead of sleeping.
*/
export class DashboardReleaseService {
constructor(
private readonly store: DashboardReleaseStore,
private readonly now: () => Date = () => new Date(),
private readonly newId: () => string = () => crypto.randomUUID(),
) {}
/**
* Freeze a document as the next version and open a DRAFT release for it.
* The document is re-parsed here even if the caller already parsed it: what
* gets frozen must be what the model accepts, not what a caller assembled.
*/
async draft(args: {
scope: TenantScope;
dashboardId: string;
document: unknown;
createdBy?: string;
rollbackOfVersion?: number;
}): Promise<DraftResult> {
assertScope(args.scope);
const document = parseDashboardDocument(args.document);
const existingDraft = await this.store.findByState(args.scope, args.dashboardId, 'draft');
if (existingDraft) {
conflict(
`dashboard "${args.dashboardId}" already has an open draft (version ${existingDraft.version}); ` +
`publish or abandon it before opening another — two drafts race to publish and the loser's ` +
`edits vanish`,
);
}
const next = (await this.store.maxVersion(args.scope, args.dashboardId)) + 1;
const at = this.now().toISOString();
const hash = dashboardDocumentHash(document);
const version: DashboardVersionRecord = {
dashboard_version_id: this.newId(),
dashboard_id: args.dashboardId,
tenant_id: args.scope.tenant_id,
project_id: args.scope.project_id,
version: next,
layout_json: JSON.stringify(document),
layout_hash: hash,
created_at: at,
...(args.createdBy ? { created_by: args.createdBy } : {}),
};
const release: DashboardRelease = {
release_id: this.newId(),
dashboard_id: args.dashboardId,
dashboard_version_id: version.dashboard_version_id,
tenant_id: args.scope.tenant_id,
project_id: args.scope.project_id,
version: next,
state: 'draft',
document_hash: hash,
created_at: at,
...(args.createdBy ? { created_by: args.createdBy } : {}),
...(args.rollbackOfVersion ? { rollback_of_version: args.rollbackOfVersion } : {}),
};
await this.store.insertVersionAndRelease(version, release);
return { release, version };
}
/**
* DRAFT -> PUBLISHED, superseding whatever was live, in one transaction.
* The pre-check exists to return a friendly CONFLICT; the guarantee is the
* partial unique index (sql/010).
*/
async publish(args: {
scope: TenantScope;
releaseId: string;
}): Promise<{ published: DashboardRelease; superseded?: DashboardRelease }> {
assertScope(args.scope);
const release = await this.store.getRelease(args.scope, args.releaseId);
if (!release) notFound(`dashboard release "${args.releaseId}" does not exist in this tenant`);
if (release.state !== 'draft') {
conflict(
`release "${args.releaseId}" is ${release.state}, not draft — ${release.state} -> published is ` +
`not a legal transition (BI-P061)`,
);
}
const live = await this.store.findByState(args.scope, release.dashboard_id, 'published');
const at = this.now().toISOString();
await this.store.publishAtomically({
scope: args.scope,
releaseId: release.release_id,
publishedAt: at,
...(live ? { supersedes: { release_id: live.release_id, superseded_at: at } } : {}),
});
return {
published: { ...release, state: 'published', published_at: at },
...(live
? {
superseded: {
...live,
state: 'superseded',
superseded_at: at,
superseded_by_release_id: release.release_id,
},
}
: {}),
};
}
/** The release currently serving, or undefined if the dashboard has never been published. */
async live(scope: TenantScope, dashboardId: string): Promise<DashboardRelease | undefined> {
assertScope(scope);
return this.store.findByState(scope, dashboardId, 'published');
}
/** The frozen document behind a release, re-parsed on the way out. */
async documentOf(scope: TenantScope, release: DashboardRelease): Promise<DashboardDocument> {
assertScope(scope);
const version = await this.store.getVersion(scope, release.dashboard_version_id);
if (!version) notFound(`dashboard version "${release.dashboard_version_id}" is missing`);
const document = parseDashboardDocument(JSON.parse(version.layout_json));
const hash = dashboardDocumentHash(document);
if (hash !== release.document_hash) {
// Storage returned something other than what was released. Refusing is
// the only safe answer: a dashboard that renders a document nobody
// approved is worse than a dashboard that will not open.
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dashboard',
`stored document for release "${release.release_id}" hashes to ${hash}, not the released ` +
`${release.document_hash} — refusing to serve an unapproved definition`,
);
}
return document;
}
/**
* BI-P061's DoD. Restores version `toVersion` by minting a NEW version with
* the SAME document and publishing it. Returns the new release plus the
* verification that the restore is faithful.
*/
async rollback(args: {
scope: TenantScope;
dashboardId: string;
toVersion: number;
createdBy?: string;
}): Promise<{ release: DashboardRelease; restored_from: number; document_hash: string }> {
assertScope(args.scope);
const target = await this.store.findVersionByNumber(args.scope, args.dashboardId, args.toVersion);
if (!target) {
notFound(`dashboard "${args.dashboardId}" has no version ${args.toVersion} in this tenant`);
}
const live = await this.store.findByState(args.scope, args.dashboardId, 'published');
if (live && live.version === args.toVersion) {
conflict(`version ${args.toVersion} is already live; a rollback to the live version is a no-op`);
}
const document = parseDashboardDocument(JSON.parse(target.layout_json));
const hash = dashboardDocumentHash(document);
if (hash !== target.layout_hash) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dashboard',
`version ${args.toVersion} hashes to ${hash} but was stored as ${target.layout_hash} — ` +
`refusing to roll back to a definition that has changed underneath us`,
);
}
const { release } = await this.draft({
scope: args.scope,
dashboardId: args.dashboardId,
document,
...(args.createdBy ? { createdBy: args.createdBy } : {}),
rollbackOfVersion: args.toVersion,
});
const { published } = await this.publish({ scope: args.scope, releaseId: release.release_id });
if (published.document_hash !== target.layout_hash) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dashboard',
`rollback published ${published.document_hash}, not the requested ${target.layout_hash}`,
);
}
return { release: published, restored_from: args.toVersion, document_hash: published.document_hash };
}
/** Full ledger, oldest first. A rollback is an entry here, never a deletion. */
async history(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]> {
assertScope(scope);
const rows = await this.store.listReleases(scope, dashboardId);
return [...rows].sort((a, b) => a.version - b.version);
}
}
/**
* In-memory store. Used by the unit tests, and it enforces the same two
* uniqueness rules the SQL does — a store that is laxer than the database
* makes the unit tests pass on states production can never reach.
*/
export class InMemoryDashboardReleaseStore implements DashboardReleaseStore {
private readonly versions: DashboardVersionRecord[] = [];
private readonly releases: DashboardRelease[] = [];
private scoped<T extends { tenant_id: string; project_id: string }>(scope: TenantScope, rows: T[]): T[] {
return rows.filter((r) => r.tenant_id === scope.tenant_id && r.project_id === scope.project_id);
}
async maxVersion(scope: TenantScope, dashboardId: string): Promise<number> {
return this.scoped(scope, this.versions)
.filter((v) => v.dashboard_id === dashboardId)
.reduce((max, v) => Math.max(max, v.version), 0);
}
async insertVersionAndRelease(version: DashboardVersionRecord, release: DashboardRelease): Promise<void> {
const scope = { tenant_id: release.tenant_id, project_id: release.project_id };
if (
release.state === 'draft' &&
this.scoped(scope, this.releases).some(
(r) => r.dashboard_id === release.dashboard_id && r.state === 'draft',
)
) {
conflict('one open draft per dashboard');
}
if (this.versions.some((v) => v.dashboard_id === version.dashboard_id && v.version === version.version)) {
conflict(`version ${version.version} already exists for "${version.dashboard_id}"`);
}
this.versions.push({ ...version });
this.releases.push({ ...release });
}
async getRelease(scope: TenantScope, releaseId: string): Promise<DashboardRelease | undefined> {
const r = this.scoped(scope, this.releases).find((x) => x.release_id === releaseId);
return r ? { ...r } : undefined;
}
async getVersion(scope: TenantScope, id: string): Promise<DashboardVersionRecord | undefined> {
const v = this.scoped(scope, this.versions).find((x) => x.dashboard_version_id === id);
return v ? { ...v } : undefined;
}
async findVersionByNumber(
scope: TenantScope,
dashboardId: string,
version: number,
): Promise<DashboardVersionRecord | undefined> {
const v = this.scoped(scope, this.versions).find(
(x) => x.dashboard_id === dashboardId && x.version === version,
);
return v ? { ...v } : undefined;
}
async findByState(
scope: TenantScope,
dashboardId: string,
state: DashboardReleaseState,
): Promise<DashboardRelease | undefined> {
const r = this.scoped(scope, this.releases).find(
(x) => x.dashboard_id === dashboardId && x.state === state,
);
return r ? { ...r } : undefined;
}
async listReleases(scope: TenantScope, dashboardId: string): Promise<DashboardRelease[]> {
return this.scoped(scope, this.releases)
.filter((r) => r.dashboard_id === dashboardId)
.map((r) => ({ ...r }));
}
async publishAtomically(args: {
scope: TenantScope;
releaseId: string;
publishedAt: string;
supersedes?: { release_id: string; superseded_at: string };
}): Promise<void> {
const target = this.releases.find(
(r) =>
r.release_id === args.releaseId &&
r.tenant_id === args.scope.tenant_id &&
r.project_id === args.scope.project_id,
);
if (!target) notFound(`release "${args.releaseId}" not found`);
if (!isLegalTransition(target.state, 'published')) {
conflict(`illegal transition ${target.state} -> published`);
}
const alreadyLive = this.releases.find(
(r) =>
r.dashboard_id === target.dashboard_id &&
r.state === 'published' &&
r.release_id !== args.supersedes?.release_id,
);
if (alreadyLive) conflict('one published release per dashboard');
if (args.supersedes) {
const live = this.releases.find((r) => r.release_id === args.supersedes?.release_id);
if (!live) notFound(`release "${args.supersedes.release_id}" not found`);
if (!isLegalTransition(live.state, 'superseded')) {
conflict(`illegal transition ${live.state} -> superseded`);
}
live.state = 'superseded';
live.superseded_at = args.supersedes.superseded_at;
live.superseded_by_release_id = target.release_id;
}
target.state = 'published';
target.published_at = args.publishedAt;
}
}
+474
View File
@@ -0,0 +1,474 @@
/**
* src/analytics/domain/contract.ts
* =================================
* BI-P004 — ANALYTICS DATA CONTRACT.
*
* `DatasetContract` is the machine-checkable promise a dataset makes to every
* consumer downstream of it. BI-P004's DoD is a GATE, not a document:
*
* "Breaking dataset schema changes fail compatibility gate unless
* explicitly versioned."
*
* so `checkContractCompatibility` is the deliverable and the prose is the
* commentary. The gate is fail-closed: an unrecognised change is BREAKING,
* never "probably fine".
*
* Versioning is semver-shaped (`major.minor.patch`) with one rule that is not
* negotiable: a BREAKING change requires a MAJOR bump. A breaking change
* shipped under a minor bump is exactly the failure this gate exists to
* catch, so it is reported as a violation rather than tolerated.
*/
import crypto from "node:crypto";
import { SiaxError } from "../../platform/errors";
import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities";
import type { AnalyticsTier } from "./tiers";
export interface ContractField {
name: string;
type: AnalyticsFieldType;
nullable: boolean;
role: FieldRole;
/** Part of the dataset's primary identifier. */
primary_identifier?: boolean;
classification?: PiiClassification;
description?: string;
}
export interface ContractQualityRule {
rule_id: string;
kind: string;
field_name?: string;
severity: "warn" | "block";
}
/** BI-P004's named contents, one field each. */
export interface DatasetContract {
dataset_id: string;
/** semver `major.minor.patch`. */
version: string;
tier: AnalyticsTier;
fields: ContractField[];
/** Field names forming the primary identifier, in order. */
primary_identifiers: string[];
/** Field names usable as dimensions (subset of `fields`). */
dimensions: string[];
/** Field names usable as measures (subset of `fields`). */
measures: string[];
/** Freshness SLO in seconds. */
freshness_slo_seconds: number;
retention_days: number;
/** Dataset-level PII classification (the max of its fields, at least). */
pii_classification: PiiClassification;
/** Upstream refs: source ids, dataset ids, external system refs. */
source_lineage: string[];
quality_rules: ContractQualityRule[];
/** GDPR purpose limitation — what this data may be USED for. */
allowed_purposes: string[];
}
export interface SemVer {
major: number;
minor: number;
patch: number;
}
const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/;
export function parseSemVer(v: string): SemVer {
const m = SEMVER_RE.exec(v);
if (!m) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`);
}
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) };
}
// ── change classification ──────────────────────────────────────────────────
export type ChangeSeverity = "compatible" | "breaking";
export interface ContractChange {
severity: ChangeSeverity;
/** Stable machine code so callers can allow-list specific changes. */
code: string;
field?: string;
detail: string;
}
/**
* Type widening that does NOT break a reader. Deliberately conservative:
* int64 -> float64 loses exactness for large integers, so it is NOT here.
* Everything absent from this table is breaking.
*/
const SAFE_TYPE_WIDENING: Partial<Record<AnalyticsFieldType, readonly AnalyticsFieldType[]>> = {
date: ["timestamp"],
};
function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean {
if (from === to) return true;
return (SAFE_TYPE_WIDENING[from] ?? []).includes(to);
}
/**
* Diff two contracts for the SAME dataset and classify every change.
*
* Breaking (a reader written against `previous` can now fail or silently
* change meaning):
* - field removed
* - field type changed other than a listed safe widening
* - nullable false -> true (readers may not handle nulls)
* - a field's role changes (a measure becoming a dimension changes meaning)
* - primary identifier set changes
* - a field leaves `measures` or `dimensions`
* - freshness SLO loosened (a consumer's own SLO may depend on it)
* - retention shortened (history a consumer relied on disappears)
* - an allowed purpose is removed (a legitimate consumer becomes illegitimate)
* - PII classification raised (consumers may lack clearance)
* - a blocking quality rule is removed or downgraded to warn
*
* Compatible: additive nullable fields, new dimensions/measures, new quality
* rules, loosened PII classification, added purposes, longer retention,
* tightened freshness.
*/
export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] {
if (previous.dataset_id !== next.dataset_id) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
"contract compatibility is only defined between versions of the SAME dataset",
);
}
const changes: ContractChange[] = [];
const prevFields = new Map(previous.fields.map((f) => [f.name, f]));
const nextFields = new Map(next.fields.map((f) => [f.name, f]));
for (const [name, pf] of prevFields) {
const nf = nextFields.get(name);
if (!nf) {
changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` });
continue;
}
if (!typeChangeIsCompatible(pf.type, nf.type)) {
changes.push({
severity: "breaking",
code: "field_type_changed",
field: name,
detail: `field "${name}" type ${pf.type} -> ${nf.type}`,
});
} else if (pf.type !== nf.type) {
changes.push({
severity: "compatible",
code: "field_type_widened",
field: name,
detail: `field "${name}" widened ${pf.type} -> ${nf.type}`,
});
}
if (!pf.nullable && nf.nullable) {
changes.push({
severity: "breaking",
code: "field_became_nullable",
field: name,
detail: `field "${name}" became nullable`,
});
}
if (pf.nullable && !nf.nullable) {
changes.push({
severity: "compatible",
code: "field_became_required",
field: name,
detail: `field "${name}" is no longer nullable`,
});
}
if (pf.role !== nf.role) {
changes.push({
severity: "breaking",
code: "field_role_changed",
field: name,
detail: `field "${name}" role ${pf.role} -> ${nf.role}`,
});
}
}
for (const [name, nf] of nextFields) {
if (prevFields.has(name)) continue;
if (nf.nullable) {
changes.push({
severity: "compatible",
code: "nullable_field_added",
field: name,
detail: `nullable field "${name}" added`,
});
} else {
// A required new field breaks producers, and breaks readers that
// reconstruct rows (they now have no value to supply).
changes.push({
severity: "breaking",
code: "required_field_added",
field: name,
detail: `required (non-nullable) field "${name}" added`,
});
}
}
const idsEqual =
previous.primary_identifiers.length === next.primary_identifiers.length &&
previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v);
if (!idsEqual) {
changes.push({
severity: "breaking",
code: "primary_identifier_changed",
detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`,
});
}
for (const d of previous.dimensions) {
if (!next.dimensions.includes(d)) {
changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` });
}
}
for (const d of next.dimensions) {
if (!previous.dimensions.includes(d)) {
changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` });
}
}
for (const m of previous.measures) {
if (!next.measures.includes(m)) {
changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` });
}
}
for (const m of next.measures) {
if (!previous.measures.includes(m)) {
changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` });
}
}
if (next.freshness_slo_seconds > previous.freshness_slo_seconds) {
changes.push({
severity: "breaking",
code: "freshness_slo_loosened",
detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`,
});
}
if (next.retention_days < previous.retention_days) {
changes.push({
severity: "breaking",
code: "retention_shortened",
detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`,
});
}
for (const p of previous.allowed_purposes) {
if (!next.allowed_purposes.includes(p)) {
changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` });
}
}
const PII_RANK: Record<PiiClassification, number> = {
none: 0,
pseudonymous: 1,
personal: 2,
sensitive: 3,
};
if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) {
changes.push({
severity: "breaking",
code: "pii_classification_raised",
detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`,
});
}
const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r]));
for (const [id, pr] of prevRules) {
const nr = next.quality_rules.find((r) => r.rule_id === id);
if (!nr && pr.severity === "block") {
changes.push({
severity: "breaking",
code: "blocking_quality_rule_removed",
detail: `blocking quality rule "${id}" removed`,
});
} else if (nr && pr.severity === "block" && nr.severity === "warn") {
changes.push({
severity: "breaking",
code: "blocking_quality_rule_downgraded",
detail: `quality rule "${id}" downgraded block -> warn`,
});
}
}
if (previous.tier !== next.tier) {
changes.push({
severity: "breaking",
code: "tier_changed",
detail: `tier ${previous.tier} -> ${next.tier}`,
});
}
return changes;
}
export interface CompatibilityVerdict {
compatible: boolean;
/** True when a MAJOR bump legitimises the breaking changes. */
explicitly_versioned: boolean;
changes: ContractChange[];
breaking: ContractChange[];
previous_version: string;
next_version: string;
/** Human-readable reason when `compatible` is false. */
reason?: string;
}
/**
* BI-P004's gate. Returns a verdict; `assertContractCompatible` throws.
*
* Rules, in order:
* 1. the version must move FORWARD (equal or lower is rejected outright);
* 2. breaking changes REQUIRE a major bump — that is what "explicitly
* versioned" means;
* 3. a major bump with no breaking change is allowed (over-declaring is
* safe; under-declaring is what we refuse).
*/
export function checkContractCompatibility(
previous: DatasetContract,
next: DatasetContract,
): CompatibilityVerdict {
const pv = parseSemVer(previous.version);
const nv = parseSemVer(next.version);
const changes = diffContracts(previous, next);
const breaking = changes.filter((c) => c.severity === "breaking");
const forward =
nv.major > pv.major ||
(nv.major === pv.major && nv.minor > pv.minor) ||
(nv.major === pv.major && nv.minor === pv.minor && nv.patch > pv.patch);
if (!forward) {
return {
compatible: false,
explicitly_versioned: false,
changes,
breaking,
previous_version: previous.version,
next_version: next.version,
reason: `version must move forward: ${previous.version} -> ${next.version}`,
};
}
const majorBumped = nv.major > pv.major;
if (breaking.length > 0 && !majorBumped) {
return {
compatible: false,
explicitly_versioned: false,
changes,
breaking,
previous_version: previous.version,
next_version: next.version,
reason:
`${breaking.length} breaking change(s) require a MAJOR version bump ` +
`(${previous.version} -> ${next.version} is not one): ` +
breaking.map((b) => b.code).join(", "),
};
}
return {
compatible: true,
explicitly_versioned: majorBumped && breaking.length > 0,
changes,
breaking,
previous_version: previous.version,
next_version: next.version,
};
}
/** Throwing form for call sites that treat incompatibility as a failure. */
export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict {
const verdict = checkContractCompatibility(previous, next);
if (!verdict.compatible) {
throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed");
}
return verdict;
}
/**
* Structural validation of a single contract (independent of history).
* Catches the contract that is internally inconsistent before it is ever
* compared to anything.
*/
export function validateContract(contract: DatasetContract): void {
parseSemVer(contract.version);
if (!contract.dataset_id.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id");
}
if (contract.fields.length === 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`);
}
const names = new Set<string>();
for (const f of contract.fields) {
if (names.has(f.name)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`);
}
names.add(f.name);
}
for (const label of ["primary_identifiers", "dimensions", "measures"] as const) {
for (const n of contract[label]) {
if (!names.has(n)) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`${label} references unknown field "${n}" in ${contract.dataset_id}`,
);
}
}
}
if (contract.primary_identifiers.length === 0) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`contract ${contract.dataset_id} declares no primary identifier`,
);
}
if (contract.allowed_purposes.length === 0) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`,
);
}
if (contract.freshness_slo_seconds <= 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive");
}
if (contract.retention_days < 0) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative");
}
}
/**
* Stable hash of a contract's MEANING (field order-insensitive), used as
* `DatasetVersion.contract_hash` so two versions that differ only in comment
* text do not look like schema drift.
*/
export function contractHash(contract: DatasetContract): string {
const canonical = {
dataset_id: contract.dataset_id,
version: contract.version,
tier: contract.tier,
fields: [...contract.fields]
.map((f) => ({
name: f.name,
type: f.type,
nullable: f.nullable,
role: f.role,
primary_identifier: f.primary_identifier ?? false,
classification: f.classification ?? "none",
}))
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
primary_identifiers: contract.primary_identifiers,
dimensions: [...contract.dimensions].sort(),
measures: [...contract.measures].sort(),
freshness_slo_seconds: contract.freshness_slo_seconds,
retention_days: contract.retention_days,
pii_classification: contract.pii_classification,
source_lineage: [...contract.source_lineage].sort(),
quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)),
allowed_purposes: [...contract.allowed_purposes].sort(),
};
return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
}
+536
View File
@@ -0,0 +1,536 @@
/**
* src/analytics/domain/entities.ts
* =================================
* BI-P002 — ANALYTICS DOMAIN MODEL.
*
* The nineteen canonical analytics entities, named EXACTLY as the program
* lists them (no renaming, no merging, no "we already have something like
* that"):
*
* AnalyticsDataset DatasetVersion DatasetField
* DatasetGrant MetricDefinition MetricVersion
* DimensionDefinition SemanticModel AnalyticsQuery
* QueryExecution QueryResult DashboardDefinition
* DashboardVersion ReportDefinition ReportRun
* DataQualityRule DataQualityRun LineageReference
* BIApplicationBinding
*
* Every tenant-owned record carries `tenant_id` + `project_id` where
* applicable. The four version entities the program names as immutable
* (MetricVersion, SemanticModel version, DashboardVersion, DatasetVersion)
* are `readonly` here AND immutable in the database — sql/008 installs BEFORE
* UPDATE/DELETE triggers, because a TypeScript `readonly` is a compiler
* opinion, not a storage guarantee.
*
* NAME COLLISION, deliberate: src/services/dataset-version-service.ts already
* exports an interface called `DatasetVersion` for the ST0RE-artifact-backed
* dataset versioning that predates this program. That one is about *dataset
* content artifacts*; this one is about *analytics dataset schema versions*
* and lives in its own module namespace (`AnalyticsDatasetVersion` is NOT
* used as the name — BI-P002 lists `DatasetVersion` verbatim and that is what
* ships). Import them qualified when both are needed in one file.
*
* ARCHITECTURE BOUNDARY: these entities are owned by 0pt. A product repo
* (VehIQ) supplies domain MEANING through `BIApplicationBinding`; it never
* defines its own copies of these tables and never receives engine
* credentials.
*/
import type { AnalyticsTier } from "./tiers";
/** Every tenant-owned analytics record. */
export interface TenantScoped {
tenant_id: string;
project_id: string;
}
// ── field/typing vocabulary ────────────────────────────────────────────────
export const ANALYTICS_FIELD_TYPES = [
"string",
"int64",
"float64",
"bool",
"date",
"timestamp",
"decimal",
"json",
] as const;
export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number];
/** A field's analytical role. Measures aggregate; dimensions group. */
export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const;
export type FieldRole = (typeof FIELD_ROLES)[number];
/** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */
export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const;
export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number];
// ── 1. AnalyticsDataset ────────────────────────────────────────────────────
export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const;
export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number];
/**
* BI-P003 DoD: "Every dataset declares tier, schema, owner, source, freshness,
* quality, retention, classification." All eight are REQUIRED fields here —
* an optional declaration is not a declaration.
*/
export interface AnalyticsDataset extends TenantScoped {
dataset_id: string;
name: string;
/** BI-P003 tier. */
tier: AnalyticsTier;
/** Contract id + version this dataset's schema conforms to (BI-P004). */
schema_contract_id: string;
schema_contract_version: string;
/** Accountable human/team, not the person who typed the create call. */
owner_id: string;
/** Upstream: a 0pt source_id, another dataset_id, or an external ref. */
source_ref: string;
/** Freshness SLO in seconds — how stale the data may be before it is late. */
freshness_slo_seconds: number;
/** DataQualityRule ids that gate this dataset. */
quality_rule_ids: string[];
/** Retention in days. 0 = keep indefinitely (must be justified in policy). */
retention_days: number;
classification: PiiClassification;
status: AnalyticsDatasetStatus;
/** BI-P003 escape hatch for RAW/NORMALIZED exposure to a BI surface. */
raw_access_exception?: string;
description?: string;
created_at: string;
updated_at?: string;
created_by?: string;
}
// ── 2. DatasetVersion (IMMUTABLE) ──────────────────────────────────────────
export interface DatasetVersion extends TenantScoped {
readonly dataset_version_id: string;
readonly dataset_id: string;
/** Monotonic per dataset, starting at 1. */
readonly version: number;
/** The frozen DatasetContract JSON at this version (BI-P004). */
readonly contract_hash: string;
readonly schema_contract_version: string;
readonly record_count?: number;
readonly created_at: string;
readonly created_by?: string;
}
// ── 3. DatasetField ────────────────────────────────────────────────────────
export interface DatasetField extends TenantScoped {
dataset_field_id: string;
dataset_id: string;
/** The dataset version this field description belongs to. */
dataset_version: number;
name: string;
field_type: AnalyticsFieldType;
nullable: boolean;
role: FieldRole;
/** True for the field(s) forming the dataset's primary identifier. */
primary_identifier: boolean;
classification: PiiClassification;
description?: string;
}
// ── 4. DatasetGrant ────────────────────────────────────────────────────────
export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const;
export type GrantLevel = (typeof GRANT_LEVELS)[number];
/**
* Who may use a dataset, and FOR WHAT. `allowed_purposes` is the GDPR purpose
* limitation carried into analytics — a grant without a purpose is refused
* by the repository layer.
*/
export interface DatasetGrant extends TenantScoped {
grant_id: string;
dataset_id: string;
/** Subject: a CL0UD principal, role, or BI application binding id. */
subject_id: string;
subject_type: "user" | "role" | "service" | "bi_application";
level: GrantLevel;
allowed_purposes: string[];
expires_at?: string;
created_at: string;
created_by?: string;
}
// ── 5. MetricDefinition ────────────────────────────────────────────────────
export const METRIC_AGGREGATIONS = [
"sum",
"avg",
"min",
"max",
"count",
"count_distinct",
"ratio",
"derived",
] as const;
export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number];
export interface MetricDefinition extends TenantScoped {
metric_id: string;
/** Stable business name; unique per tenant. */
name: string;
dataset_id: string;
aggregation: MetricAggregation;
/** Measure field aggregated, or the numerator for ratio metrics. */
measure_field: string;
/** Denominator for ratio metrics. */
denominator_field?: string;
/** Dimensions this metric may legitimately be sliced by. */
allowed_dimension_ids: string[];
/** Current live version number (points at a MetricVersion). */
current_version: number;
owner_id: string;
description?: string;
created_at: string;
updated_at?: string;
}
// ── 6. MetricVersion (IMMUTABLE) ───────────────────────────────────────────
export interface MetricVersion extends TenantScoped {
readonly metric_version_id: string;
readonly metric_id: string;
readonly version: number;
/** Frozen definition JSON — what this metric MEANT at this version. */
readonly definition_json: string;
readonly definition_hash: string;
/** Free-text reason the metric changed; required for version > 1. */
readonly change_reason?: string;
readonly created_at: string;
readonly created_by?: string;
}
// ── 7. DimensionDefinition ─────────────────────────────────────────────────
export interface DimensionDefinition extends TenantScoped {
dimension_id: string;
name: string;
dataset_id: string;
/**
* Physical column/key this dimension resolves to. This is the ONLY place a
* caller-supplied dimension name becomes a physical identifier — the query
* compiler resolves through here and never splices caller input (BI-P011
* "dimension identifier injection").
*/
field_name: string;
field_type: AnalyticsFieldType;
classification: PiiClassification;
description?: string;
created_at: string;
}
// ── 8. SemanticModel (versioned; versions IMMUTABLE) ───────────────────────
export interface SemanticModel extends TenantScoped {
semantic_model_id: string;
name: string;
/** Immutable version number — a change mints a new row, never an UPDATE. */
readonly version: number;
/** MetricFlow/Cube-compatible model document (BI-P029/BI-P032 consume it). */
readonly model_json: string;
readonly model_hash: string;
dataset_ids: string[];
owner_id: string;
readonly created_at: string;
readonly created_by?: string;
}
// ── 9. AnalyticsQuery ──────────────────────────────────────────────────────
export const QUERY_SURFACES = ["metric", "sql", "projection"] as const;
export type QuerySurface = (typeof QUERY_SURFACES)[number];
/**
* A SAVED, reusable query. The request that actually runs is
* `src/analytics/engine/query-compiler.ts`'s `MetricQueryRequest`; this entity
* is its durable, governed form.
*/
export interface AnalyticsQuery extends TenantScoped {
query_id: string;
name: string;
surface: QuerySurface;
dataset_id: string;
metric_id?: string;
/** Serialized closed-DSL request. Never raw SQL for surface="metric". */
request_json: string;
owner_id: string;
created_at: string;
updated_at?: string;
}
// ── 10. QueryExecution ─────────────────────────────────────────────────────
export const EXECUTION_STATES = [
"queued",
"running",
"succeeded",
"failed",
"cancelled",
"rejected",
] as const;
export type ExecutionState = (typeof EXECUTION_STATES)[number];
export interface QueryExecution extends TenantScoped {
execution_id: string;
query_id?: string;
/** Engine-side identifier used to CANCEL this execution (BI-P009). */
engine_query_id: string;
state: ExecutionState;
/** Which provider ran it: "clickhouse" | "duckdb" | "trino". */
provider: string;
submitted_at: string;
started_at?: string;
finished_at?: string;
duration_ms?: number;
rows_read?: number;
bytes_read?: number;
result_rows?: number;
/** SiaxErrorCode when state="failed"/"rejected". Never a raw engine body. */
error_code?: string;
correlation_id?: string;
}
// ── 11. QueryResult ────────────────────────────────────────────────────────
export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const;
export type ResultEncoding = (typeof RESULT_ENCODINGS)[number];
export interface QueryResult extends TenantScoped {
result_id: string;
execution_id: string;
encoding: ResultEncoding;
row_count: number;
/** Canonical ST0RE ref when materialised; inline results carry no bytes here. */
artifact_ref?: string;
/** True when the engine truncated at max_result_rows (BI-P012). */
truncated: boolean;
expires_at?: string;
created_at: string;
}
// ── 12. DashboardDefinition ────────────────────────────────────────────────
export interface DashboardDefinition extends TenantScoped {
dashboard_id: string;
name: string;
/** Live version pointer. */
current_version: number;
owner_id: string;
/** Datasets the dashboard reads — checked against BI-P003 tier rules. */
dataset_ids: string[];
description?: string;
created_at: string;
updated_at?: string;
}
// ── 13. DashboardVersion (IMMUTABLE) ───────────────────────────────────────
export interface DashboardVersion extends TenantScoped {
readonly dashboard_version_id: string;
readonly dashboard_id: string;
readonly version: number;
/** Frozen layout/panel document. */
readonly layout_json: string;
readonly layout_hash: string;
readonly created_at: string;
readonly created_by?: string;
}
// ── 14. ReportDefinition ───────────────────────────────────────────────────
export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const;
export type ReportFormat = (typeof REPORT_FORMATS)[number];
export interface ReportDefinition extends TenantScoped {
report_id: string;
name: string;
dashboard_id?: string;
query_id?: string;
format: ReportFormat;
/** Cron expression; undefined = on-demand only. */
schedule_cron?: string;
owner_id: string;
created_at: string;
updated_at?: string;
}
// ── 15. ReportRun ──────────────────────────────────────────────────────────
export interface ReportRun extends TenantScoped {
report_run_id: string;
report_id: string;
state: ExecutionState;
started_at: string;
finished_at?: string;
/** ST0RE artifact ref of the produced report. */
artifact_ref?: string;
error_code?: string;
correlation_id?: string;
}
// ── 16. DataQualityRule ────────────────────────────────────────────────────
export const QUALITY_RULE_KINDS = [
"not_null",
"unique",
"accepted_values",
"range",
"freshness",
"row_count",
"referential",
"custom",
] as const;
export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number];
export const QUALITY_SEVERITIES = ["warn", "block"] as const;
export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number];
export interface DataQualityRule extends TenantScoped {
rule_id: string;
dataset_id: string;
kind: QualityRuleKind;
/** Field the rule applies to; omitted for dataset-level rules. */
field_name?: string;
/** Rule parameters (bounds, accepted values, max age). */
params_json: string;
severity: QualitySeverity;
created_at: string;
created_by?: string;
}
// ── 17. DataQualityRun ─────────────────────────────────────────────────────
export interface DataQualityRun extends TenantScoped {
quality_run_id: string;
rule_id: string;
dataset_id: string;
dataset_version?: number;
passed: boolean;
/** Rows that violated the rule. */
violation_count: number;
evaluated_rows: number;
started_at: string;
finished_at?: string;
correlation_id?: string;
}
// ── 18. LineageReference ───────────────────────────────────────────────────
export const LINEAGE_DIRECTIONS = ["input", "output"] as const;
export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number];
/**
* The queryable projection of an OpenLineage edge (BI-P027). AUD0 stays the
* system of record; this is 0pt's local index so "what fed this dashboard"
* is answerable without leaving the analytics plane.
*/
export interface LineageReference extends TenantScoped {
lineage_reference_id: string;
/** OpenLineage run id this edge came from. */
run_id: string;
job_name: string;
direction: LineageDirection;
/** The analytics entity this edge touches. */
entity_type: "dataset" | "metric" | "dashboard" | "report";
entity_id: string;
/** Namespaced dataset name on the other side of the edge. */
counterpart_ref: string;
observed_at: string;
}
// ── 19. BIApplicationBinding ───────────────────────────────────────────────
export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const;
export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number];
/**
* How an external BI surface is attached to the analytics plane.
*
* NORMATIVE (BI program §0 + the SIAX/0pt ownership boundary): this binding
* carries NO engine credential. A bound application authenticates to the 0pt
* Analytics API with a CL0UD identity and receives compiled, tenant-forced
* results — it never learns a ClickHouse DSN, user, or password. The absence
* of a credential field here is the enforcement, not a comment.
*/
export interface BIApplicationBinding extends TenantScoped {
binding_id: string;
kind: BiApplicationKind;
name: string;
/** CL0UD principal the application authenticates as. */
principal_id: string;
/** Datasets/metrics this application may reach at all. */
dataset_ids: string[];
metric_ids: string[];
allowed_purposes: string[];
enabled: boolean;
created_at: string;
created_by?: string;
}
// ── registry of the canonical nineteen ─────────────────────────────────────
/**
* The canonical entity names, verbatim from BI-P002. Persisted-schema tests
* assert sql/008 defines a table for every one of these, so a future rename
* or omission is a test failure rather than drift.
*/
export const ANALYTICS_DOMAIN_ENTITIES = [
"AnalyticsDataset",
"DatasetVersion",
"DatasetField",
"DatasetGrant",
"MetricDefinition",
"MetricVersion",
"DimensionDefinition",
"SemanticModel",
"AnalyticsQuery",
"QueryExecution",
"QueryResult",
"DashboardDefinition",
"DashboardVersion",
"ReportDefinition",
"ReportRun",
"DataQualityRule",
"DataQualityRun",
"LineageReference",
"BIApplicationBinding",
] as const;
export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number];
/** Entity -> physical table in sql/008_analytics_domain_model.sql. */
export const ANALYTICS_ENTITY_TABLES: Record<AnalyticsDomainEntity, string> = {
AnalyticsDataset: "opt_analytics_dataset",
DatasetVersion: "opt_analytics_dataset_version",
DatasetField: "opt_analytics_dataset_field",
DatasetGrant: "opt_analytics_dataset_grant",
MetricDefinition: "opt_analytics_metric_definition",
MetricVersion: "opt_analytics_metric_version",
DimensionDefinition: "opt_analytics_dimension_definition",
SemanticModel: "opt_analytics_semantic_model",
AnalyticsQuery: "opt_analytics_query",
QueryExecution: "opt_analytics_query_execution",
QueryResult: "opt_analytics_query_result",
DashboardDefinition: "opt_analytics_dashboard_definition",
DashboardVersion: "opt_analytics_dashboard_version",
ReportDefinition: "opt_analytics_report_definition",
ReportRun: "opt_analytics_report_run",
DataQualityRule: "opt_analytics_data_quality_rule",
DataQualityRun: "opt_analytics_data_quality_run",
LineageReference: "opt_analytics_lineage_reference",
BIApplicationBinding: "opt_analytics_bi_application_binding",
};
/** The four entities BI-P002 requires to be immutable once written. */
export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [
"DatasetVersion",
"MetricVersion",
"SemanticModel",
"DashboardVersion",
];
+83
View File
@@ -0,0 +1,83 @@
/**
* src/analytics/domain/tiers.ts
* ==============================
* BI-P003 — ANALYTICS DATA TIERS.
*
* The four canonical tiers every analytics dataset in SIAX belongs to. The
* tier is a CONTRACT about where a dataset may be read from and by whom, not
* a naming convention:
*
* RAW -> ST0RE Parquet/Iceberg (as-ingested, no business meaning)
* NORMALIZED -> ST0RE Iceberg (typed, deduplicated, still source-shaped)
* CURATED -> ST0RE Iceberg (business entities, modelled)
* SERVING -> ClickHouse / projection (query-optimised, dashboard-facing)
*
* BI-P003's normative sentence is "No business dashboard reads arbitrary raw
* ingestion tables by default" — `assertReadableByBiSurface` is where that
* sentence stops being prose. A BI surface (dashboard, report, embedded
* chart) may read SERVING and CURATED; RAW and NORMALIZED require an
* explicit, recorded exception on the dataset itself.
*
* ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product
* repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers.
*/
import { SiaxError } from "../../platform/errors";
export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const;
export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number];
/** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */
export const TIER_PHYSICAL_HOME: Record<AnalyticsTier, string> = {
RAW: "st0re:iceberg|parquet",
NORMALIZED: "st0re:iceberg",
CURATED: "st0re:iceberg",
SERVING: "clickhouse:projection",
};
/** Tiers a BI presentation surface may read without an explicit exception. */
export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"];
export function isAnalyticsTier(v: unknown): v is AnalyticsTier {
return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v);
}
/**
* Rank used for promotion checks: a dataset may only be BUILT FROM a tier at
* or below its own (SERVING can be built from CURATED; CURATED must never be
* built from SERVING, which would make the projection its own source).
*/
export const TIER_RANK: Record<AnalyticsTier, number> = {
RAW: 0,
NORMALIZED: 1,
CURATED: 2,
SERVING: 3,
};
/**
* BI-P003 DoD gate. `raw_access_exception` is the recorded, auditable escape
* hatch — absent it, a RAW/NORMALIZED dataset is invisible to dashboards.
*/
export function assertReadableByBiSurface(
dataset: { dataset_id: string; tier: AnalyticsTier; raw_access_exception?: string },
surface: string,
): void {
if (BI_READABLE_TIERS.includes(dataset.tier)) return;
if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return;
throw new SiaxError(
"AUTHORIZATION_DENIED",
"0pt.analytics",
`BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` +
`without a recorded raw_access_exception (BI-P003)`,
);
}
/** A dataset may not declare a source tier ABOVE its own tier. */
export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void {
if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`,
);
}
}
+563
View File
@@ -0,0 +1,563 @@
/**
* src/analytics/engine/clickhouse-engine.ts
* ==========================================
* BI-P009 — ClickHouse as the CANONICAL `SiaxAnalyticsEngine` provider.
*
* PRESERVATION (BI-P009's first sentence): `ClickHouseAnalytics`
* (src/analytics/clickhouse-analytics.ts) is untouched by this change — not
* one line. This engine COMPOSES it:
*
* * `health()` delegates to `ClickHouseAnalytics.health()`.
* * `executeMetric()` on the pre-existing `opt_metric_events` table
* delegates to `ClickHouseAnalytics.queryMetric()`, i.e. keeps the
* already-proven forced-tenant path exactly as it is.
* * everything else (the BI-P010 tables) goes through
* `src/analytics/engine/query-compiler.ts`, which forces the same
* predicate for the same reason.
*
* "Do not regress the existing forced tenant predicate" is enforced by
* tests/analytics/tenant-predicate-regression.test.ts, which fails if either
* compiler stops emitting `tenant_id = {tenant:String}`.
*
* BI-P012 governance is applied on EVERY statement, including the delegated
* ones — the settings ride on the HTTP request, and `readonly=1` means the
* query text cannot re-set them.
*
* LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts
* ClickHouse pipelines, never quarantined market data.
*/
import crypto from "node:crypto";
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors";
import { ClickHouseAnalytics } from "../clickhouse-analytics";
import {
compileMetricQuery,
compileProjectionQuery,
dimensionCatalogFrom,
FORCED_TENANT_PREDICATE,
type DimensionCatalog,
} from "./query-compiler";
import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema";
import {
applyQuota,
clickhouseSettings,
ConcurrencyGate,
DEFAULT_BUDGETS,
planningSettings,
type ResourceBudget,
} from "./resource-governance";
import type {
AnalystQueryRequest,
AnalyticsIdentity,
EngineResult,
HealthReport,
MetricRequest,
ProjectionIngestRequest,
ProjectionRefreshResult,
QueryEstimate,
QueryExplanation,
SiaxAnalyticsEngine,
} from "./siax-analytics-engine";
export interface ClickHouseEngineOptions {
baseUrl: string;
/** "user:password" — from an env/secret ref, never a literal in code. */
basicAuth?: string;
database?: string;
timeoutMs?: number;
/** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */
dimensionCatalogs?: Record<string, Record<string, string>>;
schema?: Omit<AnalyticsSchemaOptions, "database">;
}
const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined };
export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
readonly provider = "clickhouse";
/** PRESERVED pre-existing implementation, composed not replaced. */
private readonly legacy: ClickHouseAnalytics;
private readonly baseUrl: string;
private readonly authHeader?: string;
private readonly database: string;
private readonly timeoutMs: number;
private readonly dimensionCatalogs: Record<string, Record<string, string>>;
private readonly schemaOpts: Omit<AnalyticsSchemaOptions, "database">;
private readonly gate: ConcurrencyGate;
/** query_id -> AbortController, so cancelQuery can stop the client side too. */
private readonly inFlight = new Map<string, AbortController>();
constructor(opts: ClickHouseEngineOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
this.database = opts.database ?? "opt_analytics";
this.timeoutMs = opts.timeoutMs ?? 30_000;
this.dimensionCatalogs = opts.dimensionCatalogs ?? {};
this.schemaOpts = opts.schema ?? {};
this.legacy = new ClickHouseAnalytics({
baseUrl: opts.baseUrl,
basicAuth: opts.basicAuth,
database: this.database,
timeoutMs: this.timeoutMs,
});
this.gate = new ConcurrencyGate((tenantId) => this.budgetForTenant(tenantId).maxConcurrentQueries);
}
/** Access to the preserved implementation, for callers that still use it. */
get legacyAnalytics(): ClickHouseAnalytics {
return this.legacy;
}
private budgetForTenant(_tenantId: string): ResourceBudget {
return DEFAULT_BUDGETS.INTERACTIVE;
}
private budgetFor(identity: AnalyticsIdentity): ResourceBudget {
const tier = identity.budget_tier ?? "INTERACTIVE";
return applyQuota(DEFAULT_BUDGETS[tier], identity.quota);
}
private catalogFor(identity: AnalyticsIdentity): DimensionCatalog {
const pairs = this.dimensionCatalogs[identity.tenant_id];
return pairs ? dimensionCatalogFrom(pairs) : EMPTY_CATALOG;
}
/** Fail closed: a missing/blank verified tenant is never a wildcard. */
private assertIdentity(identity: AnalyticsIdentity): void {
if (!identity?.tenant_id || !identity.tenant_id.trim()) {
throw new SiaxError(
"UNAUTHENTICATED",
"0pt.analytics",
"analytics requires a verified CL0UD identity carrying tenant_id",
);
}
if (!identity.principal_id || !identity.principal_id.trim()) {
throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id");
}
}
// ── schema ───────────────────────────────────────────────────────────────
/** Apply BI-P010 DDL (idempotent) plus the preserved legacy table. */
async ensureSchema(): Promise<void> {
await this.legacy.ensureSchema();
for (const stmt of analyticsSchemaDdl({ database: this.database, ...this.schemaOpts })) {
await this.ddl(stmt);
}
}
// ── BI-P009 capability 1: health ─────────────────────────────────────────
async health(): Promise<HealthReport> {
try {
const legacyOk = await this.legacy.health();
if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" };
const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {});
const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v;
return { healthy: true, provider: this.provider, version };
} catch (err) {
return {
healthy: false,
provider: this.provider,
detail: err instanceof SiaxError ? err.code : "unreachable",
};
}
}
// ── BI-P009 capability 2: executeMetric ──────────────────────────────────
async executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise<EngineResult> {
this.assertIdentity(identity);
const budget = this.budgetFor(identity);
const release = this.gate.acquire(identity.tenant_id);
const started = Date.now();
const queryId = this.newQueryId(identity);
try {
const compiled = compileMetricQuery(
{
tenant_id: identity.tenant_id, // FORCED from the verified identity
project_id: identity.project_id,
dataset_id: request.dataset_id,
metric: request.metric,
aggregate: request.aggregate,
group_by: request.group_by,
filters: request.filters,
from_time: request.from_time,
to_time: request.to_time,
limit: request.limit,
table: request.table,
},
this.database,
this.catalogFor(identity),
budget.maxResultRows,
);
const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as
| ClickHouseJsonResponse
| undefined;
const groupNames = request.group_by ?? [];
const rows = (json?.data ?? []).map((r) => {
const groups: Record<string, string> = {};
groupNames.forEach((name, i) => {
const v = (r as Record<string, unknown>)[`grp${i}`];
groups[name] = v === undefined || v === null ? "" : String(v);
});
return {
...(groupNames.length ? { groups } : {}),
value: Number((r as { value: number | string }).value),
};
});
const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), budget.maxResultRows));
return {
rows,
query_id: queryId,
truncated: rows.length >= limit,
stats: {
duration_ms: Date.now() - started,
rows_read: json?.statistics?.rows_read,
bytes_read: json?.statistics?.bytes_read,
result_rows: rows.length,
},
};
} finally {
release();
this.inFlight.delete(queryId);
}
}
/**
* Delegating path to the PRESERVED implementation for the pre-existing
* `opt_metric_events` table. Kept so no existing caller has to migrate and
* the already-proven forced-predicate path stays in service.
*/
async executeLegacyMetric(
identity: AnalyticsIdentity,
request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number },
): Promise<EngineResult> {
this.assertIdentity(identity);
const started = Date.now();
const rows = await this.legacy.queryMetric({
tenant_id: identity.tenant_id, // FORCED from the verified identity
metric: request.metric,
aggregate: request.aggregate,
dataset_id: request.dataset_id,
group_by: request.group_by,
from_time: request.from_time,
to_time: request.to_time,
limit: request.limit,
});
return {
rows: rows.map((r) => ({
...(r.group !== undefined && request.group_by ? { groups: { [request.group_by]: r.group } } : {}),
value: r.value,
})),
query_id: this.newQueryId(identity),
truncated: false,
stats: { duration_ms: Date.now() - started, result_rows: rows.length },
};
}
// ── BI-P009 capability 3: executeQuery (analyst SQL) ─────────────────────
async executeQuery(identity: AnalyticsIdentity, _request: AnalystQueryRequest): Promise<EngineResult> {
this.assertIdentity(identity);
// Deliberate, documented refusal — see AnalystQueryRequest's doc comment.
// Opening a raw-SQL surface before BI-P017 (SQL dialect contract) and
// BI-P033 (analytics authorization) exist would create precisely the
// "raw SQL attempt" hole BI-P011's adversarial list enumerates.
throw new SiaxError(
"CAPABILITY_UNAVAILABLE",
"0pt.analytics",
"analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land",
);
}
// ── BI-P009 capability 4: estimateQuery ──────────────────────────────────
async estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryEstimate> {
this.assertIdentity(identity);
const budget = this.budgetFor(identity);
const compiled = compileMetricQuery(
{
tenant_id: identity.tenant_id,
project_id: identity.project_id,
dataset_id: request.dataset_id,
metric: request.metric,
aggregate: request.aggregate,
group_by: request.group_by,
filters: request.filters,
from_time: request.from_time,
to_time: request.to_time,
limit: request.limit,
table: request.table,
},
this.database,
this.catalogFor(identity),
budget.maxResultRows,
);
// `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read
// AFTER primary-key pruning — which is why tenant_id leads every ORDER BY
// in schema.ts: the estimate for one tenant does not include everyone.
const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, "");
let rowsToRead = 0;
let partsRead = 0;
try {
const res = (await this.raw(
`EXPLAIN ESTIMATE ${estimateSql} FORMAT JSON`,
compiled.params,
planningSettings(budget),
)) as { data?: { rows?: number | string; parts?: number | string }[] } | undefined;
for (const row of res?.data ?? []) {
rowsToRead += Number(row.rows ?? 0);
partsRead += Number(row.parts ?? 0);
}
} catch (err) {
// An estimate that cannot be produced must not read as "cheap".
throw err instanceof SiaxError
? err
: new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query");
}
// Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes.
// 128 B/row is a deliberately PESSIMISTIC constant for these tables
// (measured mean in the BI-P010 benchmark is lower), so the gate errs
// toward refusing rather than admitting an oversized query.
const bytesToRead = rowsToRead * 128;
const within = rowsToRead <= budget.maxRowsToRead && bytesToRead <= budget.maxBytesToRead;
return {
estimated_rows_to_read: rowsToRead,
estimated_bytes_to_read: bytesToRead,
within_budget: within,
budget,
...(within
? {}
: {
reason:
`estimate ${rowsToRead} rows / ${bytesToRead} bytes over budget ` +
`(${budget.maxRowsToRead} rows / ${budget.maxBytesToRead} bytes), parts=${partsRead}`,
}),
};
}
// ── BI-P009 capability 5: cancelQuery ────────────────────────────────────
/**
* Two-sided cancellation. The client-side abort stops us waiting; the
* server-side KILL QUERY stops ClickHouse burning CPU. Cancelling is
* tenant-scoped: the query id embeds the tenant and is checked, so tenant A
* cannot cancel tenant B's work.
*/
async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise<boolean> {
this.assertIdentity(identity);
if (!queryId.startsWith(this.queryIdPrefix(identity))) {
throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant");
}
this.inFlight.get(queryId)?.abort();
this.inFlight.delete(queryId);
try {
// KILL QUERY is a mutation of server state, so it cannot ride the
// readonly=1 settings the SELECT path uses.
await this.raw(
`KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`,
{ param_killId: queryId },
{ max_execution_time: "10" },
);
return true;
} catch {
return false;
}
}
// ── BI-P009 capability 6: explain ────────────────────────────────────────
async explain(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryExplanation> {
this.assertIdentity(identity);
const budget = this.budgetFor(identity);
const compiled = compileMetricQuery(
{
tenant_id: identity.tenant_id,
project_id: identity.project_id,
dataset_id: request.dataset_id,
metric: request.metric,
aggregate: request.aggregate,
group_by: request.group_by,
filters: request.filters,
from_time: request.from_time,
to_time: request.to_time,
limit: request.limit,
table: request.table,
},
this.database,
this.catalogFor(identity),
budget.maxResultRows,
);
const sql = compiled.sql.replace(/ FORMAT JSON$/, "");
const res = (await this.raw(
`EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`,
compiled.params,
planningSettings(budget),
)) as string | undefined;
return {
plan: typeof res === "string" ? res : JSON.stringify(res ?? ""),
forced_tenant_predicate: FORCED_TENANT_PREDICATE,
table: compiled.table,
};
}
// ── BI-P009 capability 7: ingestProjection ───────────────────────────────
async ingestProjection(
identity: AnalyticsIdentity,
request: ProjectionIngestRequest,
): Promise<{ ingested: number }> {
this.assertIdentity(identity);
if (request.rows.length === 0) return { ingested: 0 };
const now = new Date();
const version = Math.floor(now.getTime());
const body = request.rows
.map((r) => {
if (!r.entity_key || !r.entity_key.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key");
}
return JSON.stringify({
// tenant/project come from the VERIFIED identity, never the row —
// a row-supplied tenant_id is the INF0 Wave-2 bug class.
tenant_id: identity.tenant_id,
project_id: identity.project_id ?? "",
dataset_id: request.dataset_id,
entity_key: r.entity_key,
as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""),
version: r.version ?? version,
attributes: r.attributes ?? {},
measures: r.measures ?? {},
});
})
.join("\n");
await this.raw(
`INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`,
{},
{},
undefined,
body,
);
return { ingested: request.rows.length };
}
// ── BI-P009 capability 8: refreshProjection ──────────────────────────────
/**
* ReplacingMergeTree deduplicates on a background merge, so a read straight
* after an ingest can still see superseded rows. `OPTIMIZE ... FINAL` forces
* the merge for THIS tenant's partition only — the table is
* `PARTITION BY (tenant_id)` precisely so one tenant's refresh does not
* rewrite another tenant's parts (BI-P012's "without degrading unrelated
* tenant" applies to maintenance too, not only to queries).
*/
async refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise<ProjectionRefreshResult> {
this.assertIdentity(identity);
await this.raw(
`OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`,
{ param_tenant: identity.tenant_id },
{ max_execution_time: "120" },
);
const compiled = compileProjectionQuery(
{ tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 },
this.database,
1,
);
const countSql =
`SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` +
`WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`;
const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as
| { data?: { value: number | string }[] }
| undefined;
return {
dataset_id,
row_count: Number(res?.data?.[0]?.value ?? 0),
refreshed_at: new Date().toISOString(),
};
}
// ── transport ────────────────────────────────────────────────────────────
private queryIdPrefix(identity: AnalyticsIdentity): string {
return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`;
}
private newQueryId(identity: AnalyticsIdentity): string {
return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`;
}
/** DDL path: no readonly, no result parsing beyond errors. */
private async ddl(sql: string): Promise<void> {
await this.raw(sql, {}, { max_execution_time: "60" });
}
private async raw(
sql: string,
params: Record<string, string>,
settings: Record<string, string>,
queryId?: string,
body?: string,
): Promise<unknown> {
const url = new URL(this.baseUrl);
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
if (queryId) url.searchParams.set("query_id", queryId);
const headers: Record<string, string> = {};
if (this.authHeader) headers["Authorization"] = this.authHeader;
const controller = new AbortController();
if (queryId) this.inFlight.set(queryId, controller);
const timeout = setTimeout(() => controller.abort(), this.timeoutMs);
let res: Response;
try {
res = await fetch(url.toString(), {
method: "POST",
headers,
body: body ?? "",
signal: controller.signal,
});
} catch (cause) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
} finally {
clearTimeout(timeout);
}
const text = await res.text();
if (!res.ok) {
// ClickHouse error text can echo the query; keep only the numeric code
// so a caller never receives another tenant's identifiers back.
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
"clickhouse",
`query failed (CH code ${code})`,
);
}
if (!text) return undefined;
let parsed: unknown;
try {
parsed = JSON.parse(text);
} catch {
return text;
}
// DEFENCE IN DEPTH against a partial result that looks complete.
// ClickHouse normally answers a breached limit with an HTTP error status
// (measured on 24.8: 408 for max_execution_time, 500 for
// max_rows_to_read), which `res.ok` above already catches. But when rows
// have already been flushed, the JSON body can carry an `exception` key
// under a 200 — a truncated answer indistinguishable from a whole one.
// Treat any body carrying an exception as a failure, whatever the status.
if (parsed && typeof parsed === "object" && "exception" in parsed) {
const ex = String((parsed as { exception?: unknown }).exception ?? "");
const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown";
throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`);
}
return parsed;
}
}
interface ClickHouseJsonResponse {
data?: Record<string, unknown>[];
statistics?: { rows_read?: number; bytes_read?: number; elapsed?: number };
}
+304
View File
@@ -0,0 +1,304 @@
/**
* src/analytics/engine/query-compiler.ts
* =======================================
* BI-P011 — CLICKHOUSE TENANT ISOLATION, layer 2:
* "`0pt` query compiler forces tenant".
*
* The compiler is the ONLY thing in this repo that turns an analytics request
* into SQL text. Its contract:
*
* * `tenant_id` is not an argument the caller may omit — it comes from the
* VERIFIED CL0UD identity (layer 1) and is written into the WHERE clause
* by this function, unconditionally, as a bound parameter.
* * caller-supplied VALUES are never spliced into SQL text. They travel as
* ClickHouse HTTP `param_*` query parameters.
* * caller-supplied IDENTIFIERS (a dimension name, a group-by column) are
* never spliced either. They are RESOLVED against a declared allow-list
* of `DimensionDefinition`s and only the resolved, already-known physical
* name reaches the SQL — so "dimension identifier injection" has no
* surface, because caller text never becomes an identifier.
* * there is no raw-SQL entry point. `compileMetricQuery` cannot express a
* UNION, a subquery, or a second table, so a caller cannot reach a table
* the compiler did not choose.
*
* WHY A SEPARATE MODULE from clickhouse-analytics.ts: that file's compiler
* already forces the tenant predicate correctly and BI-P009 says PRESERVE IT.
* This module is the generalised compiler for the BI-P010 tables; it repeats
* the guarantee rather than replacing it, and
* tests/analytics/tenant-predicate-regression.test.ts asserts BOTH.
*/
import { SiaxError } from "../../platform/errors";
import type { AnalyticsTable } from "./schema";
/** A physical identifier that has been VALIDATED, not merely received. */
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
export function assertSafeIdentifier(value: string, what: string): string {
if (!IDENT_RE.test(value)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`);
}
return value;
}
export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const;
export type MetricAggregate = (typeof METRIC_AGGREGATES)[number];
/**
* The dimensions a caller is allowed to name, resolved from
* `DimensionDefinition` rows. The compiler looks a caller's dimension name up
* HERE; a miss is a rejection, never a pass-through.
*/
export interface DimensionCatalog {
/** caller-facing dimension name -> physical dimension key in dim_names. */
resolve(name: string): string | undefined;
}
export function dimensionCatalogFrom(pairs: Record<string, string>): DimensionCatalog {
const map = new Map(Object.entries(pairs));
return {
resolve(name: string): string | undefined {
// Own-property lookup on a Map — no prototype-chain surprises
// ("constructor", "__proto__" resolve to undefined, as they must).
return map.get(name);
},
};
}
export interface MetricQueryRequest {
/** Supplied by the engine from the verified identity, NOT by the caller. */
tenant_id: string;
project_id?: string;
dataset_id?: string;
metric: string;
aggregate: MetricAggregate;
/** Caller-facing dimension names; resolved through the catalog. */
group_by?: string[];
/** Equality filters on dimensions: caller-facing name -> value. */
filters?: Record<string, string>;
from_time?: string;
to_time?: string;
limit?: number;
/** Which BI-P010 table to read. Chosen by the engine, not the caller. */
table?: AnalyticsTable;
}
export interface CompiledQuery {
sql: string;
/** ClickHouse HTTP `param_*` entries. Values NEVER appear in `sql`. */
params: Record<string, string>;
/** The predicate the compiler forced, for assertion in tests. */
forcedTenantPredicate: string;
table: AnalyticsTable;
}
/**
* The exact predicate text every compiled query must contain. Exported so a
* regression test can assert on the literal string — if someone deletes the
* forcing, the test fails on a missing substring rather than on a subtle
* behavioural difference that a mocked engine would happily hide.
*/
export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}";
const AGG_SQL: Record<MetricAggregate, string> = {
sum: "sum(value)",
avg: "avg(value)",
min: "min(value)",
max: "max(value)",
count: "count()",
count_distinct: "uniqExact(event_id)",
};
const MAX_GROUP_BY = 8;
/**
* Compile a closed-DSL metric request into tenant-forced SQL.
*
* @param req the request; `tenant_id` MUST already be the verified identity.
* @param db the analytics database name (engine-controlled, validated).
* @param catalog the tenant's declared dimensions.
* @param maxResultRows the governance ceiling (BI-P012); `limit` is clamped.
*/
export function compileMetricQuery(
req: MetricQueryRequest,
db: string,
catalog: DimensionCatalog,
maxResultRows: number,
): CompiledQuery {
if (!req.tenant_id || !req.tenant_id.trim()) {
// Fail closed. A blank tenant must never compile to "no predicate".
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
}
assertSafeIdentifier(db, "database");
const table: AnalyticsTable = req.table ?? "opt_analytics_event";
assertSafeIdentifier(table, "table");
if (!METRIC_AGGREGATES.includes(req.aggregate)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate");
}
const params: Record<string, string> = {
param_tenant: req.tenant_id,
param_metric: req.metric,
};
// ── the forced predicate. First condition, always, no branch. ─────────
const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"];
if (req.project_id) {
conditions.push("project_id = {project:String}");
params.param_project = req.project_id;
}
if (req.dataset_id) {
conditions.push("dataset_id = {dataset:String}");
params.param_dataset = req.dataset_id;
}
if (req.from_time) {
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
params.param_fromTime = req.from_time;
}
if (req.to_time) {
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
params.param_toTime = req.to_time;
}
// ── filters: names resolved through the catalog, values parameterized ──
let filterIdx = 0;
for (const [name, value] of Object.entries(req.filters ?? {})) {
const physical = catalog.resolve(name);
if (physical === undefined) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`);
}
// `physical` came from the catalog, but validate anyway: a corrupted or
// hostile catalog row must not become an injection vector either.
assertSafeIdentifier(physical, "dimension");
const nameKey = `f${filterIdx}n`;
const valKey = `f${filterIdx}v`;
conditions.push(
`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`,
);
params[`param_${nameKey}`] = physical;
params[`param_${valKey}`] = value;
filterIdx += 1;
}
// ── group by: same resolution, plus a hard cardinality ceiling ─────────
const groupBy = req.group_by ?? [];
if (groupBy.length > MAX_GROUP_BY) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.analytics",
`at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`,
);
}
const selectParts: string[] = [];
const groupParts: string[] = [];
groupBy.forEach((name, i) => {
const physical = catalog.resolve(name);
if (physical === undefined) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`);
}
assertSafeIdentifier(physical, "dimension");
const key = `g${i}`;
// The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text.
selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`);
groupParts.push(`grp${i}`);
params[`param_${key}`] = physical;
});
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
const sql =
`SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` +
`FROM ${db}.${table} ` +
`WHERE ${conditions.join(" AND ")} ` +
(groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") +
`LIMIT ${limit} FORMAT JSON`;
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table };
}
/**
* Compile a projection lookup (SERVING tier, BI-P010's opt_dataset_projection).
* Same forcing, different table and key.
*/
export function compileProjectionQuery(
req: { tenant_id: string; project_id?: string; dataset_id: string; entity_keys?: string[]; limit?: number },
db: string,
maxResultRows: number,
): CompiledQuery {
if (!req.tenant_id || !req.tenant_id.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
}
assertSafeIdentifier(db, "database");
const params: Record<string, string> = {
param_tenant: req.tenant_id,
param_dataset: req.dataset_id,
};
const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"];
if (req.project_id) {
conditions.push("project_id = {project:String}");
params.param_project = req.project_id;
}
if (req.entity_keys && req.entity_keys.length > 0) {
conditions.push("has({entityKeys:Array(String)}, entity_key)");
params.param_entityKeys = JSON.stringify(req.entity_keys);
}
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
const sql =
`SELECT entity_key, as_of, version, attributes, measures ` +
`FROM ${db}.opt_dataset_projection ` +
`WHERE ${conditions.join(" AND ")} ` +
`ORDER BY entity_key LIMIT ${limit} FORMAT JSON`;
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" };
}
/**
* BI-P011 layer-3 bootstrap: the DDL for a RESTRICTED per-tenant ClickHouse
* user with a row policy pinning `tenant_id`.
*
* This is the layer that survives a compiler bug. The compiler forcing the
* predicate is layer 2; if a future refactor drops it, layer 3 still returns
* zero foreign rows because the SERVER filters them. The BI-P011 proof runs
* the adversarial suite twice — once through an unrestricted user (hole open)
* and once through this restricted user (hole closed) — because a claim of
* defence in depth that has never been tested with one layer removed is not
* evidence.
*
* `tenantId` is validated as an identifier-safe token AND embedded as a
* quoted literal; ClickHouse's CREATE ROW POLICY takes no bound parameters,
* so validation is the only protection available and it is applied strictly.
*/
export function restrictedTenantUserDdl(args: {
db: string;
tenantId: string;
userName: string;
password: string;
tables: readonly string[];
settingsProfile?: string;
}): string[] {
assertSafeIdentifier(args.db, "database");
assertSafeIdentifier(args.userName, "user");
// A tenant id is `t_<slug>` in this estate; enforce that shape strictly
// rather than trusting the caller.
if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe");
}
if (args.password.includes("'") || args.password.includes("\\")) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded");
}
const profile = args.settingsProfile;
if (profile) assertSafeIdentifier(profile, "settings profile");
const stmts: string[] = [
`CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` +
(profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`),
];
for (const table of args.tables) {
assertSafeIdentifier(table, "table");
stmts.push(
`CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` +
`FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`,
);
stmts.push(`GRANT SELECT ON ${args.db}.${table} TO ${args.userName}`);
}
return stmts;
}
+222
View File
@@ -0,0 +1,222 @@
/**
* src/analytics/engine/resource-governance.ts
* ============================================
* BI-P012 — CLICKHOUSE RESOURCE GOVERNANCE.
*
* Every setting BI-P012 names, in one place, with a value:
*
* max_execution_time max_result_rows max_rows_to_read
* max_bytes_to_read max_memory_usage max_concurrent_queries
*
* Three layers, and the order matters:
*
* 1. SERVER-SIDE SETTINGS travel with every statement on the HTTP
* interface. ClickHouse enforces them; the client cannot opt out
* because the client is what writes them. `readonly=1` on top means a
* SELECT cannot re-set them mid-query (readonly=1 forbids SETTINGS
* changes as well as writes — that is the whole reason it is here and
* not merely `readonly=2`).
* 2. `max_concurrent_queries` is NOT a per-query setting in ClickHouse; it
* is a per-USER quota. `concurrencySlotSql` renders the CREATE
* SETTINGS PROFILE that carries it, applied to the tenant's restricted
* user (BI-P011 layer 3) so one tenant's fan-out cannot consume the
* whole server.
* 3. A local admission gate (`ConcurrencyGate`) refuses the N+1st
* in-flight query per tenant before it ever reaches the network. Two
* layers because the ClickHouse quota is per-user and one 0pt process
* can still queue work faster than the server rejects it.
*
* Budget tiers exist because "one limit for everyone" is how a small tenant
* subsidises a query bomb. `INTERACTIVE` is the dashboard default and is
* deliberately mean.
*/
import { SiaxError } from "../../platform/errors";
export interface ResourceBudget {
/** Seconds. ClickHouse `max_execution_time`. */
maxExecutionTimeSec: number;
/** Rows returned to the caller. ClickHouse `max_result_rows`. */
maxResultRows: number;
/** Rows SCANNED (not returned). ClickHouse `max_rows_to_read`. */
maxRowsToRead: number;
/** Bytes scanned. ClickHouse `max_bytes_to_read`. */
maxBytesToRead: number;
/** Bytes of RAM per query. ClickHouse `max_memory_usage`. */
maxMemoryUsageBytes: number;
/** In-flight queries per tenant. Enforced by quota + local gate. */
maxConcurrentQueries: number;
}
export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const;
export type BudgetTier = (typeof BUDGET_TIERS)[number];
/**
* Defaults. Chosen against the shape of the analytics tables in
* `src/analytics/engine/schema.ts`, not plucked from the air:
* INTERACTIVE must answer a dashboard panel inside a human's patience budget,
* so it caps the SCAN, which is what actually costs time on a MergeTree.
*/
export const DEFAULT_BUDGETS: Record<BudgetTier, ResourceBudget> = {
INTERACTIVE: {
maxExecutionTimeSec: 10,
maxResultRows: 10_000,
maxRowsToRead: 50_000_000,
maxBytesToRead: 2 * 1024 * 1024 * 1024,
maxMemoryUsageBytes: 2 * 1024 * 1024 * 1024,
maxConcurrentQueries: 8,
},
ANALYTICAL: {
maxExecutionTimeSec: 60,
maxResultRows: 500_000,
maxRowsToRead: 2_000_000_000,
maxBytesToRead: 32 * 1024 * 1024 * 1024,
maxMemoryUsageBytes: 8 * 1024 * 1024 * 1024,
maxConcurrentQueries: 4,
},
BATCH: {
maxExecutionTimeSec: 900,
maxResultRows: 10_000_000,
maxRowsToRead: 50_000_000_000,
maxBytesToRead: 512 * 1024 * 1024 * 1024,
maxMemoryUsageBytes: 24 * 1024 * 1024 * 1024,
maxConcurrentQueries: 2,
},
};
/**
* A CL0UD-issued quota may only ever TIGHTEN a tier, never loosen it. A
* control-plane bug that hands out a generous number must not be able to
* disarm the engine's own ceiling.
*/
export function applyQuota(base: ResourceBudget, quota: Partial<ResourceBudget> | undefined): ResourceBudget {
if (!quota) return { ...base };
const pick = (b: number, q: number | undefined): number =>
typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b;
return {
maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec),
maxResultRows: pick(base.maxResultRows, quota.maxResultRows),
maxRowsToRead: pick(base.maxRowsToRead, quota.maxRowsToRead),
maxBytesToRead: pick(base.maxBytesToRead, quota.maxBytesToRead),
maxMemoryUsageBytes: pick(base.maxMemoryUsageBytes, quota.maxMemoryUsageBytes),
maxConcurrentQueries: pick(base.maxConcurrentQueries, quota.maxConcurrentQueries),
};
}
/**
* The settings that ride along on the HTTP request. `readonly: "1"` is what
* makes them un-overridable from inside the query text.
*
* `timeout_overflow_mode` / `result_overflow_mode` / `read_overflow_mode` are
* set to `throw` EXPLICITLY: ClickHouse's default for some of these is
* `break`, which silently returns a PARTIAL result. A partial result that
* looks complete is the analytics equivalent of a false green, so the engine
* refuses it.
*/
export function clickhouseSettings(budget: ResourceBudget): Record<string, string> {
return {
readonly: "1",
max_execution_time: String(budget.maxExecutionTimeSec),
timeout_overflow_mode: "throw",
max_result_rows: String(budget.maxResultRows),
result_overflow_mode: "throw",
max_rows_to_read: String(budget.maxRowsToRead),
max_bytes_to_read: String(budget.maxBytesToRead),
read_overflow_mode: "throw",
max_memory_usage: String(budget.maxMemoryUsageBytes),
};
}
/**
* Settings for a PLANNER call (`EXPLAIN ESTIMATE`, `EXPLAIN PLAN`).
*
* Measured, not assumed: running `EXPLAIN ESTIMATE` under the caller's own
* `max_rows_to_read` makes ClickHouse refuse the ESTIMATE itself with code 158
* whenever the estimate would be over budget — i.e. exactly the case the
* estimate exists to detect. An estimator that cannot describe an expensive
* query is useless, so the read ceilings are deliberately NOT applied here.
* The planner still runs `readonly = 1` under a short deadline and a memory
* cap, so this is a narrower envelope, never an unbounded one.
*/
export function planningSettings(budget: ResourceBudget): Record<string, string> {
return {
readonly: "1",
max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)),
timeout_overflow_mode: "throw",
max_memory_usage: String(budget.maxMemoryUsageBytes),
};
}
/** Every setting name BI-P012 requires, for the DoD assertion. */
export const REQUIRED_GOVERNANCE_SETTINGS = [
"max_execution_time",
"max_result_rows",
"max_rows_to_read",
"max_bytes_to_read",
"max_memory_usage",
"max_concurrent_queries",
] as const;
/**
* `max_concurrent_queries` lives on a SETTINGS PROFILE, not on a statement.
* Rendered here so BI-P011's restricted-user bootstrap and BI-P012's budget
* come from one source.
*/
export function concurrencyProfileSql(profileName: string, budget: ResourceBudget): string[] {
return [
`CREATE SETTINGS PROFILE IF NOT EXISTS ${profileName} SETTINGS ` +
`max_execution_time = ${budget.maxExecutionTimeSec}, ` +
`max_result_rows = ${budget.maxResultRows}, ` +
`max_rows_to_read = ${budget.maxRowsToRead}, ` +
`max_bytes_to_read = ${budget.maxBytesToRead}, ` +
`max_memory_usage = ${budget.maxMemoryUsageBytes}, ` +
`readonly = 1`,
`CREATE QUOTA IF NOT EXISTS ${profileName}_quota ` +
`FOR INTERVAL 1 SECOND MAX queries = ${budget.maxConcurrentQueries * 20} ` +
`TO ALL EXCEPT default`,
];
}
/**
* Local admission control. One instance per process; keyed by tenant so a
* noisy tenant is throttled without touching anyone else's slots — which is
* exactly BI-P012's DoD wording ("without degrading unrelated tenant").
*/
export class ConcurrencyGate {
private readonly inFlight = new Map<string, number>();
constructor(private readonly limitFor: (tenantId: string) => number) {}
/** Returns a release function. Throws RATE_LIMITED when the tenant is full. */
acquire(tenantId: string): () => void {
const limit = this.limitFor(tenantId);
const current = this.inFlight.get(tenantId) ?? 0;
if (current >= limit) {
throw new SiaxError(
"RATE_LIMITED",
"0pt.analytics",
`tenant already has ${current} analytics queries in flight (limit ${limit})`,
);
}
this.inFlight.set(tenantId, current + 1);
let released = false;
return () => {
if (released) return;
released = true;
const n = (this.inFlight.get(tenantId) ?? 1) - 1;
if (n <= 0) this.inFlight.delete(tenantId);
else this.inFlight.set(tenantId, n);
};
}
/** Observability: in-flight count for a tenant (0 when idle). */
inFlightFor(tenantId: string): number {
return this.inFlight.get(tenantId) ?? 0;
}
/** Total in-flight across all tenants. */
totalInFlight(): number {
let n = 0;
for (const v of this.inFlight.values()) n += v;
return n;
}
}
+169
View File
@@ -0,0 +1,169 @@
/**
* src/analytics/engine/schema.ts
* ===============================
* BI-P010 — CLICKHOUSE PRODUCTION SCHEMA.
*
* BI-P010's explicit warning is "Avoid one giant unbounded table for every
* analytics use case." The existing `opt_metric_events` (owned by
* src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one
* table for one job: low-cardinality capability metrics. It stays, and it is
* NOT extended into the catch-all.
*
* Four tables, four jobs, four retention profiles:
*
* opt_metric_events (PRE-EXISTING, not defined here) — capability
* metrics, Map(String,String) dimensions, 90d.
* opt_analytics_event — high-volume per-tenant event stream. Wide
* dimension arrays, SHORT TTL (raw events are not
* the serving tier; they age into the rollups).
* opt_metric_daily — AggregatingMergeTree rollup fed by a
* MATERIALIZED VIEW off opt_analytics_event.
* This is what a dashboard time-series reads;
* long TTL, tiny footprint.
* opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED
* dataset: one row per (tenant, dataset, entity),
* ReplacingMergeTree so a refresh is idempotent.
*
* Why the split is load-bearing, not cosmetic:
* * different ORDER BY keys. A dashboard filtering (tenant, metric, day)
* and a projection looked up by (tenant, dataset, entity_key) cannot
* share one primary key without one of them degenerating to a full scan.
* * different TTLs. Raw events must expire fast; rollups must not.
* * different cardinality. `LowCardinality(String)` is a win on
* tenant/metric/dataset and a LOSS on a free-form entity key, so they
* cannot be the same column type in the same table.
*
* EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it
* is what makes the forced tenant predicate (BI-P011) a primary-key range
* scan instead of a filter applied after reading everyone's rows, and it is
* what makes a per-tenant row policy cheap.
*
* LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this
* schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts
* ClickHouse pipelines or quarantined market data.
*/
export interface AnalyticsSchemaOptions {
database: string;
/** Raw event retention. Deliberately short — rollups carry the history. */
eventRetentionDays?: number;
/** Rollup retention. */
rollupRetentionDays?: number;
/** Serving-projection retention. */
projectionRetentionDays?: number;
}
/** Physical tables this module owns (opt_metric_events is NOT one of them). */
export const ANALYTICS_TABLES = [
"opt_analytics_event",
"opt_metric_daily",
"opt_dataset_projection",
] as const;
export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number];
/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */
export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const;
/**
* DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so
* applying it to a live database is idempotent (the same contract
* `ClickHouseAnalytics.ensureSchema` already keeps).
*/
export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] {
const db = opts.database;
const eventTtl = Math.floor(opts.eventRetentionDays ?? 30);
const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095);
const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365);
return [
`CREATE DATABASE IF NOT EXISTS ${db}`,
// ── raw event stream ────────────────────────────────────────────────
// Dimensions as parallel arrays (not Map) so ClickHouse can prune on
// dimension VALUES via the bloom filter index; Map(String,String) forces
// reading the whole map. `event_id` makes deduplication possible on
// replay without a ReplacingMergeTree's merge latency.
`CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
metric LowCardinality(String),
event_id String,
event_time DateTime64(3, 'UTC'),
value Float64,
dim_names Array(LowCardinality(String)),
dim_values Array(String),
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, project_id, dataset_id, metric, event_time)
TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY
SETTINGS index_granularity = 8192`,
// ── daily rollup (serving tier for time series) ─────────────────────
`CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
metric LowCardinality(String),
day Date,
sum_state AggregateFunction(sum, Float64),
count_state AggregateFunction(count, Float64),
min_state AggregateFunction(min, Float64),
max_state AggregateFunction(max, Float64)
)
ENGINE = AggregatingMergeTree()
PARTITION BY toYYYYMM(day)
ORDER BY (tenant_id, project_id, dataset_id, metric, day)
TTL day + INTERVAL ${rollupTtl} DAY`,
`CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv
TO ${db}.opt_metric_daily AS
SELECT
tenant_id,
project_id,
dataset_id,
metric,
toDate(event_time) AS day,
sumState(value) AS sum_state,
countState(value) AS count_state,
minState(value) AS min_state,
maxState(value) AS max_state
FROM ${db}.opt_analytics_event
GROUP BY tenant_id, project_id, dataset_id, metric, day`,
// ── serving projection for a CURATED dataset ────────────────────────
// ReplacingMergeTree(version) so refreshProjection is idempotent: the
// same (tenant, dataset, entity_key) re-ingested at a higher version
// supersedes rather than duplicates.
`CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
entity_key String,
as_of DateTime64(3, 'UTC'),
version UInt64,
attributes Map(String, String),
measures Map(String, Float64),
INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4
)
ENGINE = ReplacingMergeTree(version)
PARTITION BY (tenant_id)
ORDER BY (tenant_id, project_id, dataset_id, entity_key)
TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`,
];
}
/**
* Statement that proves a table's ORDER BY leads with tenant_id — used by the
* BI-P011 test suite so "tenant-first primary key" is asserted against the
* live server, not against this file's own comments.
*/
export function sortingKeySql(db: string, table: string): string {
return (
`SELECT sorting_key FROM system.tables ` +
`WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
);
}
@@ -0,0 +1,180 @@
/**
* src/analytics/engine/siax-analytics-engine.ts
* ==============================================
* BI-P009 — the provider-neutral analytics engine interface.
*
* Eight capabilities, named exactly as BI-P009 lists them:
*
* health executeMetric executeQuery estimateQuery
* explain cancelQuery ingestProjection refreshProjection
*
* ClickHouse is the CANONICAL provider (BI-P009); DuckDB (BI-P014) and Trino
* (BI-P015) implement the same interface later. Nothing above this interface
* may import a provider module directly — that is what makes "replaceable
* provider" true rather than aspirational.
*
* NORMATIVE OWNERSHIP (BI program §0): every method takes an
* `AnalyticsIdentity` that the CALLER cannot forge, because it is produced by
* CL0UD verification upstream, not by request-body parsing. Product repos
* (VehIQ) call an API in front of this interface; they never hold a provider
* credential and never construct an engine.
*/
import type { CompiledQuery, MetricAggregate } from "./query-compiler";
import type { AnalyticsTable } from "./schema";
import type { BudgetTier, ResourceBudget } from "./resource-governance";
/**
* The VERIFIED caller. `tenant_id` here is the one the engine forces into
* every predicate — it comes from CL0UD identity verification (BI-P011 layer
* 1), never from a request body. This estate has already been burned once by
* a tenant read out of a request body (INF0 Wave 2, cross-tenant RAG access),
* which is why the type is separate from the request rather than a field on
* it.
*/
export interface AnalyticsIdentity {
tenant_id: string;
project_id?: string;
/** CL0UD principal, for audit + per-principal quota. */
principal_id: string;
correlation_id?: string;
/** Budget tier granted by CL0UD entitlements (BI-P012). */
budget_tier?: BudgetTier;
/** CL0UD quota, which may only TIGHTEN the tier. */
quota?: Partial<ResourceBudget>;
}
export interface MetricRequest {
metric: string;
aggregate: MetricAggregate;
dataset_id?: string;
group_by?: string[];
filters?: Record<string, string>;
from_time?: string;
to_time?: string;
limit?: number;
table?: AnalyticsTable;
}
export interface AnalyticsRow {
/** Resolved group values, keyed by the CALLER-FACING dimension name. */
groups?: Record<string, string>;
value: number;
}
export interface EngineResult {
rows: AnalyticsRow[];
/** Engine-side id; the handle `cancelQuery` needs. */
query_id: string;
/** True when the governance ceiling cut the result short (BI-P012). */
truncated: boolean;
stats: QueryStats;
}
export interface QueryStats {
duration_ms: number;
rows_read?: number;
bytes_read?: number;
result_rows: number;
}
export interface QueryEstimate {
/** Rows the engine expects to SCAN. */
estimated_rows_to_read: number;
estimated_bytes_to_read: number;
/** False when the estimate exceeds the caller's budget (BI-P012). */
within_budget: boolean;
budget: ResourceBudget;
/** Why it was refused, when `within_budget` is false. */
reason?: string;
}
export interface QueryExplanation {
/** Provider's plan text. Never contains caller-supplied SQL fragments. */
plan: string;
/** The predicate the compiler forced — assertable, not narrated. */
forced_tenant_predicate: string;
table: string;
}
export interface ProjectionRow {
entity_key: string;
as_of?: string;
version?: number;
attributes?: Record<string, string>;
measures?: Record<string, number>;
}
export interface ProjectionIngestRequest {
dataset_id: string;
rows: ProjectionRow[];
}
export interface ProjectionRefreshResult {
dataset_id: string;
/** Rows visible after the refresh settled. */
row_count: number;
refreshed_at: string;
}
export interface HealthReport {
healthy: boolean;
provider: string;
/** Provider version when reachable. */
version?: string;
/** Short operator-facing reason. Never a raw engine body. */
detail?: string;
}
/**
* A raw, ANALYST-facing query (BI-P017 will govern its SQL dialect). It is a
* SEPARATE method from `executeMetric` on purpose: `executeMetric` has no
* raw-SQL surface at all, and keeping them apart means the closed DSL cannot
* silently grow one.
*
* The canonical ClickHouse provider in this pass DELIBERATELY refuses this
* method (`CAPABILITY_UNAVAILABLE`): opening an analyst SQL surface before
* BI-P017's dialect contract and BI-P033's authorization exist would be
* exactly the "raw SQL attempt" BI-P011's adversarial list names. Declaring
* it and refusing it is honest; implementing it early is not.
*/
export interface AnalystQueryRequest {
sql: string;
dataset_ids: string[];
limit?: number;
}
export interface SiaxAnalyticsEngine {
readonly provider: string;
health(): Promise<HealthReport>;
executeMetric(identity: AnalyticsIdentity, request: MetricRequest): Promise<EngineResult>;
executeQuery(identity: AnalyticsIdentity, request: AnalystQueryRequest): Promise<EngineResult>;
estimateQuery(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryEstimate>;
cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise<boolean>;
explain(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryExplanation>;
ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>;
refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise<ProjectionRefreshResult>;
}
/** The eight capability names, for the provider contract test. */
export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [
"health",
"executeMetric",
"executeQuery",
"estimateQuery",
"cancelQuery",
"explain",
"ingestProjection",
"refreshProjection",
] as const;
export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number];
/** Re-exported so provider modules do not reach across into the compiler. */
export type { CompiledQuery };
+183
View File
@@ -0,0 +1,183 @@
/**
* src/analytics/execution/cancellation.ts
* ========================================
* BI-P019 — QUERY CANCELLATION.
*
* "Integrate engine cancellation. DoD: User cancellation stops expensive
* query within defined timeout."
*
* THE THING THAT MAKES THIS REAL: "stops" is VERIFIED, not requested.
*
* The obvious implementation aborts the HTTP request and issues a KILL. Both
* of those are requests. The client abort stops US waiting; it does not stop
* the server, which keeps burning CPU on a query nobody will read. `KILL QUERY`
* without `SYNC` returns immediately and tells you nothing. Even `KILL QUERY
* SYNC` can return while the query is still winding down.
*
* So `cancelAndVerify` does three things and reports on all three:
* 1. abort the client side;
* 2. issue the engine's kill;
* 3. POLL the engine's process list until the query is GONE, or the deadline
* passes — and if the deadline passes, it says so with
* `verified_stopped: false` rather than returning a cheerful `true`.
*
* A caller that treats `cancelQuery()` returning `true` as "the query stopped"
* is the failure this module exists to prevent, which is why the return type
* is an outcome object and not a boolean.
*/
import { SiaxError } from '../../platform/errors';
/** Whatever can list and kill running queries. ClickHouse and Trino both can. */
export interface EngineProcessControl {
readonly engine: string;
/** True when a query with this id is still running on the engine. */
isRunning(queryId: string): Promise<boolean>;
/** Ask the engine to stop it. Returning true means "accepted", not "stopped". */
requestKill(queryId: string): Promise<boolean>;
/** Live progress for the running query, when the engine exposes it (BI-P021). */
progress?(queryId: string): Promise<QueryProgress | undefined>;
}
export interface QueryProgress {
rows_read: number;
bytes_read: number;
elapsed_ms: number;
}
export interface CancellationOutcome {
query_id: string;
requested_at: string;
/** The engine accepted the kill request. */
kill_accepted: boolean;
/**
* The query was OBSERVED to be gone from the engine's process list.
* `false` means it may still be running — the honest answer.
*/
verified_stopped: boolean;
/** Milliseconds from the cancel request to the observation. */
elapsed_ms: number;
/** The deadline this cancellation was held to. */
timeout_ms: number;
/** Present when the query had already finished before the request landed. */
already_finished: boolean;
}
export interface CancellationOptions {
/** The defined timeout BI-P019's DoD refers to. */
timeoutMs?: number;
pollIntervalMs?: number;
/** Aborts the client-side wait. */
abort?: () => void;
now?: () => number;
sleep?: (ms: number) => Promise<void>;
}
export const DEFAULT_CANCELLATION_TIMEOUT_MS = 5_000;
const DEFAULT_POLL_INTERVAL_MS = 100;
const defaultSleep = (ms: number): Promise<void> => new Promise((r) => setTimeout(r, ms));
export async function cancelAndVerify(
control: EngineProcessControl,
queryId: string,
opts: CancellationOptions = {},
): Promise<CancellationOutcome> {
if (!queryId?.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'cancellation requires a query id');
}
const timeoutMs = opts.timeoutMs ?? DEFAULT_CANCELLATION_TIMEOUT_MS;
const pollIntervalMs = opts.pollIntervalMs ?? DEFAULT_POLL_INTERVAL_MS;
const now = opts.now ?? Date.now;
const sleep = opts.sleep ?? defaultSleep;
const startedMs = now();
const requestedAt = new Date(startedMs).toISOString();
// Was it even running? Cancelling a finished query is a no-op, and reporting
// it as a successful stop would overstate what happened.
//
// FAIL CLOSED when the engine cannot be asked: an unreachable engine means
// "unknown", and the safe reading of unknown is "still running" — so the
// kill is still issued and the verification loop still has to observe the
// query gone. Letting this throw would abandon the cancellation entirely at
// the exact moment the operator most needs it to be attempted.
let runningBefore: boolean;
try {
runningBefore = await control.isRunning(queryId);
} catch {
runningBefore = true;
}
// 1. client side: stop waiting.
opts.abort?.();
// 2. engine side: ask it to stop.
let killAccepted = false;
try {
killAccepted = await control.requestKill(queryId);
} catch {
killAccepted = false;
}
if (!runningBefore) {
return {
query_id: queryId,
requested_at: requestedAt,
kill_accepted: killAccepted,
verified_stopped: true,
elapsed_ms: now() - startedMs,
timeout_ms: timeoutMs,
already_finished: true,
};
}
// 3. VERIFY. This loop is the whole point of the module.
let verified = false;
for (;;) {
let stillRunning: boolean;
try {
stillRunning = await control.isRunning(queryId);
} catch {
// An engine we cannot ask is an engine we cannot vouch for. Keep trying
// until the deadline, then report unverified.
stillRunning = true;
}
if (!stillRunning) {
verified = true;
break;
}
if (now() - startedMs >= timeoutMs) break;
await sleep(pollIntervalMs);
}
return {
query_id: queryId,
requested_at: requestedAt,
kill_accepted: killAccepted,
verified_stopped: verified,
elapsed_ms: now() - startedMs,
timeout_ms: timeoutMs,
already_finished: false,
};
}
/**
* The same call, for a caller that wants a failure rather than a report.
* Throws `RETRYABLE_FAILURE` when the query outlived the deadline — because
* "we asked it to stop and it did not" is an operational condition someone
* has to see, not a value to ignore.
*/
export async function cancelOrThrow(
control: EngineProcessControl,
queryId: string,
opts: CancellationOptions = {},
): Promise<CancellationOutcome> {
const outcome = await cancelAndVerify(control, queryId, opts);
if (!outcome.verified_stopped) {
throw new SiaxError(
'RETRYABLE_FAILURE',
`0pt.analytics.${control.engine}`,
`query was not observed to stop within ${outcome.timeout_ms}ms of cancellation`,
);
}
return outcome;
}
@@ -0,0 +1,138 @@
/**
* src/analytics/execution/clickhouse-process-control.ts
* ======================================================
* BI-P019/BI-P021 — the ClickHouse implementation of `EngineProcessControl`.
*
* Three statements, each chosen for a measured reason:
*
* isRunning `SELECT count() FROM system.processes WHERE query_id = …`
* — the authority on whether the server is still working. Note
* that `KILL QUERY SYNC` returning is NOT this: a killed query
* can still appear here briefly while it unwinds, which is
* exactly why cancellation polls instead of trusting the kill.
*
* requestKill `KILL QUERY WHERE query_id = … ASYNC` — ASYNC, not SYNC,
* because the verification loop is what establishes the stop and
* a SYNC kill would block this call for up to the query's
* remaining lifetime while providing no stronger guarantee.
*
* progress `read_rows`, `read_bytes` and `elapsed` from the same table —
* live counters, which is what makes BI-P021's in-flight budget
* enforcement possible at all.
*
* KILL and the system-table reads are ADMINISTRATIVE. They run on the engine's
* own credential, never on the tenant's restricted user (which has no access
* to system.processes — measured: ACCESS_DENIED, code 497). The tenant-scoping
* check therefore happens HERE, on the query id, before any statement is sent.
*/
import { SiaxError } from '../../platform/errors';
import type { EngineProcessControl, QueryProgress } from './cancellation';
export interface ClickHouseProcessControlOptions {
baseUrl: string;
/** "user:password" for the ADMIN credential. From a secret ref. */
basicAuth?: string;
timeoutMs?: number;
/**
* Restricts every operation to query ids carrying this prefix, so a caller
* cannot kill or inspect another tenant's work through this object.
*/
queryIdPrefix?: string;
}
export class ClickHouseProcessControl implements EngineProcessControl {
readonly engine = 'clickhouse';
private readonly baseUrl: string;
private readonly authHeader?: string;
private readonly timeoutMs: number;
private readonly prefix?: string;
constructor(opts: ClickHouseProcessControlOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, '');
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined;
this.timeoutMs = opts.timeoutMs ?? 10_000;
this.prefix = opts.queryIdPrefix;
}
private assertScoped(queryId: string): void {
if (this.prefix && !queryId.startsWith(this.prefix)) {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this scope');
}
if (!/^[A-Za-z0-9_-]{1,128}$/.test(queryId)) {
// The id reaches ClickHouse as a bound parameter, but a malformed id is
// a bug worth surfacing rather than a string worth binding.
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'malformed query id');
}
}
async isRunning(queryId: string): Promise<boolean> {
this.assertScoped(queryId);
const body = await this.send(
'SELECT count() AS n FROM system.processes WHERE query_id = {qid:String} FORMAT JSON',
{ param_qid: queryId },
);
const n = Number((JSON.parse(body) as { data: Array<{ n: string | number }> }).data[0]?.n ?? 0);
return n > 0;
}
async requestKill(queryId: string): Promise<boolean> {
this.assertScoped(queryId);
await this.send(
'KILL QUERY WHERE query_id = {qid:String} ASYNC FORMAT JSON',
{ param_qid: queryId },
false,
);
return true;
}
async progress(queryId: string): Promise<QueryProgress | undefined> {
this.assertScoped(queryId);
const body = await this.send(
'SELECT read_rows, read_bytes, toUInt64(elapsed * 1000) AS elapsed_ms FROM system.processes ' +
'WHERE query_id = {qid:String} FORMAT JSON',
{ param_qid: queryId },
);
const row = (JSON.parse(body) as { data: Array<Record<string, string | number>> }).data[0];
if (!row) return undefined;
return {
rows_read: Number(row.read_rows ?? 0),
bytes_read: Number(row.read_bytes ?? 0),
elapsed_ms: Number(row.elapsed_ms ?? 0),
};
}
private async send(sql: string, params: Record<string, string>, expectJson = true): Promise<string> {
const url = new URL(this.baseUrl);
url.searchParams.set('query', sql);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
// NOT readonly: KILL mutates server state. The other two would tolerate
// readonly, but sharing one path keeps the credential story simple.
url.searchParams.set('max_execution_time', '10');
const headers: Record<string, string> = {};
if (this.authHeader) headers.Authorization = this.authHeader;
let res: Response;
try {
res = await fetch(url.toString(), {
method: 'POST',
headers,
body: '',
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
}
const text = await res.text();
if (!res.ok) {
const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
'PERMANENT_FAILURE',
'clickhouse',
`process-control statement failed (CH code ${code})`,
);
}
if (expectJson && !text.trim()) {
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'empty response from process-control statement');
}
return text;
}
}
@@ -0,0 +1,274 @@
/**
* src/analytics/execution/query-execution-record.ts
* ==================================================
* BI-P018 — QUERY EXECUTION MODEL.
*
* Every field BI-P018 lists, verbatim, and one it lists by its absence:
*
* "Every production query has traceable execution metadata WITHOUT
* unnecessarily storing sensitive raw SQL."
*
* So the record carries `compiled_query_hash` and never the statement. That is
* enforced, not documented: `assertNoRawSql` scans a finished record for the
* statement it describes, and
* tests/analytics/query-execution-model.test.ts fails if any field carries it.
*
* The ONE place raw SQL is retained is the analyst statement history
* (BI-P017), where a privileged human ran unreviewed SQL against tenant data
* and "who ran what" has to survive. That history is a separate record with a
* separate retention posture; this one is the per-execution telemetry that
* every query in the estate produces, and it must be safe to keep for a long
* time and to ship to AUD0.
*
* BI-P002's `QueryExecution` entity is the persisted row; this module is the
* lifecycle that produces it, plus the four fields BI-P018 adds to it
* (compiled_query_hash, datasets, metrics, cache_status, cost_units).
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { ExecutionState } from '../domain/entities';
/** BI-P020's outcome, recorded per execution so cache behaviour is measurable. */
export const CACHE_STATUSES = ['hit', 'miss', 'bypass', 'stale_refused', 'not_cacheable'] as const;
export type CacheStatus = (typeof CACHE_STATUSES)[number];
/** Which surface produced the query — the "user/workload" axis BI-P018 names. */
export const WORKLOAD_KINDS = [
'dashboard',
'report',
'api',
'analyst_sql',
'ai_analyst',
'scheduled',
'internal',
] as const;
export type WorkloadKind = (typeof WORKLOAD_KINDS)[number];
export interface QueryExecutionRecord {
query_id: string;
execution_id: string;
tenant_id: string;
project_id?: string;
/** The CL0UD principal. A human, a service, or a job — never absent. */
principal_id: string;
workload: WorkloadKind;
submitted_at: string;
started_at?: string;
completed_at?: string;
engine: string;
/**
* sha256 of the COMPILED statement plus its bound parameter values.
*
* Parameters are included deliberately: two executions of the same SQL with
* different filters are different work with different cost, and a hash that
* collapsed them would make the cost record meaningless. They are HASHED,
* not stored, so the record still carries no tenant data.
*/
compiled_query_hash: string;
datasets: string[];
metrics: string[];
status: ExecutionState;
/** Rows RETURNED to the caller. */
rows?: number;
/** Rows and bytes SCANNED — the cost axis, not the result size. */
rows_scanned?: number;
bytes_scanned?: number;
duration_ms?: number;
cache_status: CacheStatus;
/** BI-P021 units. Present once the query has settled, including on failure. */
cost_units?: number;
/** A SiaxErrorCode. Never an engine body, never a fragment of the statement. */
error?: string;
correlation_id?: string;
}
/**
* Hash a compiled statement and its bound parameters into the one value the
* record keeps.
*
* Parameters are sorted so two logically identical executions hash equal
* regardless of key iteration order — otherwise the cache-key and cost
* aggregations built on this hash would fragment for no reason.
*/
export function compiledQueryHash(sql: string, params: Record<string, string> = {}): string {
const canonical = JSON.stringify([
sql.trim(),
Object.keys(params)
.sort()
.map((k) => [k, params[k]]),
]);
return crypto.createHash('sha256').update(canonical).digest('hex');
}
/**
* The legal state machine. Written down because "status" fields drift into
* meaninglessness when any value can follow any other — a `succeeded` row that
* later becomes `running` makes every downstream aggregation a lie.
*/
const LEGAL_TRANSITIONS: Record<ExecutionState, readonly ExecutionState[]> = {
queued: ['running', 'cancelled', 'rejected', 'failed'],
running: ['succeeded', 'failed', 'cancelled'],
succeeded: [],
failed: [],
cancelled: [],
rejected: [],
};
export function assertLegalTransition(from: ExecutionState, to: ExecutionState): void {
if (!LEGAL_TRANSITIONS[from].includes(to)) {
throw new SiaxError('CONFLICT', '0pt.analytics', `illegal execution transition ${from} -> ${to}`);
}
}
/** Terminal states: a record in one of these will never change again. */
export const TERMINAL_STATES: readonly ExecutionState[] = ['succeeded', 'failed', 'cancelled', 'rejected'];
export interface ExecutionSink {
write(record: QueryExecutionRecord): Promise<void> | void;
}
export interface ExecutionRecorderOptions {
tenant_id: string;
project_id?: string;
principal_id: string;
workload: WorkloadKind;
engine: string;
correlation_id?: string;
sink: ExecutionSink;
now?: () => number;
}
/**
* Owns one execution's record from submission to a terminal state.
*
* Every transition WRITES. A recorder that only writes at the end loses
* exactly the executions that matter: the ones that never finished.
*/
export class QueryExecutionRecorder {
private record: QueryExecutionRecord;
private readonly sink: ExecutionSink;
private readonly now: () => number;
constructor(
opts: ExecutionRecorderOptions,
queryId: string,
compiled: { hash: string; datasets: string[]; metrics: string[] },
) {
if (!opts.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'an execution record requires a verified tenant',
);
}
if (!opts.principal_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'an execution record requires a verified principal',
);
}
this.now = opts.now ?? Date.now;
this.sink = opts.sink;
this.record = {
query_id: queryId,
execution_id: crypto.randomUUID(),
tenant_id: opts.tenant_id,
project_id: opts.project_id,
principal_id: opts.principal_id,
workload: opts.workload,
submitted_at: new Date(this.now()).toISOString(),
engine: opts.engine,
compiled_query_hash: compiled.hash,
datasets: [...compiled.datasets],
metrics: [...compiled.metrics],
status: 'queued',
cache_status: 'miss',
correlation_id: opts.correlation_id,
};
}
get current(): QueryExecutionRecord {
return { ...this.record, datasets: [...this.record.datasets], metrics: [...this.record.metrics] };
}
async submitted(): Promise<void> {
await this.sink.write(this.current);
}
async started(): Promise<void> {
assertLegalTransition(this.record.status, 'running');
this.record.status = 'running';
this.record.started_at = new Date(this.now()).toISOString();
await this.sink.write(this.current);
}
/** A cache hit never reached the engine; record it as such, not as a fast query. */
async servedFromCache(rows: number, costUnits = 0): Promise<void> {
assertLegalTransition(this.record.status, 'running');
this.record.status = 'running';
this.record.started_at = this.record.started_at ?? new Date(this.now()).toISOString();
this.record.cache_status = 'hit';
await this.finish('succeeded', { rows, rows_scanned: 0, bytes_scanned: 0, cost_units: costUnits });
}
setCacheStatus(status: CacheStatus): void {
this.record.cache_status = status;
}
async finish(
status: ExecutionState,
stats: {
rows?: number;
rows_scanned?: number;
bytes_scanned?: number;
cost_units?: number;
error?: string;
},
): Promise<QueryExecutionRecord> {
assertLegalTransition(this.record.status, status);
const completedMs = this.now();
this.record.status = status;
this.record.completed_at = new Date(completedMs).toISOString();
const startedMs = this.record.started_at
? Date.parse(this.record.started_at)
: Date.parse(this.record.submitted_at);
this.record.duration_ms = Math.max(0, completedMs - startedMs);
if (stats.rows !== undefined) this.record.rows = stats.rows;
if (stats.rows_scanned !== undefined) this.record.rows_scanned = stats.rows_scanned;
if (stats.bytes_scanned !== undefined) this.record.bytes_scanned = stats.bytes_scanned;
if (stats.cost_units !== undefined) this.record.cost_units = stats.cost_units;
if (stats.error !== undefined) this.record.error = stats.error;
assertNoRawSql(this.record);
await this.sink.write(this.current);
return this.current;
}
}
/**
* Refuse to emit a record that carries statement text.
*
* Checked at runtime rather than trusted to the type, because the fields that
* leak are the string ones a future change widens — `error` becoming an engine
* body is the obvious one, and ClickHouse's error text echoes the query.
*/
export function assertNoRawSql(record: QueryExecutionRecord): void {
const suspicious = /\b(select|insert|update|delete|drop|create|alter|from|where|join|union)\b/i;
const stringFields: Array<[string, string | undefined]> = [
['error', record.error],
['query_id', record.query_id],
['compiled_query_hash', record.compiled_query_hash],
['correlation_id', record.correlation_id],
...record.datasets.map((d, i) => [`datasets[${i}]`, d] as [string, string]),
...record.metrics.map((m, i) => [`metrics[${i}]`, m] as [string, string]),
];
for (const [name, value] of stringFields) {
if (value && suspicious.test(value)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`execution record field "${name}" looks like SQL; BI-P018 stores a hash, never the statement`,
);
}
}
}
@@ -0,0 +1,143 @@
/**
* src/analytics/ingestion/analytics-ingestion-service.ts
* =======================================================
* BI-P129: one entry point that takes an `AnalyticsIngestionSpec` and lands
* rows in a canonical analytics tier, whichever provider supplies the
* connector.
*
* The sequence, and why it is this sequence:
*
* 1. plan — every gate checkable without touching the source
* (tier, physical home, licence, provider readiness).
* Cheap checks first: a spec that can never be legal
* should not open a session to find that out.
* 2. connect+discover— ask the source what it actually looks like.
* 3. landability — the discovered shape must satisfy the BI-P004
* contract. This is the check that cannot be done at
* step 1, and doing it here is the whole reason the
* service exists rather than a bare `planX()` call.
* 4. run — hand the connector to the EXISTING P062 runner with
* a landing sink. The runner owns paging, envelopes,
* schema conformance, quality gating, the idempotency
* ledger and the durable checkpoint. None of that is
* re-implemented here.
* 5. disconnect — always, including on failure.
*
* Step 4 is worth spelling out because it is where a "new ingestion system"
* would normally get built by accident. There is no second runner. The
* analytics plane contributes a TARGET and four GATES; the highway is the one
* that already exists and already has crash/resume tests.
*/
import { SiaxError } from '../../platform/errors';
import { IngestionRunner, type IngestionRunReport } from '../../ingestion/ingestion-runner';
import type { IngestionRepository } from '../../repositories/ingestion-repository';
import type { SchemaRegistry } from '../../schema/schema-registry';
import type { LineageStore } from '../../lineage/lineage-store';
import type { QualityEnforcementPolicy } from '../../quality/quality-policy';
import type { RecordSchemaFields } from '../../schema/record-schema';
import {
planAnalyticsIngestion,
type AnalyticsConnectorProvider,
type AnalyticsIngestionSpec,
type PlannedAnalyticsIngestion,
} from './connector-provider';
import { assertDiscoveryLandable, landingSink, type LandingWriter } from './landing';
export interface AnalyticsIngestionDeps {
repo: IngestionRepository;
schemas: SchemaRegistry;
writer: LandingWriter;
lineage?: LineageStore;
/** Runtime environment name; "production" turns gate 4 on. */
runtimeEnv: string;
}
export interface AnalyticsIngestionResult {
plan: PlannedAnalyticsIngestion;
/** What discovery reported, per partition — recorded so a later drift is provable. */
discovered: Record<string, RecordSchemaFields>;
run: IngestionRunReport;
}
export interface AnalyticsIngestionOptions {
run_id?: string;
page_size?: number;
max_pages?: number;
partition_key?: string;
quality_policy?: QualityEnforcementPolicy;
correlation_id?: string;
}
export class AnalyticsIngestionService {
constructor(private readonly deps: AnalyticsIngestionDeps) {}
/** Gates 1, 3 and 4 — no IO, no session. Safe to call from a UI pre-flight. */
plan(provider: AnalyticsConnectorProvider, spec: AnalyticsIngestionSpec): PlannedAnalyticsIngestion {
return planAnalyticsIngestion(provider, spec, this.deps.runtimeEnv);
}
async ingest(
provider: AnalyticsConnectorProvider,
spec: AnalyticsIngestionSpec,
options: AnalyticsIngestionOptions = {},
): Promise<AnalyticsIngestionResult> {
const plan = this.plan(provider, spec);
const runId = options.run_id ?? `run_${spec.source_id}_${Date.now()}`;
const connector = plan.connector;
const discovered: Record<string, RecordSchemaFields> = {};
await connector.connect(spec.config, {
tenant_id: spec.tenant_id,
project_id: spec.project_id,
source_id: spec.source_id,
run_id: runId,
});
try {
const discovery = await connector.discover();
for (const partition of discovery.partitions) {
if (options.partition_key && partition.partition_key !== options.partition_key) continue;
discovered[partition.partition_key] = partition.schema;
}
// Gate 2 — the one that needs the source to have answered.
assertDiscoveryLandable(spec.target, discovered);
} finally {
// The runner opens its own session; this discovery session is ours to close.
await connector.disconnect();
}
const runner = new IngestionRunner(this.deps.repo, this.deps.schemas, this.deps.lineage);
const run = await runner.run({
source_id: spec.source_id,
tenant_id: spec.tenant_id,
project_id: spec.project_id,
mode: spec.mode,
connector,
config: spec.config,
sink: landingSink({
target: spec.target,
tenant_id: spec.tenant_id,
project_id: spec.project_id,
run_id: runId,
writer: this.deps.writer,
}),
run_id: runId,
page_size: options.page_size,
max_pages: options.max_pages,
partition_key: options.partition_key,
quality_policy: options.quality_policy,
output_ref: `0pt:dataset:${spec.target.dataset_id}`,
correlation_id: options.correlation_id,
});
if (run.status === 'failed') {
throw new SiaxError(
'RETRYABLE_FAILURE',
'0pt.analytics.ingestion',
`ingestion run ${run.run_id} for dataset ${spec.target.dataset_id} failed: ` +
`${run.failure?.code ?? 'unknown'} ${run.failure?.detail ?? ''}`.trim(),
);
}
return { plan, discovered, run };
}
}
@@ -0,0 +1,442 @@
/**
* src/analytics/ingestion/connector-provider.ts
* ==============================================
* BI-P129 — ANALYTICS DATA INGESTION.
*
* BI-P129 names four provider kinds for 0pt's `ConnectorProvider`:
*
* native — connectors 0pt implements and owns (P061 contract)
* dlt — the dlt Python framework (BI-P130)
* Meltano/Singer — the Singer tap ecosystem (BI-P131)
* Airbyte — "only optional where legal/license architecture accepts it"
*
* WHAT THIS FILE IS. It is the seam between 0pt's EXISTING ingestion highway
* (`src/connectors/connector.ts` P061 + `src/ingestion/ingestion-runner.ts`
* P062) and the ANALYTICS tier/contract model (BI-P003 tiers, BI-P004
* `DatasetContract`). It deliberately does NOT define a second connector
* interface: a provider's job is to hand back a P061 `Connector`, and the
* runner that already exists then does the reading, checkpointing and
* quality gating. A parallel ingestion stack would be exactly the "do not
* define parallel models" failure this programme keeps paying for.
*
* WHAT IT ADDS, which the P061 stack has no opinion about: an ingestion run
* must LAND somewhere, and where it lands is an analytics contract question.
* Four gates, all fail-closed, all attacked in
* tests/analytics/analytics-ingestion-providers.test.ts:
*
* 1. assertLandingTierIngestible — source data may land in RAW, or in
* NORMALIZED with a declared normalization. It may NEVER land directly
* in CURATED or SERVING. BI-P003's rule ("no business dashboard reads
* arbitrary raw ingestion tables by default") is trivially defeated by
* relabelling the raw landing table SERVING; this gate is what stops
* that, because the tier is checked at the point of WRITE, not only at
* the point of read.
*
* 2. assertContractCoversDiscovery — every field the connector actually
* discovered must be declared in the target `DatasetContract`, with a
* compatible type. An UNDECLARED field is refused rather than landed:
* an undeclared column is an unclassified column, and an unclassified
* column carrying personal data is a GDPR incident that every freshness
* and quality check would still report green.
*
* 3. assertPhysicalHomeMatchesTier — a RAW/NORMALIZED landing goes to
* ST0RE Parquet/Iceberg (the shapes st0re#45 standardised), never
* straight into ClickHouse. Writing ingestion output into the serving
* engine skips the lake, and with it every replay, backfill and
* erasure path that depends on the files existing.
*
* 4. assertProviderRunnable — a provider whose own `providerReadiness()`
* says `verified: false` cannot execute in a production runtime without
* a recorded exception naming a human. This is the structural version
* of the honesty rule `0pt#35` set with MetricFlow: an unverified
* provider may exist, may be planned, may be tested — but it cannot
* quietly become production capability because someone forgot it was
* unverified.
*
* ARCHITECTURE BOUNDARY (BI programme §0): 0pt owns ingestion into analytics
* tiers. A product repo declares what the rows MEAN through
* `BIApplicationBinding`; it never runs its own ingestion into these tiers
* and never receives ClickHouse credentials.
*/
import { SiaxError } from '../../platform/errors';
import type { Connector, ConnectorConfig, ValidationResult } from '../../connectors/connector';
import type { RecordSchemaFields, FieldType } from '../../schema/record-schema';
import type { IngestionMode } from '../../types/ingestion';
import type { AnalyticsFieldType } from '../domain/entities';
import type { AnalyticsTier } from '../domain/tiers';
import type { DatasetContract } from '../domain/contract';
import type { ProviderReadiness } from '../metrics/semantic-provider';
import type { N0dRequiredRole } from '../../clients/n0d-client';
/**
* Deliberately the SAME `ProviderReadiness` shape BI-P029 introduced, not a
* second one. An estate with two readiness vocabularies has none.
*/
export type { ProviderReadiness };
// ── provider kinds ─────────────────────────────────────────────────────────
export const CONNECTOR_PROVIDER_KINDS = ['native', 'dlt', 'singer', 'airbyte'] as const;
export type ConnectorProviderKind = (typeof CONNECTOR_PROVIDER_KINDS)[number];
/**
* BI-P129: "Airbyte only optional where legal/license architecture accepts
* it." That acceptance has NOT been given in this estate — Airbyte's server
* is Elastic License 2.0 (ELv2), which restricts offering the software as a
* managed service to third parties, and 0pt's whole purpose is offering
* managed data capability to customers. So `airbyte` is a declared kind with
* no implementation, and registering one is refused here rather than
* somewhere downstream. When a licence review says otherwise, this constant
* is the single place that changes.
*/
export const PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE: readonly ConnectorProviderKind[] = ['airbyte'];
// ── landing target ─────────────────────────────────────────────────────────
/** Physical formats an analytics landing may use. st0re#45's shapes. */
export const LANDING_FORMATS = ['parquet', 'iceberg'] as const;
export type LandingFormat = (typeof LANDING_FORMATS)[number];
/** Where an ingestion run's output physically goes. */
export interface LandingPhysical {
format: LandingFormat;
/** ST0RE lakehouse namespace. */
namespace: string;
table_name: string;
}
export interface AnalyticsLandingTarget {
dataset_id: string;
tier: AnalyticsTier;
/** The BI-P004 contract the landed rows must satisfy. */
contract: DatasetContract;
physical: LandingPhysical;
/**
* Required when `tier` is NORMALIZED: what normalization was applied
* between the source bytes and the landed rows. Free text, recorded — an
* empty or missing value means the dataset is really RAW wearing a
* NORMALIZED label, and is refused.
*/
normalization?: string;
}
/** One ingestion the analytics plane is asked to perform. */
export interface AnalyticsIngestionSpec {
source_id: string;
tenant_id: string;
project_id: string;
mode: IngestionMode;
provider: ConnectorProviderKind;
/** Provider-specific connector selector (a native connector id, a tap name, a dlt source). */
connector_ref: string;
config: ConnectorConfig;
target: AnalyticsLandingTarget;
/**
* Records the human who accepted running an UNVERIFIED provider in a
* production runtime. Absent = an unverified provider is refused.
*/
unverified_provider_exception?: string;
}
// ── execution plan ─────────────────────────────────────────────────────────
export type ProviderRuntime = 'in_process' | 'n0d';
/**
* Where a provider's work actually runs. BI-P130 says dlt executes on N0D;
* this is the type that carries that decision instead of a comment.
*/
export interface ProviderExecutionPlan {
runtime: ProviderRuntime;
/** N0D job_type; set iff runtime is "n0d". */
job_type?: string;
required_role?: N0dRequiredRole;
/** Named profile from N0D_RESOURCE_PROFILES; set iff runtime is "n0d". */
resource_profile?: string;
/** Operator-facing reason, in plain words. */
reason: string;
}
// ── the port ───────────────────────────────────────────────────────────────
export interface AnalyticsConnectorProvider {
readonly provider: ConnectorProviderKind;
/** What this provider has and has NOT demonstrated. Never optimistic. */
providerReadiness(): ProviderReadiness;
/** Pure, no IO, never throws — the P061 `validate` posture, at spec level. */
validateSpec(spec: AnalyticsIngestionSpec): ValidationResult;
/** The P061 connector this spec ingests through. */
connectorFor(spec: AnalyticsIngestionSpec): Connector;
executionPlan(spec: AnalyticsIngestionSpec): ProviderExecutionPlan;
}
// ── gate 1: landing tier ───────────────────────────────────────────────────
/** Tiers an ingestion run may write into at all. */
export const INGESTIBLE_TIERS: readonly AnalyticsTier[] = ['RAW', 'NORMALIZED'];
export function assertLandingTierIngestible(target: AnalyticsLandingTarget): void {
if (!INGESTIBLE_TIERS.includes(target.tier)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`ingestion may not land directly in ${target.tier} (dataset ${target.dataset_id}): ` +
`CURATED and SERVING are DERIVED tiers, built from a landed RAW/NORMALIZED dataset. ` +
`Landing source data there relabels raw ingestion as dashboard-readable and defeats BI-P003.`,
);
}
if (target.tier === 'NORMALIZED' && !(target.normalization ?? '').trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`dataset ${target.dataset_id} lands in NORMALIZED but declares no normalization — ` +
`that is a RAW dataset wearing a NORMALIZED label (BI-P003)`,
);
}
}
// ── gate 2: contract covers what was discovered ────────────────────────────
/**
* How a connector's wire-level field type maps onto the analytics type
* vocabulary. Conservative on purpose: a connector's `number` may be an
* integer or a float, so it satisfies EITHER analytics numeric type, but
* nothing else. Anything not listed here is a mismatch, not a guess.
*/
const CONNECTOR_TYPE_SATISFIES: Record<FieldType, readonly AnalyticsFieldType[]> = {
string: ['string', 'decimal', 'date', 'timestamp'],
number: ['int64', 'float64', 'decimal'],
boolean: ['bool'],
timestamp: ['timestamp', 'date'],
object: ['json'],
array: ['json'],
};
export interface ContractCoverageProblem {
code: 'undeclared_field' | 'type_mismatch' | 'missing_required_field' | 'nullability_mismatch';
field: string;
detail: string;
}
/**
* Compare what the connector says the source LOOKS LIKE against what the
* contract says the dataset IS. Returns every problem; never throws.
*/
export function contractCoverageProblems(
contract: DatasetContract,
discovered: RecordSchemaFields,
): ContractCoverageProblem[] {
const problems: ContractCoverageProblem[] = [];
const byName = new Map(contract.fields.map((f) => [f.name, f]));
for (const [name, spec] of Object.entries(discovered)) {
const field = byName.get(name);
if (!field) {
problems.push({
code: 'undeclared_field',
field: name,
detail:
`source field "${name}" is not declared in contract ${contract.dataset_id}@${contract.version}; ` +
`an undeclared field is an unclassified field and is refused, not landed`,
});
continue;
}
const satisfies = CONNECTOR_TYPE_SATISFIES[spec.type] ?? [];
if (!satisfies.includes(field.type)) {
problems.push({
code: 'type_mismatch',
field: name,
detail: `source field "${name}" is ${spec.type}; contract declares ${field.type}`,
});
}
// `FieldSpec.required` is the connector-side vocabulary; the contract
// speaks `nullable`. required:false === nullable:true.
if (spec.required === false && field.nullable === false) {
problems.push({
code: 'nullability_mismatch',
field: name,
detail: `source field "${name}" is optional at the source; contract declares it NOT NULL`,
});
}
}
for (const field of contract.fields) {
if (field.nullable === false && !(field.name in discovered)) {
problems.push({
code: 'missing_required_field',
field: field.name,
detail: `contract requires non-null field "${field.name}" but the source does not produce it`,
});
}
}
return problems;
}
export function assertContractCoversDiscovery(
contract: DatasetContract,
discovered: RecordSchemaFields,
): void {
const problems = contractCoverageProblems(contract, discovered);
if (problems.length > 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`source shape does not satisfy dataset contract ${contract.dataset_id}@${contract.version}: ` +
problems.map((p) => `[${p.code}] ${p.detail}`).join('; '),
);
}
}
// ── gate 3: physical home ──────────────────────────────────────────────────
export function assertPhysicalHomeMatchesTier(target: AnalyticsLandingTarget): void {
if (!(LANDING_FORMATS as readonly string[]).includes(target.physical.format)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`landing format "${target.physical.format}" is not a lakehouse format; ` +
`RAW/NORMALIZED land as Parquet or Iceberg in ST0RE (BI-P003, st0re#45)`,
);
}
if (!target.physical.namespace.trim() || !target.physical.table_name.trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`landing target for ${target.dataset_id} is missing a ST0RE namespace/table name`,
);
}
}
// ── gate 4: unverified providers may not run in production ─────────────────
export function assertProviderRunnable(
provider: AnalyticsConnectorProvider,
spec: AnalyticsIngestionSpec,
runtimeEnv: string,
): void {
const readiness = provider.providerReadiness();
if (readiness.verified) return;
const isProduction = runtimeEnv === 'production';
const exception = (spec.unverified_provider_exception ?? '').trim();
if (!isProduction) return;
if (exception.length === 0) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.ingestion',
`provider "${provider.provider}" reports verified: false ` +
`(open: ${readiness.open.join('; ')}) and may not run in production ` +
`without a recorded unverified_provider_exception`,
);
}
}
// ── planning ───────────────────────────────────────────────────────────────
export interface PlannedAnalyticsIngestion {
spec: AnalyticsIngestionSpec;
connector: Connector;
execution: ProviderExecutionPlan;
readiness: ProviderReadiness;
/** The tier the rows land in — echoed so a caller cannot claim a different one later. */
landing_tier: AnalyticsTier;
}
/**
* Run every gate that can be checked WITHOUT touching the source, then hand
* back the connector. `assertContractCoversDiscovery` is deliberately NOT
* here: discovery requires a live session, so it runs in `landingSink`'s
* caller / `AnalyticsIngestionService` once `discover()` has answered.
*/
export function planAnalyticsIngestion(
provider: AnalyticsConnectorProvider,
spec: AnalyticsIngestionSpec,
runtimeEnv: string,
): PlannedAnalyticsIngestion {
if (provider.provider !== spec.provider) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`spec asks for provider "${spec.provider}" but was given "${provider.provider}"`,
);
}
if (PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE.includes(spec.provider)) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.ingestion',
`provider "${spec.provider}" requires a licence/architecture acceptance this estate has not given ` +
`(BI-P129: "Airbyte only optional where legal/license architecture accepts it")`,
);
}
const verdict = provider.validateSpec(spec);
if (!verdict.valid) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`provider "${provider.provider}" rejected the spec: ${verdict.errors.join('; ')}`,
);
}
assertLandingTierIngestible(spec.target);
assertPhysicalHomeMatchesTier(spec.target);
assertProviderRunnable(provider, spec, runtimeEnv);
return {
spec,
connector: provider.connectorFor(spec),
execution: provider.executionPlan(spec),
readiness: provider.providerReadiness(),
landing_tier: spec.target.tier,
};
}
// ── registry ───────────────────────────────────────────────────────────────
/**
* The set of providers a runtime actually has. Registration is explicit:
* there is no discovery-by-filename, because a provider appearing in the
* registry because a file exists is how an unreviewed integration becomes
* live.
*/
export class ConnectorProviderRegistry {
private readonly providers = new Map<ConnectorProviderKind, AnalyticsConnectorProvider>();
register(provider: AnalyticsConnectorProvider): this {
if (this.providers.has(provider.provider)) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.ingestion',
`provider "${provider.provider}" is already registered`,
);
}
if (PROVIDER_KINDS_REQUIRING_LICENCE_ACCEPTANCE.includes(provider.provider)) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.ingestion',
`provider "${provider.provider}" may not be registered: no licence/architecture acceptance exists ` +
`for it in this estate (BI-P129)`,
);
}
this.providers.set(provider.provider, provider);
return this;
}
get(kind: ConnectorProviderKind): AnalyticsConnectorProvider {
const p = this.providers.get(kind);
if (!p) {
throw new SiaxError(
'NOT_FOUND',
'0pt.analytics.ingestion',
`no connector provider registered for kind "${kind}"`,
);
}
return p;
}
has(kind: ConnectorProviderKind): boolean {
return this.providers.has(kind);
}
/** Readiness of every registered provider — the operator's one-screen answer. */
readinessReport(): ProviderReadiness[] {
return [...this.providers.values()].map((p) => p.providerReadiness());
}
}
+160
View File
@@ -0,0 +1,160 @@
/**
* src/analytics/ingestion/landing.ts
* ===================================
* BI-P129: the write side. An ingestion run's envelopes have to become FILES
* in a canonical tier, and this is the only place in the analytics plane that
* decides what those files look like.
*
* WHERE THE BOUNDARY IS. 0pt does not own object storage. ST0RE owns the
* lakehouse (namespaces, Iceberg tables, snapshots — `st0re#45`), and 0pt's
* existing `src/services/iceberg-write-service.ts` is already the client for
* it. So `LandingWriter` is a PORT, not an implementation: it names exactly
* what the analytics plane needs from a lake writer (append a batch to one
* tier-owned table, tell me what landed) and nothing more. The default
* implementation here is in-memory and is used by tests and by dry runs; the
* ST0RE-backed one is the existing service, injected.
*
* WHAT THIS FILE ENFORCES, and why each rule is not decoration:
*
* - Every landed batch carries `tier` and `dataset_id`. A batch of rows
* with no tier is a batch nobody can decide the read rules for later.
*
* - `assertContractCoversDiscovery` (BI-P129 gate 2) runs HERE, at the
* point discovery has actually answered, not at plan time when the
* source has not been asked yet. A gate that runs before the fact it
* checks is available is a gate that checks nothing.
*
* - Deletes are LANDED, not dropped. `IngestionEnvelope` with
* `operation: "delete"` carries `payload: null`; a landing that silently
* skipped it would produce exactly the failure this programme has already
* paid for once — an aggregate that never sees a DELETE and overstates a
* number while every freshness check reports green. Staleness is not
* correctness, and a delete that never lands is not staleness at all,
* it is permanent wrongness.
*
* - The batch records `envelope_id`s, so a replay is detectable downstream.
* The runner's guarantee is at-least-once (documented in
* src/ingestion/ingestion-runner.ts); a landing that pretended otherwise
* would be lying about the pipeline it sits in.
*/
import { SiaxError } from '../../platform/errors';
import type { IngestionEnvelope } from '../../types/ingestion';
import type { RecordSchemaFields } from '../../schema/record-schema';
import type { AnalyticsTier } from '../domain/tiers';
import {
assertContractCoversDiscovery,
type AnalyticsLandingTarget,
type LandingFormat,
} from './connector-provider';
/** One physical batch appended to a tier-owned table. */
export interface LandedBatch {
dataset_id: string;
tier: AnalyticsTier;
format: LandingFormat;
namespace: string;
table_name: string;
tenant_id: string;
project_id: string;
run_id: string;
partition_key: string;
/** Rows written, including tombstones. */
row_count: number;
/** Rows whose payload was null (CDC deletes). Counted separately, never dropped. */
tombstone_count: number;
envelope_ids: string[];
}
/**
* The port. One method: append a batch. Deliberately narrow — every extra
* method here is a piece of ST0RE's contract leaking into 0pt's.
*/
export interface LandingWriter {
append(batch: LandedBatch): Promise<void>;
}
/** Test/dry-run writer. Keeps everything it was given so a test can assert on it. */
export class InMemoryLandingWriter implements LandingWriter {
readonly batches: LandedBatch[] = [];
async append(batch: LandedBatch): Promise<void> {
this.batches.push(batch);
}
get rowsLanded(): number {
return this.batches.reduce((n, b) => n + b.row_count, 0);
}
get tombstonesLanded(): number {
return this.batches.reduce((n, b) => n + b.tombstone_count, 0);
}
}
export interface LandingSinkInput {
target: AnalyticsLandingTarget;
tenant_id: string;
project_id: string;
run_id: string;
writer: LandingWriter;
}
/**
* Build an `IngestionSink` (the runner's own type) that lands envelopes into
* the declared analytics tier.
*
* Note the sink is per-run: the runner commits its checkpoint only AFTER
* this resolves, so a writer that throws leaves the page unconsumed and the
* next run re-reads it. That ordering belongs to the runner and is not
* re-implemented here — it is relied upon.
*/
export function landingSink(input: LandingSinkInput): (envelopes: IngestionEnvelope[]) => Promise<void> {
return async (envelopes: IngestionEnvelope[]): Promise<void> => {
if (envelopes.length === 0) return;
const partitions = new Set(envelopes.map((e) => e.partition_key));
for (const partition of partitions) {
const forPartition = envelopes.filter((e) => e.partition_key === partition);
await input.writer.append({
dataset_id: input.target.dataset_id,
tier: input.target.tier,
format: input.target.physical.format,
namespace: input.target.physical.namespace,
table_name: input.target.physical.table_name,
tenant_id: input.tenant_id,
project_id: input.project_id,
run_id: input.run_id,
partition_key: partition,
row_count: forPartition.length,
tombstone_count: forPartition.filter((e) => e.payload === null).length,
envelope_ids: forPartition.map((e) => e.envelope_id),
});
}
};
}
/**
* BI-P129 gate 2, applied once the connector's `discover()` has answered.
* Called by `AnalyticsIngestionService`; exported so an operator pre-flight
* can run it without starting a run.
*/
export function assertDiscoveryLandable(
target: AnalyticsLandingTarget,
discoveredByPartition: Record<string, RecordSchemaFields>,
): void {
const partitions = Object.keys(discoveredByPartition);
if (partitions.length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`source for dataset ${target.dataset_id} discovered no partitions; there is nothing to land`,
);
}
for (const partition of partitions) {
try {
assertContractCoversDiscovery(target.contract, discoveredByPartition[partition]);
} catch (err) {
const detail = err instanceof Error ? err.message : String(err);
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.ingestion',
`partition "${partition}": ${detail}`,
);
}
}
}
@@ -0,0 +1,97 @@
/**
* src/analytics/ingestion/native-provider.ts
* ===========================================
* BI-P129, provider kind `native`: connectors 0pt implements and owns.
*
* There is nothing to integrate here — the connectors already exist and are
* already conformance-tested (`src/connectors/conformance.ts`, P061). This
* provider's whole job is to select one and report, honestly, that it is
* verified: these connectors run in this repo's CI, in-process, with no
* third-party runtime and no unreviewed dependency.
*
* The two shipped natives are deliberately heterogeneous (that was P061's
* point and it is BI-P129's DoD too — "two real source types"):
*
* http_json — remote HTTP API, opaque server-issued page token, health
* is a network round trip, record count unknown until the
* server says it is done.
* jsonl_file — local file drop, cursor is a line offset the connector
* computes itself, health is a stat(), record count exact,
* and it emits real CDC deletes.
*
* They share no code but the contract, so "both land in a canonical tier"
* is a statement about the SEAM, not about one connector.
*/
import { HttpJsonConnector, type FetchImpl } from '../../connectors/http-json-connector';
import { JsonlFileConnector } from '../../connectors/jsonl-file-connector';
import type { Connector, ValidationResult } from '../../connectors/connector';
import type {
AnalyticsConnectorProvider,
AnalyticsIngestionSpec,
ProviderExecutionPlan,
ProviderReadiness,
} from './connector-provider';
/** Native connector ids this provider can build. */
export const NATIVE_CONNECTOR_IDS = ['http_json', 'jsonl_file'] as const;
export type NativeConnectorId = (typeof NATIVE_CONNECTOR_IDS)[number];
export class NativeConnectorProvider implements AnalyticsConnectorProvider {
readonly provider = 'native' as const;
/**
* Optional injection point so a caller can supply an `HttpJsonConnector`
* with a specific `fetchImpl` (the connector's own documented test seam).
* With nothing supplied, a real connector talking to the configured URL is
* built — this is not a mock fallback.
*/
constructor(private readonly fetchImpl?: FetchImpl) {}
providerReadiness(): ProviderReadiness {
return {
provider: this.provider,
pinned_version: 'in-repo',
verified: true,
proven: [
'both native connectors pass the P061 conformance suite (tests/p0/connector-conformance.test.ts)',
'both land into a canonical analytics tier through the P062 ingestion runner ' +
'(tests/analytics/analytics-ingestion-providers.test.ts)',
'no third-party runtime and no unreviewed dependency is involved',
],
open: [
'only http_json and jsonl_file exist natively; every other source type needs dlt/Singer or a new native connector',
],
};
}
validateSpec(spec: AnalyticsIngestionSpec): ValidationResult {
const errors: string[] = [];
if (!(NATIVE_CONNECTOR_IDS as readonly string[]).includes(spec.connector_ref)) {
errors.push(
`unknown native connector "${spec.connector_ref}" (have: ${NATIVE_CONNECTOR_IDS.join(', ')})`,
);
return { valid: false, errors };
}
// Delegate the config check to the connector itself — it is the authority
// on its own config, and its `validate` is pure and never throws.
const connector = this.connectorFor(spec);
const verdict = connector.validate(spec.config);
errors.push(...verdict.errors);
return { valid: errors.length === 0, errors };
}
connectorFor(spec: AnalyticsIngestionSpec): Connector {
if (spec.connector_ref === 'http_json') {
return this.fetchImpl ? new HttpJsonConnector(this.fetchImpl) : new HttpJsonConnector();
}
if (spec.connector_ref === 'jsonl_file') return new JsonlFileConnector();
throw new Error(`unknown native connector "${spec.connector_ref}"`);
}
executionPlan(_spec: AnalyticsIngestionSpec): ProviderExecutionPlan {
return {
runtime: 'in_process',
reason: 'native connectors are Node code in this repo; they need no separate runtime and no N0D job',
};
}
}
@@ -0,0 +1,170 @@
/**
* src/analytics/materialization/drift-detector.ts
* ================================================
* BI-P105 — is the pre-aggregate still telling the truth?
*
* The comparison is bucket-by-bucket between the aggregate and a recomputation
* from the source over the same window. Three things make it a real detector
* rather than a ceremony:
*
* 1. A MISSING BUCKET IS DRIFT. The nastiest pre-aggregate failure is not a
* wrong value, it is an absent row: late-arriving data, a materialized view
* that was created after the backfill, a refresh that failed silently. Every
* value it DOES hold is correct, so a value-only comparison passes while the
* total is wrong. Set difference is checked first, before any arithmetic.
*
* 2. ZERO BUCKETS IS `insufficient_data`, NEVER `match`. A comparison over an
* empty window finds no discrepancies — the same false green BI-P025 found
* in a quality check over an empty table. The router treats
* `insufficient_data` as "do not serve", not as "fine".
*
* 3. RELATIVE AND ABSOLUTE TOLERANCE BOTH HAVE TO HOLD. A 0.1% tolerance on a
* revenue bucket of 40M is 40 000 kr of invisible error; an absolute-only
* tolerance flags every rounding difference on a large bucket. Requiring
* both means a discrepancy has to be small in kronor AND small in
* proportion.
*/
import type { DriftTolerance, DriftVerdict, MaterializationStatus } from './materialization';
/** One aggregated row, keyed by its dimension tuple. */
export interface AggregateBucket {
/** Stable key for the dimension tuple, e.g. "2026-09-01|SE". */
key: string;
/** metric name -> value. */
values: Record<string, number>;
}
export interface DriftDiscrepancy {
key: string;
metric: string;
source_value: number;
aggregate_value: number;
absolute_delta: number;
/** |delta| / |source|, or Infinity when the source value is 0 and the aggregate is not. */
relative_delta: number;
}
export interface DriftReport {
verdict: DriftVerdict;
checked_at: string;
buckets_compared: number;
/** In the source recomputation, absent from the aggregate. Always drift. */
missing_in_aggregate: string[];
/** In the aggregate, absent from the source. Always drift (deleted rows). */
extra_in_aggregate: string[];
discrepancies: DriftDiscrepancy[];
worst?: DriftDiscrepancy;
tolerance: DriftTolerance;
/**
* What a caller would have been told if it had trusted the aggregate: the
* total the aggregate reports vs the total the source computes, per metric.
* This is the number that makes a drift report actionable — "0.4% off" means
* nothing until it is "1 284 500 kr reported, 1 271 900 kr true".
*/
totals: Record<string, { source: number; aggregate: number; delta: number }>;
}
const MAX_REPORTED_DISCREPANCIES = 50;
function relative(sourceValue: number, delta: number): number {
if (sourceValue === 0) return delta === 0 ? 0 : Number.POSITIVE_INFINITY;
return Math.abs(delta / sourceValue);
}
/**
* Compare a recomputation from source against the pre-aggregate.
*
* `fromSource` is the truth. Both sides must cover the same window; the caller
* is responsible for that, and `missing_in_aggregate` will catch it loudly if
* they do not.
*/
export function detectDrift(args: {
fromSource: AggregateBucket[];
fromAggregate: AggregateBucket[];
tolerance: DriftTolerance;
metrics: string[];
now?: Date;
}): DriftReport {
const { fromSource, fromAggregate, tolerance, metrics } = args;
const checked_at = (args.now ?? new Date()).toISOString();
const sourceByKey = new Map(fromSource.map((b) => [b.key, b]));
const aggByKey = new Map(fromAggregate.map((b) => [b.key, b]));
const missing_in_aggregate = [...sourceByKey.keys()].filter((k) => !aggByKey.has(k)).sort();
const extra_in_aggregate = [...aggByKey.keys()].filter((k) => !sourceByKey.has(k)).sort();
const discrepancies: DriftDiscrepancy[] = [];
let buckets_compared = 0;
const totals: DriftReport['totals'] = {};
for (const m of metrics) totals[m] = { source: 0, aggregate: 0, delta: 0 };
for (const [key, sourceBucket] of sourceByKey) {
for (const m of metrics) totals[m].source += sourceBucket.values[m] ?? 0;
const aggBucket = aggByKey.get(key);
if (!aggBucket) continue;
buckets_compared += 1;
for (const m of metrics) {
const sv = sourceBucket.values[m] ?? 0;
const av = aggBucket.values[m] ?? 0;
const delta = av - sv;
const rel = relative(sv, delta);
if (Math.abs(delta) > tolerance.absolute || rel > tolerance.relative) {
discrepancies.push({
key,
metric: m,
source_value: sv,
aggregate_value: av,
absolute_delta: delta,
relative_delta: rel,
});
}
}
}
for (const bucket of aggByKey.values()) {
for (const m of metrics) totals[m].aggregate += bucket.values[m] ?? 0;
}
for (const m of metrics) totals[m].delta = totals[m].aggregate - totals[m].source;
discrepancies.sort((a, b) => Math.abs(b.absolute_delta) - Math.abs(a.absolute_delta));
// A comparison that compared nothing has not proved anything. This is the
// BI-P025 false green, restated for pre-aggregates.
const verdict: DriftVerdict =
buckets_compared === 0 && missing_in_aggregate.length === 0 && extra_in_aggregate.length === 0
? 'insufficient_data'
: discrepancies.length > 0 || missing_in_aggregate.length > 0 || extra_in_aggregate.length > 0
? 'drifted'
: 'match';
const report: DriftReport = {
verdict,
checked_at,
buckets_compared,
missing_in_aggregate,
extra_in_aggregate,
discrepancies: discrepancies.slice(0, MAX_REPORTED_DISCREPANCIES),
tolerance,
totals,
};
if (discrepancies.length > 0) report.worst = discrepancies[0];
return report;
}
/**
* Fold a drift report into the runtime status. A drifted aggregate is taken
* OUT of service here rather than merely annotated: leaving it `serving` with
* a bad verdict relies on every future reader remembering to check the
* verdict, and one that forgets serves the wrong number.
*
* `insufficient_data` also stops service. It means the check could not prove
* anything, and an unproven aggregate is not a current one.
*/
export function applyDriftReport(status: MaterializationStatus, report: DriftReport): MaterializationStatus {
return {
...status,
state: report.verdict === 'match' ? status.state : 'drifted',
last_drift_check_at: report.checked_at,
last_drift_verdict: report.verdict,
};
}
@@ -0,0 +1,309 @@
/**
* src/analytics/materialization/materialization.ts
* =================================================
* BI-P105 — MATERIALIZATION / PRE-AGGREGATION.
*
* A pre-aggregate is the one optimisation that fails SILENTLY. A slow query is
* a complaint; a pre-aggregate that has drifted from its source answers
* instantly, with confidence, and is wrong — no error, no timeout, nothing in
* a log. So this module is built around one rule:
*
* A MATERIALIZATION SERVES ONLY WHILE IT IS PROVABLY CURRENT AND PROVABLY
* UNDRIFTED. Every unknown routes to the source table.
*
* Six refusal reasons, each of which is a real way the aggregate becomes a
* confident liar:
*
* coverage_miss it does not carry the metric/dimension/grain
* asked for (a filter on a dimension the
* aggregate collapsed cannot be applied after
* the fact, and applying it anyway is how a
* pre-aggregate returns another slice's number)
* watermark_unknown nobody knows how current it is. BI-P026 already
* refuses to render UNKNOWN age as current; the
* same doctrine applies here.
* staleness_exceeded it is older than its own freshness contract
* window_beyond_watermark the question reaches past the newest data the
* aggregate contains — it would silently answer
* with a shorter period
* drift_never_verified it has never been compared to its source
* drift_verification_stale the last comparison is older than the interval
* the spec requires
* drift_detected the last comparison FAILED
*
* `drift_never_verified` is the one people leave out, and it is the one that
* matters most on day one: a brand-new materialization is exactly as unproven
* as a drifted one, and a router that defaults new aggregates to "serving"
* ships the bug on the day the feature launches.
*
* The DoD also asks for measured improvement and documented freshness
* semantics: scripts/bi-p105-materialization-drift.mjs measures both against a
* live ClickHouse and writes evidence/bi-p105/materialization-drift.json,
* including what happens WITHOUT this router (the drifted aggregate served).
*/
import { SiaxError } from '../../platform/errors';
import type { AnalyticsQueryRequest, TimeGrain } from '../contract/query-request';
export const MATERIALIZATION_STRATEGIES = [
'clickhouse_materialized_view',
'clickhouse_projection',
'dbt_incremental',
'cube_preaggregation',
] as const;
export type MaterializationStrategy = (typeof MATERIALIZATION_STRATEGIES)[number];
export const MATERIALIZATION_STATES = ['building', 'serving', 'drifted', 'disabled'] as const;
export type MaterializationState = (typeof MATERIALIZATION_STATES)[number];
/**
* Grains ordered coarsest-last. An aggregate at `day` can answer `day` and
* anything coarser by re-aggregating; it can NEVER answer `hour`, because the
* detail was thrown away when the rows were rolled up.
*/
const GRAIN_ORDER: readonly TimeGrain[] = ['hour', 'day', 'week', 'month', 'quarter', 'year'];
/**
* `week` does not divide `month` and `month` does not divide `quarter` by row
* count, but both roll up cleanly from `day`. Grains that CANNOT be derived by
* re-aggregating a coarser one are listed here so the router refuses rather
* than quietly returning a mis-bucketed series.
*/
function canRollUp(from: TimeGrain, to: TimeGrain): boolean {
if (from === to) return true;
const fi = GRAIN_ORDER.indexOf(from);
const ti = GRAIN_ORDER.indexOf(to);
if (fi < 0 || ti < 0 || ti < fi) return false;
// Weekly buckets do not compose into calendar months/quarters/years.
if (from === 'week' && to !== 'week') return false;
return true;
}
export interface DriftTolerance {
/** Absolute delta allowed per metric per bucket. */
absolute: number;
/** Relative delta allowed (0.001 = 0.1%). Both must hold. */
relative: number;
}
/** What a materialization IS. Declarative; the engine adapters read it. */
export interface MaterializationSpec {
materialization_id: string;
tenant_id: string;
project_id: string;
/** The dataset it pre-aggregates (BI-P003). */
dataset: string;
strategy: MaterializationStrategy;
/** Physical target (ClickHouse table, dbt model, Cube pre-agg name). */
target_ref: string;
metrics: string[];
dimensions: string[];
time_dimension?: string;
grain?: TimeGrain;
/** The freshness contract. Older than this and it stops serving. */
max_staleness_seconds: number;
/** How often the aggregate must be re-proved against its source. */
drift_check_interval_seconds: number;
tolerance: DriftTolerance;
}
export const DRIFT_VERDICTS = ['match', 'drifted', 'insufficient_data'] as const;
export type DriftVerdict = (typeof DRIFT_VERDICTS)[number];
/** Runtime state. Separate from the spec: the spec is config, this is fact. */
export interface MaterializationStatus {
materialization_id: string;
state: MaterializationState;
/**
* The newest source timestamp the aggregate represents. UNDEFINED means
* UNKNOWN, and unknown is never treated as current.
*/
data_watermark?: string;
last_refresh_at?: string;
last_drift_check_at?: string;
last_drift_verdict?: DriftVerdict;
}
export const ROUTING_REFUSALS = [
'coverage_miss',
'watermark_unknown',
'staleness_exceeded',
'window_beyond_watermark',
'drift_never_verified',
'drift_verification_stale',
'drift_detected',
'state_not_serving',
] as const;
export type RoutingRefusal = (typeof ROUTING_REFUSALS)[number];
export type RoutingDecision =
| {
source: 'materialization';
materialization_id: string;
target_ref: string;
/** Reported so a caller can show it; never inferred, always measured. */
staleness_seconds: number;
}
| { source: 'base_table'; refusal: RoutingRefusal; detail: string };
function refuse(refusal: RoutingRefusal, detail: string): RoutingDecision {
return { source: 'base_table', refusal, detail };
}
/**
* Staleness of an aggregate, in seconds, or `undefined` when it cannot be
* computed. Deliberately NOT defaulted to 0: "we could not tell" and "it is
* current" are different facts, and conflating them is the bug.
*/
export function materializationStaleness(status: MaterializationStatus, now: Date): number | undefined {
if (!status.data_watermark) return undefined;
const t = Date.parse(status.data_watermark);
if (Number.isNaN(t)) return undefined;
return Math.max(0, Math.floor((now.getTime() - t) / 1000));
}
/**
* Decide whether a request may be answered from the pre-aggregate.
*
* Fail-closed by construction: the function returns `materialization` only
* after every check has passed, and every unknown is a refusal with a named
* reason. Callers get the reason so it can be logged and counted (BI-P103
* reads the counts).
*/
export function selectQuerySource(args: {
request: AnalyticsQueryRequest;
spec: MaterializationSpec;
status: MaterializationStatus;
now: Date;
}): RoutingDecision {
const { request, spec, status, now } = args;
if (request.dataset !== spec.dataset) {
return refuse('coverage_miss', `materialization covers "${spec.dataset}", not "${request.dataset}"`);
}
if (status.state !== 'serving') {
return refuse('state_not_serving', `materialization state is "${status.state}"`);
}
// ── coverage ────────────────────────────────────────────────────────────
const metrics = new Set(spec.metrics);
for (const m of request.metrics) {
if (!metrics.has(m)) return refuse('coverage_miss', `metric "${m}" is not in the pre-aggregate`);
}
const dimensions = new Set(spec.dimensions);
for (const d of request.dimensions) {
if (!dimensions.has(d)) return refuse('coverage_miss', `dimension "${d}" is not in the pre-aggregate`);
}
for (const f of request.filters) {
if (!dimensions.has(f.dimension)) {
// The aggregate summed across this dimension. Filtering afterwards would
// return a number computed over rows the filter should have excluded.
return refuse(
'coverage_miss',
`filter on "${f.dimension}" cannot be applied: the pre-aggregate has already summed across it`,
);
}
}
const requestedGrain = request.timeRange?.grain;
if (requestedGrain) {
if (!spec.grain) return refuse('coverage_miss', 'the pre-aggregate has no time grain');
if (!canRollUp(spec.grain, requestedGrain)) {
return refuse(
'coverage_miss',
`a "${spec.grain}" pre-aggregate cannot answer at "${requestedGrain}" grain`,
);
}
}
// ── freshness ───────────────────────────────────────────────────────────
const staleness = materializationStaleness(status, now);
if (staleness === undefined) {
return refuse(
'watermark_unknown',
'the pre-aggregate has no data watermark — unknown age is not current age',
);
}
if (staleness > spec.max_staleness_seconds) {
return refuse(
'staleness_exceeded',
`${staleness}s stale, contract allows ${spec.max_staleness_seconds}s`,
);
}
const to = request.timeRange?.to;
if (to) {
const toMs = Date.parse(to);
const watermarkMs = Date.parse(status.data_watermark!);
if (!Number.isNaN(toMs) && toMs > watermarkMs) {
return refuse(
'window_beyond_watermark',
`the question runs to ${to} but the pre-aggregate only contains data to ${status.data_watermark}`,
);
}
}
// ── drift ───────────────────────────────────────────────────────────────
if (!status.last_drift_check_at || !status.last_drift_verdict) {
return refuse(
'drift_never_verified',
'the pre-aggregate has never been compared to its source; a new aggregate is as unproven as a drifted one',
);
}
if (status.last_drift_verdict !== 'match') {
return refuse('drift_detected', `last drift check returned "${status.last_drift_verdict}"`);
}
const checkedAgo = Math.floor((now.getTime() - Date.parse(status.last_drift_check_at)) / 1000);
if (Number.isNaN(checkedAgo)) {
return refuse('drift_never_verified', 'last_drift_check_at is not a timestamp');
}
if (checkedAgo > spec.drift_check_interval_seconds) {
return refuse(
'drift_verification_stale',
`last verified ${checkedAgo}s ago, spec requires every ${spec.drift_check_interval_seconds}s`,
);
}
return {
source: 'materialization',
materialization_id: spec.materialization_id,
target_ref: spec.target_ref,
staleness_seconds: staleness,
};
}
/**
* The routing decision as a freshness envelope for the result (BI-P026 shape):
* a consumer must be able to see WHICH source answered and how old it was.
* A result that does not say is a result nobody can audit.
*/
export function describeRouting(decision: RoutingDecision): {
served_from: 'materialization' | 'base_table';
materialization_id?: string;
staleness_seconds?: number;
refusal?: RoutingRefusal;
detail?: string;
} {
return decision.source === 'materialization'
? {
served_from: 'materialization',
materialization_id: decision.materialization_id,
staleness_seconds: decision.staleness_seconds,
}
: { served_from: 'base_table', refusal: decision.refusal, detail: decision.detail };
}
export function assertSpec(spec: MaterializationSpec): void {
const invalid = (d: string): never => {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.materialization', d);
};
if (!spec.materialization_id?.trim()) invalid('materialization_id is required');
if (!spec.tenant_id?.trim() || !spec.project_id?.trim()) invalid('a materialization is tenant-scoped');
if (spec.metrics.length === 0) invalid('a materialization with no metrics pre-aggregates nothing');
if (spec.max_staleness_seconds <= 0) invalid('max_staleness_seconds must be positive');
if (spec.drift_check_interval_seconds <= 0) {
invalid(
'drift_check_interval_seconds must be positive — an aggregate that is never re-verified is one ' +
'source mutation away from serving wrong numbers forever',
);
}
if (spec.tolerance.absolute < 0 || spec.tolerance.relative < 0) invalid('tolerance must be non-negative');
if (spec.grain && !GRAIN_ORDER.includes(spec.grain)) invalid('grain is not a time grain');
}
+419
View File
@@ -0,0 +1,419 @@
/**
* src/analytics/metrics/metric-model.ts
* ======================================
* BI-P028 — METRIC DOMAIN MODEL, and BI-P031 — METRIC GOVERNANCE.
*
* BI-P028 names a canonical metric with fifteen fields. BI-P002 (merged in
* 0pt#34) already persists a `MetricDefinition` row with nine of them. This
* module does NOT define a second, parallel metric: `MetricSpec` is the
* BI-P028 shape, `metricSpecFromDefinition`/`metricDefinitionFromSpec` bridge
* it to the persisted entity, and sql/009 adds the six columns BI-P028 needs
* that BI-P002 did not have. A second metric type would be the exact failure
* BI-P028's DoD is about.
*
* ── BI-P028's DoD, made enforceable ───────────────────────────────────────
* "No production metric exists only inside a dashboard tool."
*
* A statement of intent unless something refuses. `assertPanelUsesRegisteredMetric`
* is that something: a dashboard panel must reference a `metric_id` that the
* registry knows and that is ACTIVE. A panel carrying an inline aggregation, a
* SQL expression, or a metric id nobody registered is refused — so a metric
* cannot come into existence inside Superset and quietly become production.
*
* ── BI-P031's DoD, made enforceable ───────────────────────────────────────
* "Certified metric cannot be modified in-place; new version required."
*
* `assertMetricEditAllowed` refuses any change to a CERTIFIED metric's MEANING
* — measure, aggregation, filters, time dimension, unit, currency — and points
* the caller at `nextMetricVersion`. Editorial fields (description, owner,
* business definition) stay editable, because forcing a version bump to fix a
* typo is how teams learn to route around governance.
*
* The TypeScript check is layer 1. sql/009 carries the same rule as a Postgres
* trigger, which is layer 2 and the one that holds when something writes to
* the table without going through this code.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { MetricAggregation, MetricDefinition } from '../domain/entities';
// ── BI-P028 vocabulary ─────────────────────────────────────────────────────
/**
* `type` in BI-P028's field list. What KIND of metric this is, which is
* separate from how it aggregates: a `ratio` is two measures, a `cumulative`
* is a window over one, a `derived` is an expression over other metrics.
*/
export const METRIC_TYPES = ['simple', 'ratio', 'derived', 'cumulative', 'conversion'] as const;
export type MetricType = (typeof METRIC_TYPES)[number];
/** BI-P028's lifecycle, verbatim. */
export const METRIC_STATUSES = ['DRAFT', 'REVIEW', 'ACTIVE', 'DEPRECATED', 'RETIRED'] as const;
export type MetricStatus = (typeof METRIC_STATUSES)[number];
/** BI-P031's certification axis. Orthogonal to lifecycle status, on purpose. */
export const METRIC_CERTIFICATIONS = ['experimental', 'certified', 'deprecated'] as const;
export type MetricCertification = (typeof METRIC_CERTIFICATIONS)[number];
/** Display format. A number without one is a number a dashboard guesses at. */
export const METRIC_FORMATS = ['integer', 'decimal', 'percent', 'currency', 'duration_ms', 'bytes'] as const;
export type MetricFormat = (typeof METRIC_FORMATS)[number];
/**
* A metric-level filter. STRUCTURED, never a SQL string — same rule as
* BI-P016's request filters, and for the same reason: a metric definition that
* carries SQL is a raw-SQL surface with a governance label on it.
*/
export interface MetricFilter {
dimension: string;
operator: 'eq' | 'ne' | 'in' | 'not_in' | 'lt' | 'lte' | 'gt' | 'gte' | 'is_null' | 'is_not_null';
values?: string[];
}
/** BI-P028's canonical metric, field for field. */
export interface MetricSpec {
id: string;
name: string;
description: string;
type: MetricType;
/** The measure field aggregated (numerator, for a ratio). */
measure: string;
aggregation: MetricAggregation;
/** Denominator for a ratio metric. */
denominator?: string;
/** Dimension names this metric may legitimately be sliced by. */
dimensions: string[];
/** Filters baked into the metric's definition. */
filters: MetricFilter[];
/** The dimension time ranges and grains apply to. */
timeDimension: string;
unit?: string;
/** ISO-4217, required when `format` is "currency". */
currency?: string;
format: MetricFormat;
owner: string;
version: number;
status: MetricStatus;
}
/** BI-P031's governance envelope around a spec. */
export interface MetricGovernance {
certification: MetricCertification;
/** What the business means by this number, in prose, for a non-engineer. */
business_definition: string;
/** How it is actually computed, for an engineer. */
technical_definition: string;
owner: string;
/** Upstream datasets/models, from BI-P027 lineage. */
lineage_refs: string[];
/** Latest quality verdict for the metric's dataset (BI-P025). */
quality_status?: 'passing' | 'warning' | 'failing' | 'unknown';
/** Usage, from AUD0's BI-P076 events. Drives deprecation decisions. */
usage_30d?: { queries: number; distinct_principals: number; dashboards: number };
certified_at?: string;
certified_by?: string;
}
export interface GovernedMetric {
spec: MetricSpec;
governance: MetricGovernance;
}
// ── validation ─────────────────────────────────────────────────────────────
const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/;
function invalid(detail: string): never {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.metrics', detail);
}
/**
* Validate a spec. Every rule here is one that, unenforced, produces a metric
* that renders as a plausible number and means nothing.
*/
export function assertValidMetricSpec(spec: MetricSpec): void {
if (!NAME_RE.test(spec.name)) {
invalid(`metric name "${spec.name}" must be a lowercase snake_case identifier`);
}
if (!spec.description?.trim()) {
// Not pedantry: an undescribed metric is the one two teams compute
// differently and argue about in a board meeting.
invalid(`metric "${spec.name}" has no description`);
}
if (!spec.owner?.trim()) invalid(`metric "${spec.name}" has no owner`);
if (!spec.timeDimension?.trim()) {
invalid(
`metric "${spec.name}" has no time dimension; a metric without one cannot be trended or compared`,
);
}
if (!(METRIC_TYPES as readonly string[]).includes(spec.type)) invalid(`unknown metric type "${spec.type}"`);
if (!(METRIC_STATUSES as readonly string[]).includes(spec.status))
invalid(`unknown metric status "${spec.status}"`);
if (!(METRIC_FORMATS as readonly string[]).includes(spec.format))
invalid(`unknown metric format "${spec.format}"`);
if (spec.version < 1 || !Number.isInteger(spec.version))
invalid('metric version must be a positive integer');
if (spec.type === 'ratio' && !spec.denominator) {
invalid(`ratio metric "${spec.name}" has no denominator`);
}
if (spec.type !== 'ratio' && spec.denominator) {
invalid(`metric "${spec.name}" is not a ratio but carries a denominator`);
}
if (spec.format === 'currency' && !spec.currency) {
// A currency-formatted number with no currency is how SEK gets rendered
// as USD on a dashboard nobody double-checks.
invalid(`metric "${spec.name}" is currency-formatted but names no currency`);
}
if (spec.currency && !/^[A-Z]{3}$/.test(spec.currency)) {
invalid(`metric "${spec.name}" currency must be an ISO-4217 code`);
}
if (spec.format !== 'currency' && spec.currency) {
invalid(`metric "${spec.name}" names a currency but is not currency-formatted`);
}
if (new Set(spec.dimensions).size !== spec.dimensions.length) {
invalid(`metric "${spec.name}" lists a dimension twice`);
}
for (const f of spec.filters) {
if (!spec.dimensions.includes(f.dimension) && f.dimension !== spec.timeDimension) {
invalid(`metric "${spec.name}" filters on "${f.dimension}", which is not one of its dimensions`);
}
const needsValues = !['is_null', 'is_not_null'].includes(f.operator);
if (needsValues && (!f.values || f.values.length === 0)) {
invalid(`metric "${spec.name}" filter on "${f.dimension}" has no values`);
}
if (!needsValues && f.values?.length) {
invalid(`metric "${spec.name}" filter operator "${f.operator}" takes no values`);
}
}
}
/**
* The hash that makes "did the meaning change?" answerable.
*
* Deliberately covers ONLY the semantic fields. Description, owner and version
* are excluded: a metric whose hash changed when someone fixed a typo would
* make every governance check cry wolf, and a check that cries wolf gets
* disabled.
*/
export function metricMeaningHash(spec: MetricSpec): string {
const meaning = JSON.stringify([
spec.type,
spec.measure,
spec.aggregation,
spec.denominator ?? '',
[...spec.dimensions].sort(),
[...spec.filters]
.map((f) => [f.dimension, f.operator, [...(f.values ?? [])].sort()])
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
spec.timeDimension,
spec.unit ?? '',
spec.currency ?? '',
]);
return crypto.createHash('sha256').update(meaning).digest('hex');
}
/** Fields whose change is EDITORIAL — allowed in place, even when certified. */
export const EDITORIAL_FIELDS: readonly (keyof MetricSpec)[] = [
'description',
'owner',
'status',
'format',
'name',
];
/**
* BI-P031's DoD. Refuses an in-place change to a CERTIFIED metric's meaning.
*
* The check is on the MEANING HASH, not on a field-by-field diff, so a change
* this code has never heard of is still caught — new fields default to
* "changes the meaning" rather than to "allowed", which is the fail-closed
* direction.
*/
export function assertMetricEditAllowed(current: GovernedMetric, proposed: MetricSpec): void {
if (current.spec.id !== proposed.id) {
invalid('a metric edit must not change the metric id');
}
if (current.governance.certification !== 'certified') return;
if (proposed.version !== current.spec.version) {
// A version bump is the sanctioned path; it is not an "edit".
return;
}
if (metricMeaningHash(current.spec) !== metricMeaningHash(proposed)) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.metrics',
`metric "${current.spec.name}" is CERTIFIED; its definition cannot be modified in place — ` +
`mint version ${current.spec.version + 1} instead`,
);
}
}
/**
* Mint the next version. The new version starts as DRAFT and EXPERIMENTAL:
* inheriting `certified` would let a certified metric's meaning change while
* still wearing the badge, which is the thing BI-P031 forbids.
*/
export function nextMetricVersion(
current: GovernedMetric,
changes: Partial<MetricSpec>,
reason: string,
): { spec: MetricSpec; governance: MetricGovernance; change_reason: string } {
if (!reason?.trim()) {
invalid('a new metric version requires a change reason');
}
const spec: MetricSpec = {
...current.spec,
...changes,
id: current.spec.id,
version: current.spec.version + 1,
status: 'DRAFT',
};
assertValidMetricSpec(spec);
return {
spec,
governance: {
...current.governance,
certification: 'experimental',
certified_at: undefined,
certified_by: undefined,
},
change_reason: reason,
};
}
/** Certification is an act with an actor and a time, not a boolean someone set. */
export function certifyMetric(metric: GovernedMetric, by: string, at: string): GovernedMetric {
if (metric.spec.status !== 'ACTIVE') {
invalid(`only an ACTIVE metric can be certified; "${metric.spec.name}" is ${metric.spec.status}`);
}
if (!metric.governance.business_definition?.trim() || !metric.governance.technical_definition?.trim()) {
invalid(
`metric "${metric.spec.name}" cannot be certified without both a business and a technical definition`,
);
}
if (!by?.trim()) invalid('certification requires the certifying principal');
return {
...metric,
governance: { ...metric.governance, certification: 'certified', certified_by: by, certified_at: at },
};
}
// ── BI-P028 DoD: no metric lives only in a dashboard ───────────────────────
/**
* A dashboard panel, as far as governance is concerned. It may name a metric.
* It may not BE one.
*/
export interface DashboardPanelRef {
panel_id: string;
metric_id?: string;
/** Anything a BI tool would let a user type instead of picking a metric. */
inline_expression?: string;
inline_aggregation?: string;
inline_sql?: string;
}
export interface MetricLookup {
(metricId: string): GovernedMetric | undefined;
}
export function assertPanelUsesRegisteredMetric(
panel: DashboardPanelRef,
lookup: MetricLookup,
): GovernedMetric {
for (const field of ['inline_expression', 'inline_aggregation', 'inline_sql'] as const) {
if (panel[field]) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.metrics',
`panel "${panel.panel_id}" defines a metric inline via "${field}" — BI-P028: no production metric ` +
`may exist only inside a dashboard tool. Register it and reference its metric_id.`,
);
}
}
if (!panel.metric_id) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.metrics',
`panel "${panel.panel_id}" references no metric_id`,
);
}
const metric = lookup(panel.metric_id);
if (!metric) {
throw new SiaxError(
'NOT_FOUND',
'0pt.analytics.metrics',
`panel "${panel.panel_id}" references unregistered metric "${panel.metric_id}"`,
);
}
if (metric.spec.status === 'RETIRED') {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.metrics',
`panel "${panel.panel_id}" references RETIRED metric "${metric.spec.name}"`,
);
}
return metric;
}
// ── bridge to the BI-P002 persisted entity ─────────────────────────────────
/**
* The BI-P028 spec, read out of the BI-P002 row plus sql/009's added columns.
* Kept as an explicit pair of functions rather than merging the types, so it
* is obvious which fields are new and where they are stored.
*/
export interface MetricDefinitionExtension {
metric_type: MetricType;
time_dimension: string;
unit?: string;
currency?: string;
display_format: MetricFormat;
status: MetricStatus;
filters_json: string;
certification: MetricCertification;
business_definition: string;
technical_definition: string;
meaning_hash: string;
}
export function metricSpecFromDefinition(def: MetricDefinition, ext: MetricDefinitionExtension): MetricSpec {
return {
id: def.metric_id,
name: def.name,
description: def.description ?? '',
type: ext.metric_type,
measure: def.measure_field,
aggregation: def.aggregation,
denominator: def.denominator_field,
dimensions: [...def.allowed_dimension_ids],
filters: JSON.parse(ext.filters_json) as MetricFilter[],
timeDimension: ext.time_dimension,
unit: ext.unit,
currency: ext.currency,
format: ext.display_format,
owner: def.owner_id,
version: def.current_version,
status: ext.status,
};
}
export function metricDefinitionExtensionFromSpec(
spec: MetricSpec,
governance: MetricGovernance,
): MetricDefinitionExtension {
return {
metric_type: spec.type,
time_dimension: spec.timeDimension,
unit: spec.unit,
currency: spec.currency,
display_format: spec.format,
status: spec.status,
filters_json: JSON.stringify(spec.filters),
certification: governance.certification,
business_definition: governance.business_definition,
technical_definition: governance.technical_definition,
meaning_hash: metricMeaningHash(spec),
};
}
+291
View File
@@ -0,0 +1,291 @@
/**
* src/analytics/metrics/semantic-provider.ts
* ===========================================
* BI-P029 — METRICFLOW PROVIDER.
*
* ── WHAT IS AND IS NOT PROVEN HERE. READ THIS BEFORE CITING BI-P029. ──────
*
* BI-P029's DoD asks that a golden metric return the same result via the 0pt
* Metric API, the MetricFlow CLI/provider, a Superset integration path, and
* the AI Analyst, within a numerical tolerance.
*
* MetricFlow is a Python package. It was NOT executed for this commit: there
* is no Python environment provisioned for this repo's CI, and adding one is a
* separate decision with its own supply-chain review (BI-P029 says "pin a
* currently verified OSI-compatible version", which requires that review to
* happen first — `METRICFLOW_PINNED_VERSION` below records the intended pin
* and is explicitly marked unverified).
*
* So what IS proven, in tests/integration/semantic-parity.integration.test.ts
* — a path this header cited from the day it was written, while the file
* itself did not exist until BI-P029's gap review; the citation was the
* evidence, which is no evidence at all — is the part that does not need
* MetricFlow: the SAME golden metric, taken
* from the SAME registry version, executed against the SAME live ClickHouse
* data by TWO INDEPENDENT compilers — 0pt's metric compiler, and a renderer
* driven only by the engine-neutral `SemanticQuerySpec` — returns the same
* number. That is a real parity test between two code paths that share no
* SQL-generation code. It is NOT a test of MetricFlow, and this file does not
* claim to be one.
*
* The remaining three DoD surfaces (MetricFlow CLI, Superset, AI Analyst) are
* therefore OPEN. Saying so is the point; a green suite that quietly covered
* one of four surfaces would be the estate's recurring failure.
*/
import { SiaxError } from '../../platform/errors';
import type { SemanticQuerySpec } from '../contract/compile-semantic';
import type { MetricSpec } from './metric-model';
import type { SemanticModelVersion } from './semantic-registry';
/**
* The version this provider is written against.
*
* LICENCE BOUNDARY — the reason this is 0.209.0 and not 0.208.1.
*
* MetricFlow was relicensed by dbt Labs at 0.209.0. Verified against the
* LICENSE file at each git tag, 2026-09-07, not inferred from PyPI metadata
* (PyPI reports no licence field for either version, so a metadata check
* would have found nothing):
*
* v0.208.1 Business Source License 1.1 — source-available, NOT open source
* v0.209.0 Apache License 2.0
*
* The previous pin sat exactly one patch below that boundary. For a
* self-hosted platform whose premise is no proprietary runtime dependency,
* a BUSL component in the semantic layer is a licence defect, not a
* preference — BUSL restricts production use until its change date.
*
* STILL UNVERIFIED, and deliberately still reported as such: no supply-chain
* review has been run and MetricFlow has never been executed here (no Python
* runtime is provisioned). `providerReadiness()` continues to return
* `verified: false`. Fixing the licence does not make the provider proven.
*/
export const METRICFLOW_PINNED_VERSION = '0.209.0';
/**
* The first MetricFlow release under Apache-2.0. Anything below this is BUSL
* and must not be pinned here; the test asserts the pin stays at or above it.
*/
export const METRICFLOW_FIRST_OPEN_SOURCE_VERSION = '0.209.0';
/** The provider-neutral port. BI-P029's `SiaxSemanticProvider`. */
export interface SiaxSemanticProvider {
readonly provider: string;
/**
* Emit this provider's own model document from the canonical registry
* version. The registry never stores it (BI-P030); it is derived.
*/
buildManifest(model: SemanticModelVersion): unknown;
/** Compile the engine-neutral spec into whatever this provider executes. */
compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery;
/** Canonical SIAX metric id -> this provider's own name for it. */
providerBindings(model: SemanticModelVersion): Record<string, string>;
}
export interface CompiledSemanticQuery {
provider: string;
/** The provider's own query representation. Never SQL text from a caller. */
query: unknown;
/** Canonical metric ids, unchanged — the BI-P030 contract. */
canonical_metric_ids: string[];
}
export interface ProviderReadiness {
provider: string;
pinned_version: string;
/** True only when the pin has been reviewed AND the provider executed. */
verified: boolean;
/** What has actually been demonstrated. Plain words for an operator. */
proven: string[];
/** What has NOT. */
open: string[];
}
/**
* MetricFlow-shaped provider.
*
* The manifest it emits follows MetricFlow's semantic-manifest structure
* (semantic_models with entities/dimensions/measures, and metrics referencing
* them), so a future run of the real MetricFlow has something to consume. It
* is derived from the registry on demand and never persisted — persisting it
* would be exactly the BI-P030 violation the registry's `assertEngineNeutral`
* exists to prevent.
*/
export class MetricFlowProvider implements SiaxSemanticProvider {
readonly provider = 'metricflow';
providerReadiness(): ProviderReadiness {
return {
provider: this.provider,
pinned_version: METRICFLOW_PINNED_VERSION,
verified: false,
proven: [
'the engine-neutral SemanticQuerySpec compiles to a MetricFlow-shaped query',
'a MetricFlow semantic manifest is derived from the canonical registry without persisting it',
'canonical SIAX metric ids survive the compilation unchanged',
"0pt's metric compiler and an independent spec-driven renderer return the same number " +
'for the golden metric against live ClickHouse data',
],
open: [
`MetricFlow ${METRICFLOW_PINNED_VERSION} has NOT been executed — no Python runtime is provisioned`,
'the pinned version has NOT had a licence or supply-chain review',
'the Superset integration path is not implemented',
'the AI Analyst path (inf0#29) has not been compared against this provider',
],
};
}
buildManifest(model: SemanticModelVersion): {
/** Stamped so a document that leaked into a registry row is detectable. */
_generated_by: string;
_generated_from: { model_id: string; version: number; hash: string };
semantic_models: Array<Record<string, unknown>>;
metrics: Array<Record<string, unknown>>;
} {
const semantic_models = model.datasets.map((dataset) => {
const dims = model.dimensions.filter((d) => d.dataset_id === dataset);
const measures = model.metrics
.filter((m) => m.dimensions.some((dim) => dims.some((d) => d.name === dim)))
.map((m) => ({ name: m.measure, agg: mapAggregation(m.aggregation), expr: m.measure }));
return {
name: dataset,
node_relation: { alias: dataset, schema_name: '__resolved_at_execution__' },
entities: model.relationships
.filter((r) => r.from_dataset === dataset)
.map((r) => ({ name: r.from_field, type: r.kind === 'many_to_one' ? 'foreign' : 'primary' })),
dimensions: dims.map((d) => ({
name: d.name,
type: d.kind === 'time' ? 'time' : 'categorical',
...(d.kind === 'time' ? { type_params: { time_granularity: 'day' } } : {}),
})),
measures: dedupeBy(measures, (m) => m.name),
};
});
const metrics = model.metrics.map((m) => ({
// The CANONICAL id is what a consumer holds. The provider's own name is
// derived from it, never the other way round.
name: m.id,
description: m.description,
type: mapMetricType(m.type),
type_params: metricTypeParams(m),
filter: m.filters.length ? { where_filters: m.filters.map(structuredFilter) } : undefined,
meta: { siax_metric_id: m.id, siax_metric_version: m.version, unit: m.unit, currency: m.currency },
}));
return {
_generated_by: `${this.provider}@${METRICFLOW_PINNED_VERSION}`,
_generated_from: { model_id: model.model_id, version: model.version, hash: model.hash },
semantic_models,
metrics,
};
}
compile(spec: SemanticQuerySpec, model: SemanticModelVersion): CompiledSemanticQuery {
const known = new Map(model.metrics.map((m) => [m.id, m]));
for (const ref of spec.metrics) {
if (!known.has(ref.metric_id)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`metric "${ref.metric_id}" is not in semantic model ${model.model_id} v${model.version}`,
);
}
}
const timeBucket = spec.groupBy.find((g) => g.grain);
// Per-metric definition filters (BI-P028 `MetricSpec.filters`), kept
// SEPARATE from the request's `where`. Folding them together would apply
// one metric's meaning to every metric in the query; omitting them would
// make this provider compute a different number from the ClickHouse
// target for the same canonical metric id.
const metric_filters: Record<
string,
Array<{ dimension: string; operator: string; values: string[] }>
> = {};
for (const m of spec.metrics) {
if (!m.definition_filters?.length) continue;
metric_filters[m.metric_id] = m.definition_filters.map((f) => ({
dimension: f.name,
operator: f.operator,
values: [...f.values],
}));
}
return {
provider: this.provider,
query: {
metrics: spec.metrics.map((m) => m.metric_id),
...(Object.keys(metric_filters).length ? { metric_filters } : {}),
group_by: spec.groupBy.filter((g) => !g.grain).map((g) => g.name),
...(timeBucket ? { time_granularity: timeBucket.grain } : {}),
// STRUCTURED, not a `--where` SQL string. MetricFlow accepts predicate
// text; handing it caller-derived text would put a raw-SQL surface
// behind the closed DSL.
where: spec.where.map((w) => ({ dimension: w.name, operator: w.operator, values: w.values })),
order_by: spec.orderBy.map((o) => ({ name: o.ref, descending: o.direction === 'desc' })),
limit: spec.limit,
...(spec.timeConstraint
? { start_time: spec.timeConstraint.start, end_time: spec.timeConstraint.end }
: {}),
},
canonical_metric_ids: spec.metrics.map((m) => m.metric_id).sort(),
};
}
providerBindings(model: SemanticModelVersion): Record<string, string> {
// Identity for MetricFlow, because its metric names can be the canonical
// ids. A provider whose naming rules differ (Cube's cube.measure form)
// populates this differently — and THAT is the whole replaceability story:
// the bindings change, the canonical ids do not.
return Object.fromEntries(model.metrics.map((m) => [m.id, m.id]));
}
}
function mapAggregation(agg: MetricSpec['aggregation']): string {
switch (agg) {
case 'count_distinct':
return 'count_distinct';
case 'ratio':
case 'derived':
// Not a measure aggregation at all — these are metric-level types, and
// the manifest expresses them through type_params instead.
return 'sum';
default:
return agg;
}
}
function mapMetricType(type: MetricSpec['type']): string {
switch (type) {
case 'ratio':
return 'ratio';
case 'derived':
return 'derived';
case 'cumulative':
return 'cumulative';
case 'conversion':
return 'conversion';
default:
return 'simple';
}
}
function metricTypeParams(m: MetricSpec): Record<string, unknown> {
if (m.type === 'ratio') {
return { numerator: m.measure, denominator: m.denominator };
}
return { measure: { name: m.measure } };
}
function structuredFilter(f: MetricSpec['filters'][number]): Record<string, unknown> {
return { dimension: f.dimension, operator: f.operator, values: f.values ?? [] };
}
function dedupeBy<T>(items: T[], key: (t: T) => string): T[] {
const seen = new Set<string>();
return items.filter((i) => {
const k = key(i);
if (seen.has(k)) return false;
seen.add(k);
return true;
});
}
+316
View File
@@ -0,0 +1,316 @@
/**
* src/analytics/metrics/semantic-registry.ts
* ===========================================
* BI-P030 — SEMANTIC MODEL REGISTRY.
*
* "Persist semantic model version SEPARATELY FROM IMPLEMENTATION ENGINE.
* DoD: MetricFlow can be replaced without breaking canonical SIAX metric IDs."
*
* Which means exactly one thing in code, and it is a negative: NOTHING in this
* file may name MetricFlow, Cube, dbt, ClickHouse, Trino or DuckDB. A registry
* that stores a MetricFlow manifest is a MetricFlow registry with a neutral
* name on it. `assertEngineNeutral` enforces the negative at runtime by
* scanning a serialized version for engine vocabulary, and the test asserts it
* catches a manifest that smuggles one in — because the rule is exactly the
* kind that decays silently.
*
* The canonical SIAX metric id is the contract. A provider maps it to whatever
* its own model calls that metric (BI-P029's `providerBindings`), and swapping
* providers changes the BINDINGS, never the ids — so a dashboard, an API
* consumer or a saved query keeps working.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { MetricSpec } from './metric-model';
export const SEMANTIC_MODEL_STATUSES = ['draft', 'active', 'superseded', 'retired'] as const;
export type SemanticModelStatus = (typeof SEMANTIC_MODEL_STATUSES)[number];
export const RELATIONSHIP_KINDS = ['one_to_many', 'many_to_one', 'one_to_one'] as const;
export type RelationshipKind = (typeof RELATIONSHIP_KINDS)[number];
export interface SemanticRelationship {
from_dataset: string;
from_field: string;
to_dataset: string;
to_field: string;
kind: RelationshipKind;
}
export interface SemanticDimension {
id: string;
name: string;
dataset_id: string;
/** "categorical" | "time". A time dimension is what grains apply to. */
kind: 'categorical' | 'time';
}
/** BI-P030's fields, verbatim. */
export interface SemanticModelVersion {
model_id: string;
version: number;
tenant_id: string;
datasets: string[];
relationships: SemanticRelationship[];
/** Canonical SIAX metric ids and their specs at this version. */
metrics: MetricSpec[];
dimensions: SemanticDimension[];
valid_from: string;
status: SemanticModelStatus;
hash: string;
}
/**
* Engine vocabulary that must never appear in a persisted semantic model.
*
* Checked against the SERIALIZED version, so it catches an engine name hiding
* in a nested field somebody added later — the failure mode a type cannot
* prevent.
*/
const ENGINE_VOCABULARY = [
'metricflow',
'clickhouse',
'trino',
'duckdb',
'cube.js',
'cubejs',
'dbt_metric',
'superset',
'presto',
'bigquery',
'snowflake',
];
/** The only top-level keys a persisted semantic model may carry. */
export const SEMANTIC_MODEL_KEYS = [
'model_id',
'version',
'tenant_id',
'datasets',
'relationships',
'metrics',
'dimensions',
'valid_from',
'status',
'hash',
] as const;
/**
* The STRUCTURAL half of engine neutrality, and the load-bearing one.
*
* `assertEngineNeutral` below is a VOCABULARY scan: it catches a model that
* says "metricflow" or "clickhouse". It cannot catch a document that is
* structurally MetricFlow-shaped while avoiding the word — a limitation found
* by a test that expected it to and did not. So publishing also rejects any
* key outside the BI-P030 field list, which means a provider manifest cannot
* be stored in a registry row at all, whatever it calls itself.
*/
export function assertOnlySemanticModelFields(model: object): void {
for (const key of Object.keys(model)) {
if (!(SEMANTIC_MODEL_KEYS as readonly string[]).includes(key)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`semantic model carries field "${key}", which is not part of BI-P030's model — ` +
`provider manifests are DERIVED on demand, never persisted`,
);
}
}
}
export function assertEngineNeutral(model: Omit<SemanticModelVersion, 'hash'>): void {
const serialized = JSON.stringify(model).toLowerCase();
for (const term of ENGINE_VOCABULARY) {
if (serialized.includes(term)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`semantic model "${model.model_id}" mentions "${term}" — BI-P030 requires the model to be ` +
`persisted separately from any implementation engine, so it can be replaced without ` +
`breaking canonical SIAX metric ids`,
);
}
}
}
/**
* The version hash. Covers everything that defines the model's MEANING;
* `valid_from` and `status` are excluded because activating a model does not
* change what it says.
*/
export function semanticModelHash(
model: Omit<SemanticModelVersion, 'hash' | 'valid_from' | 'status'>,
): string {
const canonical = JSON.stringify([
model.model_id,
model.version,
model.tenant_id,
[...model.datasets].sort(),
[...model.relationships]
.map((r) => [r.from_dataset, r.from_field, r.to_dataset, r.to_field, r.kind])
.sort((a, b) => JSON.stringify(a).localeCompare(JSON.stringify(b))),
[...model.metrics].map((m) => [m.id, m.version]).sort((a, b) => String(a[0]).localeCompare(String(b[0]))),
[...model.dimensions].map((d) => [d.id, d.name, d.dataset_id, d.kind]).sort(),
]);
return crypto.createHash('sha256').update(canonical).digest('hex');
}
export interface SemanticModelStore {
put(model: SemanticModelVersion): Promise<void>;
get(tenantId: string, modelId: string, version: number): Promise<SemanticModelVersion | undefined>;
active(tenantId: string, modelId: string): Promise<SemanticModelVersion | undefined>;
versions(tenantId: string, modelId: string): Promise<SemanticModelVersion[]>;
}
export class InMemorySemanticModelStore implements SemanticModelStore {
private readonly rows = new Map<string, SemanticModelVersion>();
private key(t: string, m: string, v: number): string {
return `${t}::${m}::${v}`;
}
async put(model: SemanticModelVersion): Promise<void> {
this.rows.set(this.key(model.tenant_id, model.model_id, model.version), model);
}
async get(t: string, m: string, v: number): Promise<SemanticModelVersion | undefined> {
return this.rows.get(this.key(t, m, v));
}
async versions(t: string, m: string): Promise<SemanticModelVersion[]> {
return [...this.rows.values()]
.filter((r) => r.tenant_id === t && r.model_id === m)
.sort((a, b) => a.version - b.version);
}
async active(t: string, m: string): Promise<SemanticModelVersion | undefined> {
return (await this.versions(t, m)).find((r) => r.status === 'active');
}
}
export class SemanticModelRegistry {
constructor(private readonly store: SemanticModelStore) {}
/**
* Publish a new version. Versions are IMMUTABLE — re-publishing an existing
* (model, version) is a CONFLICT, not an update, because a dashboard pinned
* to version 7 must mean the same thing tomorrow.
*/
async publish(input: Omit<SemanticModelVersion, 'hash'>): Promise<SemanticModelVersion> {
if (!input.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.semantic',
'a semantic model requires a verified tenant',
);
}
if (input.version < 1 || !Number.isInteger(input.version)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.semantic', 'version must be a positive integer');
}
assertOnlySemanticModelFields(input);
assertEngineNeutral(input);
this.assertReferentialIntegrity(input);
const existing = await this.store.get(input.tenant_id, input.model_id, input.version);
if (existing) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.semantic',
`semantic model ${input.model_id} v${input.version} already exists and is immutable`,
);
}
const model: SemanticModelVersion = { ...input, hash: semanticModelHash(input) };
await this.store.put(model);
return model;
}
/** Activate a version, superseding the previous active one. */
async activate(
tenantId: string,
modelId: string,
version: number,
at: string,
): Promise<SemanticModelVersion> {
const target = await this.store.get(tenantId, modelId, version);
if (!target)
throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found');
if (target.status === 'retired') {
throw new SiaxError('CONFLICT', '0pt.analytics.semantic', 'a retired version cannot be re-activated');
}
const current = await this.store.active(tenantId, modelId);
if (current && current.version !== version) {
await this.store.put({ ...current, status: 'superseded' });
}
const activated = { ...target, status: 'active' as const, valid_from: at };
await this.store.put(activated);
return activated;
}
/**
* BI-P030's DoD, as a query: the canonical metric ids a model exposes.
* The engine-parity test asserts this set is identical before and after a
* provider swap.
*/
async canonicalMetricIds(tenantId: string, modelId: string, version: number): Promise<string[]> {
const model = await this.store.get(tenantId, modelId, version);
if (!model)
throw new SiaxError('NOT_FOUND', '0pt.analytics.semantic', 'semantic model version not found');
return model.metrics.map((m) => m.id).sort();
}
/**
* Every dimension and dataset a metric names must exist in the model.
*
* A model that passes validation while referencing a dimension nobody
* defined produces a metric that fails at query time, per tenant, in
* production — the worst place to find out.
*/
private assertReferentialIntegrity(model: Omit<SemanticModelVersion, 'hash'>): void {
const datasets = new Set(model.datasets);
const dimensionNames = new Set(model.dimensions.map((d) => d.name));
const timeDimensions = new Set(model.dimensions.filter((d) => d.kind === 'time').map((d) => d.name));
for (const d of model.dimensions) {
if (!datasets.has(d.dataset_id)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`dimension "${d.name}" belongs to dataset "${d.dataset_id}", which the model does not include`,
);
}
}
for (const r of model.relationships) {
for (const ds of [r.from_dataset, r.to_dataset]) {
if (!datasets.has(ds)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`relationship references dataset "${ds}", which the model does not include`,
);
}
}
}
const seenIds = new Set<string>();
for (const m of model.metrics) {
if (seenIds.has(m.id)) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.semantic',
`metric id "${m.id}" appears twice in the model`,
);
}
seenIds.add(m.id);
for (const dim of m.dimensions) {
if (!dimensionNames.has(dim)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`metric "${m.name}" names dimension "${dim}", which the model does not define`,
);
}
}
if (!timeDimensions.has(m.timeDimension)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.semantic',
`metric "${m.name}" names time dimension "${m.timeDimension}", which the model does not define as a time dimension`,
);
}
}
}
}
+434
View File
@@ -0,0 +1,434 @@
/**
* src/analytics/providers/duckdb-provider.ts
* ==========================================
* BI-P014 — DUCKDB PROVIDER.
*
* Use cases BI-P014 names: Parquet, Iceberg preview, local query, test, CSV,
* ad-hoc files, report build, notebook. Heavy jobs go through N0D.
*
* ── THE SECURITY REQUIREMENT, AND THE MEASURED SURPRISE ───────────────────
* "No uncontrolled filesystem access. Explicit ST0RE-mounted/input artifacts
* only."
*
* DuckDB is a local engine with `read_csv('/etc/passwd')` built in, so this is
* the whole prompt. Two layers, and the second is not what it looks like:
*
* LAYER 1 — MOUNT REGISTRY. A caller names ARTIFACT IDS, never paths. The
* registry resolves an id to a path, checks the artifact belongs to
* the caller's tenant, and `realpath`s the result to confirm it is
* still inside the mount root once symlinks are followed. The query
* builder is the only thing that ever writes a path into SQL.
*
* LAYER 2 — DUCKDB'S OWN SANDBOX. And here is the finding:
*
* `SET allowed_directories=[...]` ALONE IS A NO-OP.
*
* Measured on DuckDB 1.5.5: with `allowed_directories` set to one
* mount directory and nothing else, `read_csv('/etc/passwd')`
* RETURNED THE FILE, and an https:// URL was fetched over the
* network. Only when `enable_external_access = false` is ALSO set
* does the allow-list bind — the same reads then fail with
* "Permission Error: Cannot access file". `lock_configuration=true`
* afterwards stops a query re-opening either.
*
* So the obvious spelling of this control — set the allow-list and
* move on — yields a setting that appears in `duckdb_settings()`,
* looks like a sandbox, and permits everything. Same shape as
* `trivy --exit-code 0`. Hence `duckDbSessionBootstrap()` emits the
* three statements as one unit, and `assertSessionSandboxed()`
* refuses a session missing any of them.
*
* Evidence: evidence/bi-p014/duckdb-sandbox.json, written by
* tests/duckdb/duckdb-provider.live.test.ts, which runs the bypass
* corpus with LAYER 1 REMOVED and requires layer 2 to refuse it anyway.
*/
import path from 'node:path';
import fs from 'node:fs';
import { SiaxError } from '../../platform/errors';
import type { AnalyticsIdentity } from '../engine/siax-analytics-engine';
export const ARTIFACT_FORMATS = ['parquet', 'csv', 'json', 'iceberg', 'arrow_ipc'] as const;
export type ArtifactFormat = (typeof ARTIFACT_FORMATS)[number];
/**
* A file the provider is allowed to read. Registered by the ST0RE mount
* process, never by a query.
*/
export interface ArtifactMount {
artifact_id: string;
tenant_id: string;
project_id?: string;
/** Canonical ST0RE reference, for lineage. */
st0re_ref: string;
/** Path RELATIVE to the mount root. Never absolute, never traversing. */
relative_path: string;
format: ArtifactFormat;
/** Bytes, from ST0RE metadata. Drives the N0D hand-off policy. */
size_bytes: number;
}
const ARTIFACT_ID_RE = /^[A-Za-z0-9_-]{1,128}$/;
/**
* Resolves artifact ids to real paths, and refuses everything else.
*
* `realpathSync` is the load-bearing call: a relative path that passes a string
* check can still be a symlink into `/etc`. The check is on the RESOLVED path,
* after the filesystem has had its say.
*/
export class ArtifactMountRegistry {
private readonly mounts = new Map<string, ArtifactMount>();
private readonly rootReal: string;
constructor(
/** The single directory every mounted artifact lives under. */
readonly root: string,
private readonly statPath: (p: string) => string = (p) => fs.realpathSync(p),
) {
if (!path.isAbsolute(root)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
'the mount root must be an absolute path',
);
}
this.rootReal = this.statPath(root);
}
register(mount: ArtifactMount): void {
if (!ARTIFACT_ID_RE.test(mount.artifact_id)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe');
}
if (!mount.tenant_id?.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'a mount must name its owning tenant');
}
if (path.isAbsolute(mount.relative_path)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
'a mount path must be relative to the mount root',
);
}
// Resolve BEFORE storing, so a bad mount is refused at registration rather
// than at query time when the caller is a tenant.
this.resolveWithin(mount.relative_path);
this.mounts.set(mount.artifact_id, mount);
}
/**
* Resolve an artifact for a VERIFIED identity.
*
* Two refusals that are easy to conflate but must stay distinct:
* * an unknown artifact is NOT_FOUND;
* * an artifact owned by another tenant is AUTHORIZATION_DENIED, and
* deliberately NOT reported as NOT_FOUND — an operator reading the audit
* trail needs to see an attempted cross-tenant read as one.
*/
resolve(identity: AnalyticsIdentity, artifactId: string): { mount: ArtifactMount; absolutePath: string } {
if (!identity?.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.duckdb',
'artifact access requires a verified tenant',
);
}
if (!ARTIFACT_ID_RE.test(artifactId)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'artifact id is not registry-safe');
}
const mount = this.mounts.get(artifactId);
if (!mount) {
throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', `artifact "${artifactId}" is not mounted`);
}
if (mount.tenant_id !== identity.tenant_id) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.duckdb',
`artifact "${artifactId}" belongs to another tenant`,
);
}
if (mount.project_id && identity.project_id && mount.project_id !== identity.project_id) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.duckdb',
`artifact "${artifactId}" belongs to another project`,
);
}
return { mount, absolutePath: this.resolveWithin(mount.relative_path) };
}
/** Join, resolve symlinks, and confirm the result is still under the root. */
private resolveWithin(relative: string): string {
const joined = path.resolve(this.rootReal, relative);
let real: string;
try {
real = this.statPath(joined);
} catch {
throw new SiaxError('NOT_FOUND', '0pt.analytics.duckdb', 'mounted artifact is not present on disk');
}
// `startsWith(root)` alone would accept `/mnt-evil` for root `/mnt`.
if (real !== this.rootReal && !real.startsWith(`${this.rootReal}${path.sep}`)) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.duckdb',
'artifact resolves outside the mount root (traversal or symlink)',
);
}
return real;
}
/** Every directory DuckDB is told about. One entry: the root itself. */
allowedDirectories(): string[] {
return [this.rootReal];
}
list(tenantId: string): ArtifactMount[] {
return [...this.mounts.values()].filter((m) => m.tenant_id === tenantId);
}
}
// ── layer 2: the DuckDB session sandbox ────────────────────────────────────
/**
* The three statements, as ONE unit.
*
* Order matters and is not cosmetic: `allowed_directories` must be set while
* the configuration is still open, `enable_external_access = false` is what
* makes the list bind at all, and `lock_configuration = true` must come last
* or a query could undo the other two.
*/
export function duckDbSessionBootstrap(allowedDirectories: readonly string[]): string[] {
if (allowedDirectories.length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
'refusing to open a DuckDB session with no allowed directory — an empty list is not a sandbox',
);
}
for (const dir of allowedDirectories) {
if (!path.isAbsolute(dir)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', 'an allowed directory must be absolute');
}
if (dir.includes("'")) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
'an allowed directory must not contain a quote',
);
}
}
return [
`SET allowed_directories=[${allowedDirectories.map((d) => `'${d}'`).join(', ')}]`,
// MEASURED: without this, the line above permits everything, including
// /etc/passwd and outbound HTTP. See this file's header.
'SET enable_external_access=false',
'SET lock_configuration=true',
];
}
/** The settings that must all be present for a session to be sandboxed. */
export const REQUIRED_SANDBOX_SETTINGS = [
'allowed_directories',
'enable_external_access',
'lock_configuration',
] as const;
export function assertSessionSandboxed(statements: readonly string[]): void {
const joined = statements.join('\n').toLowerCase();
for (const setting of REQUIRED_SANDBOX_SETTINGS) {
if (!joined.includes(setting)) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.duckdb',
`DuckDB session is missing "${setting}" — allowed_directories WITHOUT ` +
`enable_external_access=false is a no-op, measured on 1.5.5`,
);
}
}
if (!/enable_external_access\s*=\s*false/i.test(joined)) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.duckdb',
'DuckDB session leaves external access ENABLED; the directory allow-list is then not enforced',
);
}
}
// ── the provider ───────────────────────────────────────────────────────────
export interface DuckDbRunner {
readonly engine: string;
/** Open a session with the bootstrap applied, run `sql`, return rows. */
run(args: { bootstrap: readonly string[]; sql: string }): Promise<Array<Record<string, unknown>>>;
}
export const DUCKDB_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const;
export type DuckDbAggregate = (typeof DUCKDB_AGGREGATES)[number];
/**
* A closed request. There is NO caller SQL on this provider, for the same
* reason `executeMetric` has none: an analyst SQL surface is a separate,
* role-gated thing (BI-P017), and letting the ad-hoc FILE engine grow one would
* put raw SQL next to a filesystem.
*/
export interface DuckDbArtifactQuery {
artifact_id: string;
/** Column names to select. Validated as identifiers. */
select?: string[];
aggregate?: { fn: DuckDbAggregate; column: string; alias?: string };
group_by?: string[];
/** Equality filters. Names validated, values escaped. */
filters?: Record<string, string>;
limit?: number;
}
const COLUMN_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
function column(name: string): string {
if (!COLUMN_RE.test(name)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
`"${name.slice(0, 32)}" is not a column name`,
);
}
return name;
}
const READER: Record<ArtifactFormat, (p: string) => string> = {
parquet: (p) => `read_parquet('${p}')`,
csv: (p) => `read_csv('${p}', header=true, auto_detect=true)`,
json: (p) => `read_json_auto('${p}')`,
// The iceberg extension reads a table DIRECTORY, not a single file.
iceberg: (p) => `iceberg_scan('${p}')`,
arrow_ipc: (p) => `read_arrow('${p}')`,
};
export interface N0dHandoffPolicy {
/** Above this, the query is refused here and must go to N0D. */
maxLocalBytes: number;
}
export const DEFAULT_N0D_HANDOFF: N0dHandoffPolicy = { maxLocalBytes: 2 * 1024 * 1024 * 1024 };
export class DuckDbAnalyticsProvider {
readonly provider = 'duckdb';
constructor(
private readonly registry: ArtifactMountRegistry,
private readonly runner: DuckDbRunner,
private readonly handoff: N0dHandoffPolicy = DEFAULT_N0D_HANDOFF,
) {}
/** Formats this build can read. See `readiness()` for what has been PROVEN. */
supports(format: ArtifactFormat): boolean {
return ['parquet', 'csv', 'json', 'iceberg'].includes(format);
}
/**
* What has been demonstrated against a real DuckDB, and what has not.
* Same posture as BI-P029's `providerReadiness`: a capability list nobody
* can check is a capability list that overstates.
*/
readiness(): { proven: ArtifactFormat[]; unproven: Array<{ format: ArtifactFormat; reason: string }> } {
return {
proven: ['parquet', 'csv', 'iceberg'],
unproven: [
{
format: 'arrow_ipc',
reason:
"DuckDB's `arrow` extension is a network install and the offline verification " +
'environment could not download it, so read_arrow() has NOT been exercised',
},
{ format: 'json', reason: 'no fixture was built for it in the verification run' },
],
};
}
async query(
identity: AnalyticsIdentity,
request: DuckDbArtifactQuery,
): Promise<Array<Record<string, unknown>>> {
const { mount, absolutePath } = this.registry.resolve(identity, request.artifact_id);
if (!this.supports(mount.format)) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.duckdb',
`format "${mount.format}" is not readable by this provider build`,
);
}
if (mount.size_bytes > this.handoff.maxLocalBytes) {
// BI-P014: "Run heavy jobs through N0D." Refusing is the honest form —
// silently running a 40 GB scan in the API process is how a preview
// engine takes down a control plane.
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.duckdb',
`artifact is ${mount.size_bytes} bytes, above the ${this.handoff.maxLocalBytes}-byte local ceiling — ` +
`submit it as an N0D job`,
);
}
const source = READER[mount.format](absolutePath);
const selectParts: string[] = [];
const groupParts: string[] = [];
for (const g of request.group_by ?? []) {
selectParts.push(column(g));
groupParts.push(column(g));
}
if (request.aggregate) {
const col = column(request.aggregate.column);
const alias = column(request.aggregate.alias ?? 'value');
const fn = request.aggregate.fn;
if (!DUCKDB_AGGREGATES.includes(fn)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.duckdb', `unsupported aggregate "${fn}"`);
}
const expr =
fn === 'count' ? 'count(*)' : fn === 'count_distinct' ? `count(DISTINCT ${col})` : `${fn}(${col})`;
selectParts.push(`${expr} AS ${alias}`);
} else {
for (const s of request.select ?? []) selectParts.push(column(s));
}
if (selectParts.length === 0) selectParts.push('*');
const conditions: string[] = [];
for (const [name, value] of Object.entries(request.filters ?? {})) {
// Not every DuckDB binding exposes a bound-parameter transport, so values
// are ESCAPED here — and the escaping is asserted against a hostile
// corpus in the test rather than assumed.
conditions.push(`${column(name)} = '${escapeLiteral(value)}'`);
}
const limit = Math.max(1, Math.min(Math.floor(request.limit ?? 1000), 1_000_000));
const sql =
`SELECT ${selectParts.join(', ')} FROM ${source}` +
(conditions.length ? ` WHERE ${conditions.join(' AND ')}` : '') +
(groupParts.length ? ` GROUP BY ${groupParts.join(', ')}` : '') +
` LIMIT ${limit}`;
const bootstrap = duckDbSessionBootstrap(this.registry.allowedDirectories());
assertSessionSandboxed(bootstrap);
return this.runner.run({ bootstrap, sql });
}
}
/**
* Escape a value for a DuckDB string literal.
*
* Doubling `'` is the whole grammar — DuckDB does NOT treat backslash as an
* escape inside a standard single-quoted string, so `\'` does not close one.
* A NUL is REFUSED rather than escaped: a truncating byte inside a literal is
* a parser disagreement waiting to happen, and there is no legitimate filter
* value that contains one.
*/
export function escapeLiteral(value: string): string {
if (value.includes('\u0000')) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.duckdb',
'a filter value must not contain a NUL byte',
);
}
return value.replace(/'/g, "''");
}
@@ -0,0 +1,127 @@
/**
* src/analytics/quality/clickhouse-quality-runner.ts
* ===================================================
* BI-P025 — the ClickHouse `QualityQueryRunner`.
*
* One statement per rule, and it counts BOTH numbers in a single pass:
*
* SELECT countIf(<violation expression>) AS violations, count() AS evaluated
* FROM <db>.<table> WHERE tenant_id = {tenant:String} AND dataset_id = …
*
* `evaluated` is not decoration. It is what makes an EMPTY dataset
* distinguishable from a CLEAN one — the false green
* `NativeQualityProvider` refuses. Counting only violations would make the two
* cases identical at this layer, and no amount of care upstream could recover
* the difference.
*
* The BI-P011 forced tenant predicate applies here exactly as it does to a
* metric query: `tenant_id = {tenant:String}` is condition[0], the value is
* bound, and a quality run therefore cannot read another tenant's rows even
* though it runs on an administrative credential.
*
* `unique` gets its own statement shape, because a duplicate is a property of
* a GROUP, not of a row, and pretending otherwise would make the rule count
* the wrong thing.
*/
import { SiaxError } from '../../platform/errors';
import { assertSafeIdentifier, FORCED_TENANT_PREDICATE } from '../engine/query-compiler';
import type { QualityQueryRunner } from './quality-provider';
export interface ClickHouseQualityRunnerOptions {
baseUrl: string;
basicAuth?: string;
database: string;
/** dataset_id -> physical table. An unlisted dataset is refused, not guessed. */
tables: Record<string, string>;
timeoutMs?: number;
/** Ceiling for a quality scan; a check must not become the expensive query. */
maxRowsToRead?: number;
}
/** Marker the provider emits for the `unique` rule. */
const UNIQUE_MARKER = /^__unique__\(([A-Za-z_][A-Za-z0-9_]{0,62})\)$/;
export class ClickHouseQualityRunner implements QualityQueryRunner {
readonly engine = 'clickhouse';
constructor(private readonly opts: ClickHouseQualityRunnerOptions) {
assertSafeIdentifier(opts.database, 'database');
for (const table of Object.values(opts.tables)) assertSafeIdentifier(table, 'table');
}
async count(args: {
tenant_id: string;
dataset_id: string;
violationExpression: string;
params: Record<string, string>;
}): Promise<{ violations: number; evaluated: number }> {
if (!args.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.quality',
'a quality run requires a verified tenant',
);
}
const table = this.opts.tables[args.dataset_id];
if (!table) {
throw new SiaxError(
'NOT_FOUND',
'0pt.analytics.quality',
`dataset "${args.dataset_id}" has no physical table registered for quality runs`,
);
}
const params: Record<string, string> = {
...args.params,
param_tenant: args.tenant_id,
param_dataset: args.dataset_id,
};
const where = `${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String}`;
const unique = UNIQUE_MARKER.exec(args.violationExpression.trim());
const sql = unique
? // A duplicate is a property of a GROUP. Count the rows that belong to a
// key appearing more than once, and the total rows, in one pass.
`SELECT sum(multiIf(c > 1, c, 0)) AS violations, sum(c) AS evaluated FROM ` +
`(SELECT ${assertSafeIdentifier(unique[1], 'field')} AS k, count() AS c ` +
`FROM ${this.opts.database}.${table} WHERE ${where} GROUP BY k) FORMAT JSON`
: `SELECT countIf(${args.violationExpression}) AS violations, count() AS evaluated ` +
`FROM ${this.opts.database}.${table} WHERE ${where} FORMAT JSON`;
const url = new URL(this.opts.baseUrl.replace(/\/+$/, ''));
url.searchParams.set('query', sql);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
url.searchParams.set('max_rows_to_read', String(this.opts.maxRowsToRead ?? 5_000_000_000));
url.searchParams.set('read_overflow_mode', 'throw');
url.searchParams.set('max_execution_time', '60');
url.searchParams.set('timeout_overflow_mode', 'throw');
const headers: Record<string, string> = {};
if (this.opts.basicAuth) {
headers.Authorization = `Basic ${Buffer.from(this.opts.basicAuth).toString('base64')}`;
}
let res: Response;
try {
res = await fetch(url.toString(), {
method: 'POST',
headers,
body: '',
signal: AbortSignal.timeout(this.opts.timeoutMs ?? 60_000),
});
} catch (cause) {
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
}
const text = await res.text();
if (!res.ok) {
const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `quality query failed (CH code ${code})`);
}
const row = (
JSON.parse(text) as { data: Array<{ violations: string | number; evaluated: string | number }> }
).data[0];
if (!row) {
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'quality query returned no counts');
}
return { violations: Number(row.violations ?? 0), evaluated: Number(row.evaluated ?? 0) };
}
}
+196
View File
@@ -0,0 +1,196 @@
/**
* src/analytics/quality/freshness.ts
* ===================================
* BI-P026 — DATA FRESHNESS.
*
* Five states, exactly as BI-P026 names them, and the DoD is a NEGATIVE:
*
* "Stale data is never SILENTLY presented as current."
*
* Two words carry it. "Silently" means the failure mode is not "we served old
* data" — it is "we served old data and the reader could not tell". And
* "never" means the enforcement has to sit at the boundary every result
* crosses, not in each dashboard's rendering code.
*
* So `assertFreshnessDisclosed` refuses to release a result that carries no
* freshness envelope at all. A result with no envelope is not FRESH and not
* UNKNOWN — it is unreleasable, because "we forgot to attach it" and "the data
* is current" are indistinguishable to a reader, and a default of FRESH would
* make forgetting the silent path.
*
* UNKNOWN is likewise never treated as FRESH. It is its own state precisely
* because "we could not determine the age" is the honest thing to display, and
* collapsing it into FRESH is the classic version of this bug.
*/
import { SiaxError } from '../../platform/errors';
/** BI-P026's states, verbatim and in severity order. */
export const FRESHNESS_STATES = ['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN'] as const;
export type FreshnessState = (typeof FRESHNESS_STATES)[number];
/** States a reader must be shown explicitly rather than served as current. */
export const NON_CURRENT_STATES: readonly FreshnessState[] = ['DELAYED', 'STALE', 'FAILED', 'UNKNOWN'];
export interface FreshnessPolicy {
/** How often the upstream is expected to land data. */
expectedIntervalMs: number;
/**
* Multiples of the expected interval. Past `delayedAfter` the data is
* DELAYED; past `staleAfter` it is STALE. Defaults are deliberately tight:
* a dashboard that is two cycles behind is already misleading someone.
*/
delayedAfterIntervals?: number;
staleAfterIntervals?: number;
}
export interface FreshnessInput {
/** When the newest successfully loaded data is timestamped. */
lastSuccessfulLoadAt?: string;
/** Outcome of the most recent load attempt. */
lastRunStatus?: 'succeeded' | 'failed' | 'running' | 'never_run';
lastRunAt?: string;
policy: FreshnessPolicy;
now: number;
}
export interface FreshnessEnvelope {
state: FreshnessState;
/** Age of the newest data, in ms. Absent when unknown. */
age_ms?: number;
last_successful_load_at?: string;
last_run_at?: string;
expected_interval_ms: number;
/** Plain-language line a UI can render verbatim. */
disclosure: string;
computed_at: string;
}
export function computeFreshness(input: FreshnessInput): FreshnessEnvelope {
const { policy, now } = input;
const delayedAfter = policy.expectedIntervalMs * (policy.delayedAfterIntervals ?? 1.5);
const staleAfter = policy.expectedIntervalMs * (policy.staleAfterIntervals ?? 3);
const computed_at = new Date(now).toISOString();
const base = {
last_successful_load_at: input.lastSuccessfulLoadAt,
last_run_at: input.lastRunAt,
expected_interval_ms: policy.expectedIntervalMs,
computed_at,
};
// FAILED outranks age: a pipeline that just failed may still have
// recently-timestamped data from the run before, and reporting that as FRESH
// is precisely how a broken pipeline stays invisible for a day.
if (input.lastRunStatus === 'failed') {
const age = ageOf(input.lastSuccessfulLoadAt, now);
return {
...base,
state: 'FAILED',
age_ms: age,
disclosure: `The most recent load FAILED. Showing data from ${input.lastSuccessfulLoadAt ?? 'an unknown time'}.`,
};
}
if (!input.lastSuccessfulLoadAt || input.lastRunStatus === 'never_run') {
return {
...base,
state: 'UNKNOWN',
disclosure: 'The age of this data could not be determined. Do not read it as current.',
};
}
const age = ageOf(input.lastSuccessfulLoadAt, now);
if (age === undefined) {
return { ...base, state: 'UNKNOWN', disclosure: 'The age of this data could not be determined.' };
}
if (age > staleAfter) {
return {
...base,
state: 'STALE',
age_ms: age,
disclosure: `This data is ${describeAge(age)} old, past the ${describeAge(staleAfter)} staleness threshold.`,
};
}
if (age > delayedAfter) {
return {
...base,
state: 'DELAYED',
age_ms: age,
disclosure: `This data is ${describeAge(age)} old and behind its ${describeAge(policy.expectedIntervalMs)} schedule.`,
};
}
return {
...base,
state: 'FRESH',
age_ms: age,
disclosure: `Current as of ${input.lastSuccessfulLoadAt}.`,
};
}
function ageOf(iso: string | undefined, now: number): number | undefined {
if (!iso) return undefined;
const t = Date.parse(iso);
if (Number.isNaN(t)) return undefined;
return Math.max(0, now - t);
}
function describeAge(ms: number): string {
const s = Math.round(ms / 1000);
if (s < 90) return `${s}s`;
const m = Math.round(s / 60);
if (m < 90) return `${m}m`;
const h = Math.round(m / 60);
if (h < 48) return `${h}h`;
return `${Math.round(h / 24)}d`;
}
/** Any result leaving the analytics boundary. */
export interface DisclosableResult {
freshness?: FreshnessEnvelope;
/**
* Set by a caller that has SEEN a non-current state and chosen to proceed.
* Its presence is what makes serving non-current data not-silent.
*/
acknowledged_non_current?: boolean;
}
/**
* BI-P026's DoD, as a gate.
*
* Three refusals, in order of how easy they are to get wrong:
* 1. no envelope at all — unreleasable. Not defaulted to FRESH.
* 2. a non-current state with no acknowledgement — refused, so a consumer
* that ignores the envelope cannot render stale numbers as current.
* 3. an acknowledgement with no envelope — refused, because acknowledging
* something you were never shown is not disclosure.
*/
export function assertFreshnessDisclosed(result: DisclosableResult, context: string): FreshnessEnvelope {
if (!result.freshness) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.freshness',
`${context} carries no freshness envelope — BI-P026: data with unknown age must never be released ` +
`as though it were current`,
);
}
const state = result.freshness.state;
if (!(FRESHNESS_STATES as readonly string[]).includes(state)) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.freshness',
`${context} has an unknown freshness state`,
);
}
if (NON_CURRENT_STATES.includes(state) && !result.acknowledged_non_current) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.freshness',
`${context} is ${state} and the consumer has not acknowledged it: ${result.freshness.disclosure}`,
);
}
return result.freshness;
}
/** True when the state may be presented without a caveat. Only one state may. */
export function isPresentableAsCurrent(state: FreshnessState): boolean {
return state === 'FRESH';
}
+418
View File
@@ -0,0 +1,418 @@
/**
* src/analytics/quality/quality-provider.ts
* ==========================================
* BI-P025 — DATA QUALITY PROVIDER.
*
* One interface, three provider kinds, and an honest split between them:
*
* NATIVE — this module compiles the rule into a counting query and runs it
* through an injected runner. It EXECUTES.
* DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against
* the BI-P022 contract). This module INGESTS the results. It does
* not run dbt, and `providerKind` says so rather than implying
* otherwise.
* GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here
* executes it. The adapter maps a GE validation result onto the
* canonical verdict so a GE suite counts toward the same gate.
*
* Calling all three "providers" while only one executes would be the kind of
* overstatement that turns into "we have Great Expectations coverage" in a
* status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or
* `"ingesting"`, and the gate records which produced each verdict.
*
* ── BI-P025's DoD ─────────────────────────────────────────────────────────
* "Failed critical quality gate can QUARANTINE serving projection."
*
* `quarantine.ts` is that, and the important half is what happens on the READ
* side: a quarantined projection is REFUSED, not annotated. An annotation on a
* dashboard nobody reads is not a quarantine.
*/
import { SiaxError } from '../../platform/errors';
import type { QualityRuleKind, QualitySeverity } from '../domain/entities';
/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */
export const QUALITY_RULES = [
'not_null',
'unique',
'range',
'referential',
'freshness',
'schema',
'distribution',
'custom',
] as const;
export type QualityRule = (typeof QUALITY_RULES)[number];
/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */
export const RULE_KIND_ALIASES: Record<QualityRule, QualityRuleKind> = {
not_null: 'not_null',
unique: 'unique',
range: 'range',
referential: 'referential',
freshness: 'freshness',
schema: 'custom',
distribution: 'accepted_values',
custom: 'custom',
};
export interface QualityRuleSpec {
rule_id: string;
tenant_id: string;
dataset_id: string;
rule: QualityRule;
/** Column the rule applies to; absent for dataset-level rules. */
field?: string;
severity: QualitySeverity;
params: QualityRuleParams;
description?: string;
}
export interface QualityRuleParams {
/** range */
min?: number;
max?: number;
/** distribution / accepted values */
accepted_values?: string[];
/** distribution: the share of rows that must fall in accepted_values */
min_share?: number;
/** referential */
references_dataset?: string;
references_field?: string;
/** freshness: the maximum acceptable age of the newest row */
max_age_seconds?: number;
timestamp_field?: string;
/** schema: the columns and types that must exist */
expected_columns?: Array<{ name: string; type: string }>;
/** unique / not_null take no parameters */
/** custom: an operator-authored counting query, reviewed like code */
custom_violation_sql?: string;
}
export interface QualityVerdict {
rule_id: string;
dataset_id: string;
tenant_id: string;
rule: QualityRule;
severity: QualitySeverity;
passed: boolean;
violation_count: number;
evaluated_rows: number;
/** Which provider produced this, and whether it EXECUTED or merely ingested. */
produced_by: string;
provider_kind: 'executing' | 'ingesting';
evaluated_at: string;
/** Short operator-facing note. Never raw engine output. */
detail?: string;
}
export interface SiaxDataQualityProvider {
readonly provider: string;
readonly providerKind: 'executing' | 'ingesting';
/** Rules this provider can actually evaluate. */
supports(rule: QualityRule): boolean;
evaluate(spec: QualityRuleSpec): Promise<QualityVerdict>;
}
/**
* Runs a counting query and returns two numbers. The only engine coupling in
* this module, so a DuckDB or Trino runner drops in unchanged.
*/
export interface QualityQueryRunner {
readonly engine: string;
/**
* Must apply the caller's tenant scoping. The runner is handed the tenant
* explicitly rather than trusting the SQL to carry it.
*/
count(args: {
tenant_id: string;
dataset_id: string;
violationExpression: string;
/** Bound parameters. Values NEVER appear in the expression text. */
params: Record<string, string>;
}): Promise<{ violations: number; evaluated: number }>;
}
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
function assertField(field: string | undefined, rule: QualityRule): string {
if (!field || !IDENT_RE.test(field)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
`rule "${rule}" requires a valid field name`,
);
}
return field;
}
/**
* The provider that actually runs checks.
*
* Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and
* the runner counts rows where it is true. That shape is chosen so a rule can
* never return "how many rows are fine", which is the number that looks
* healthy when a table is empty.
*/
export class NativeQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'native';
readonly providerKind = 'executing' as const;
constructor(
private readonly runner: QualityQueryRunner,
private readonly now: () => number = Date.now,
) {}
supports(rule: QualityRule): boolean {
// `schema` and `referential` need catalog/second-table access the counting
// runner does not have; they are honestly unsupported here rather than
// silently passing.
return !['schema', 'referential'].includes(rule);
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
if (!this.supports(spec.rule)) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.quality',
`the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`,
);
}
const { expression, params } = this.compile(spec);
const { violations, evaluated } = await this.runner.count({
tenant_id: spec.tenant_id,
dataset_id: spec.dataset_id,
violationExpression: expression,
params,
});
// An EMPTY dataset is not a passing dataset. A `not_null` check over zero
// rows finds zero violations and would report success while the pipeline
// is silently broken — the single most common false green in data quality.
const emptyIsFailure = spec.rule !== 'custom';
const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0;
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
passed,
violation_count: violations,
evaluated_rows: evaluated,
produced_by: `${this.provider}:${this.runner.engine}`,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail:
evaluated === 0 && emptyIsFailure
? 'dataset is EMPTY — reported as a failure, not as zero violations'
: undefined,
};
}
private compile(spec: QualityRuleSpec): { expression: string; params: Record<string, string> } {
const p = spec.params;
switch (spec.rule) {
case 'not_null': {
const f = assertField(spec.field, spec.rule);
return { expression: `${f} IS NULL OR ${f} = ''`, params: {} };
}
case 'unique': {
const f = assertField(spec.field, spec.rule);
// Counted over a grouped subquery by the runner; the expression names
// the key so the runner can build it.
return { expression: `__unique__(${f})`, params: {} };
}
case 'range': {
const f = assertField(spec.field, spec.rule);
if (p.min === undefined && p.max === undefined) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a range rule needs a min or a max',
);
}
const parts: string[] = [];
const params: Record<string, string> = {};
if (p.min !== undefined) {
parts.push(`${f} < {qmin:Float64}`);
params.param_qmin = String(p.min);
}
if (p.max !== undefined) {
parts.push(`${f} > {qmax:Float64}`);
params.param_qmax = String(p.max);
}
return { expression: parts.join(' OR '), params };
}
case 'freshness': {
const f = assertField(p.timestamp_field ?? spec.field, spec.rule);
if (!p.max_age_seconds || p.max_age_seconds <= 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a freshness rule needs max_age_seconds',
);
}
return {
expression: `${f} < now() - toIntervalSecond({qage:UInt64})`,
params: { param_qage: String(Math.floor(p.max_age_seconds)) },
};
}
case 'distribution': {
const f = assertField(spec.field, spec.rule);
if (!p.accepted_values?.length) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a distribution rule needs accepted_values',
);
}
const params: Record<string, string> = {};
const bound = p.accepted_values.map((v, i) => {
params[`param_qv${i}`] = v;
return `{qv${i}:String}`;
});
return { expression: `${f} NOT IN (${bound.join(', ')})`, params };
}
case 'custom': {
if (!p.custom_violation_sql?.trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a custom rule needs custom_violation_sql',
);
}
// Deliberately NOT parsed or guarded here. A custom rule is
// operator-authored, reviewed like code, and stored in the rule table
// — not caller input. It never reaches this function from a request
// body; `QualityRuleSpec` rows are written by an admin path.
return { expression: p.custom_violation_sql, params: {} };
}
default:
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`);
}
}
}
/**
* Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against
* the BI-P022 contract, and pretending otherwise here would misreport where
* the work happens.
*/
export interface DbtTestResult {
unique_id: string;
status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped';
failures: number;
/** dbt does not report rows evaluated; absent, not zero. */
rows_evaluated?: number;
message?: string;
}
export class DbtQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'dbt';
readonly providerKind = 'ingesting' as const;
constructor(
private readonly results: Map<string, DbtTestResult>,
private readonly now: () => number = Date.now,
) {}
supports(): boolean {
return true;
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
const result = this.results.get(spec.rule_id);
if (!result) {
// A missing result is UNKNOWN, and unknown is a FAILURE for a blocking
// rule. Treating "no result" as "passed" is how a quality gate stops
// gating the moment the test stops running.
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
passed: false,
violation_count: 0,
evaluated_rows: 0,
produced_by: this.provider,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail: 'no dbt result for this rule — reported as a failure, not as a pass',
};
}
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
// `skipped` and `error` are NOT passes. dbt's `skipped` in particular is
// the shape of l0re's silently-skipped Postgres tests.
passed: result.status === 'pass',
violation_count: result.failures,
evaluated_rows: result.rows_evaluated ?? 0,
produced_by: `dbt:${result.unique_id}`,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined,
};
}
}
/** Ingests a Great Expectations validation result. Does NOT run GE. */
export interface GreatExpectationsResult {
expectation_suite_name: string;
success: boolean;
results: Array<{
expectation_config: { expectation_type: string };
success: boolean;
result?: { unexpected_count?: number; element_count?: number };
}>;
}
export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'great_expectations';
readonly providerKind = 'ingesting' as const;
constructor(
private readonly suites: Map<string, GreatExpectationsResult>,
private readonly now: () => number = Date.now,
) {}
supports(): boolean {
return true;
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
const suite = this.suites.get(spec.rule_id);
const base = {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
produced_by: this.provider,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
};
if (!suite) {
return {
...base,
passed: false,
violation_count: 0,
evaluated_rows: 0,
detail: 'no GE result — reported as a failure',
};
}
const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0);
const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0);
return {
...base,
passed: suite.success,
violation_count: unexpected,
evaluated_rows: elements,
produced_by: `great_expectations:${suite.expectation_suite_name}`,
detail: suite.success ? undefined : 'GE suite failed',
};
}
}
+188
View File
@@ -0,0 +1,188 @@
/**
* src/analytics/quality/quarantine.ts
* ====================================
* BI-P025's DoD: "Failed critical quality gate can QUARANTINE serving
* projection."
*
* The word that matters is on the READ side. A quarantine that annotates a
* result is a label; a quarantine that REFUSES the read is a quarantine. This
* module does the second, and `assertProjectionReadable` is the single place
* every serving read passes through.
*
* Severity semantics, stated because they are the whole gate:
* `block` — a failure quarantines the projection. Reads are refused.
* `warn` — a failure is recorded and surfaced, and reads continue.
* A rule with no severity does not exist: `QualityRuleSpec.severity` is
* required, so "we forgot to mark it critical" cannot silently become `warn`.
*/
import { SiaxError } from '../../platform/errors';
import type { QualityVerdict } from './quality-provider';
export interface QuarantineRecord {
tenant_id: string;
dataset_id: string;
quarantined: boolean;
/** Rule ids whose blocking failure caused it. */
caused_by: string[];
since?: string;
released_at?: string;
released_by?: string;
/** Operator-facing summary. */
reason?: string;
}
export interface QuarantineStore {
get(tenantId: string, datasetId: string): Promise<QuarantineRecord | undefined>;
put(record: QuarantineRecord): Promise<void>;
}
export class InMemoryQuarantineStore implements QuarantineStore {
private readonly rows = new Map<string, QuarantineRecord>();
private key(t: string, d: string): string {
return `${t}::${d}`;
}
async get(t: string, d: string): Promise<QuarantineRecord | undefined> {
return this.rows.get(this.key(t, d));
}
async put(r: QuarantineRecord): Promise<void> {
this.rows.set(this.key(r.tenant_id, r.dataset_id), r);
}
}
export interface GateOutcome {
dataset_id: string;
tenant_id: string;
passed: boolean;
quarantined: boolean;
blocking_failures: QualityVerdict[];
warnings: QualityVerdict[];
evaluated: number;
}
export class QualityGate {
constructor(
private readonly store: QuarantineStore,
private readonly now: () => number = Date.now,
) {}
/**
* Apply a run's verdicts.
*
* A gate over ZERO verdicts does not pass. "No rules ran" and "every rule
* passed" produce the same green in most implementations, and that is how a
* quality gate stops gating without anyone noticing — the same shape as
* l0re's 28 silently-skipped tests and api0's `trivy --exit-code 0`.
*/
async apply(tenantId: string, datasetId: string, verdicts: QualityVerdict[]): Promise<GateOutcome> {
if (verdicts.length === 0) {
const record: QuarantineRecord = {
tenant_id: tenantId,
dataset_id: datasetId,
quarantined: true,
caused_by: [],
since: new Date(this.now()).toISOString(),
reason: 'no quality rules were evaluated — an ungated dataset is quarantined, not trusted',
};
await this.store.put(record);
return {
dataset_id: datasetId,
tenant_id: tenantId,
passed: false,
quarantined: true,
blocking_failures: [],
warnings: [],
evaluated: 0,
};
}
for (const v of verdicts) {
if (v.tenant_id !== tenantId || v.dataset_id !== datasetId) {
// A verdict from another tenant's run must never influence this gate.
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.quality',
'a quality verdict from a different tenant or dataset was supplied to this gate',
);
}
}
const blocking = verdicts.filter((v) => !v.passed && v.severity === 'block');
const warnings = verdicts.filter((v) => !v.passed && v.severity === 'warn');
if (blocking.length > 0) {
await this.store.put({
tenant_id: tenantId,
dataset_id: datasetId,
quarantined: true,
caused_by: blocking.map((v) => v.rule_id),
since: new Date(this.now()).toISOString(),
reason: `${blocking.length} blocking quality rule(s) failed`,
});
} else {
// Passing does NOT auto-release: releasing is an act with an operator
// behind it (`release()`), because a flapping rule would otherwise let a
// dataset quarantine and un-quarantine itself between reads.
const existing = await this.store.get(tenantId, datasetId);
if (!existing || !existing.quarantined) {
await this.store.put({
tenant_id: tenantId,
dataset_id: datasetId,
quarantined: false,
caused_by: [],
});
}
}
return {
dataset_id: datasetId,
tenant_id: tenantId,
passed: blocking.length === 0,
quarantined: blocking.length > 0 || (await this.isQuarantined(tenantId, datasetId)),
blocking_failures: blocking,
warnings,
evaluated: verdicts.length,
};
}
async isQuarantined(tenantId: string, datasetId: string): Promise<boolean> {
return (await this.store.get(tenantId, datasetId))?.quarantined ?? false;
}
/** Release requires an operator and a reason. It is not automatic. */
async release(tenantId: string, datasetId: string, by: string, reason: string): Promise<QuarantineRecord> {
if (!by?.trim() || !reason?.trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'releasing a quarantine requires a principal and a reason',
);
}
const record: QuarantineRecord = {
tenant_id: tenantId,
dataset_id: datasetId,
quarantined: false,
caused_by: [],
released_at: new Date(this.now()).toISOString(),
released_by: by,
reason,
};
await this.store.put(record);
return record;
}
/**
* The read-side gate. Every serving projection read passes through here.
* Refuses, rather than annotating — an annotation on a dashboard nobody
* reads is not a quarantine.
*/
async assertProjectionReadable(tenantId: string, datasetId: string): Promise<void> {
const record = await this.store.get(tenantId, datasetId);
if (record?.quarantined) {
throw new SiaxError(
'DEPENDENCY_UNAVAILABLE',
'0pt.analytics.quality',
`serving projection for "${datasetId}" is QUARANTINED: ${record.reason ?? 'failed quality gate'}`,
);
}
}
}
@@ -0,0 +1,356 @@
/**
* src/analytics/repositories/analytics-domain-repository.ts
* ==========================================================
* BI-P002 persistence for the analytics domain model (sql/008).
*
* Two rules the type system alone cannot keep, so they live here:
*
* 1. EVERY read and write is tenant-scoped. There is no method that takes a
* dataset_id without also taking the tenant it must belong to, and every
* WHERE carries `tenant_id = $tenant`. A caller cannot "forget" the
* tenant because there is no signature that lets them.
* 2. VERSION ROWS ARE APPEND-ONLY from this layer too. `createDatasetVersion`
* /`createMetricVersion` compute the next version number inside the same
* statement (`SELECT coalesce(max(version),0)+1`) so two concurrent
* minters collide on the UNIQUE(dataset_id, version) constraint instead
* of silently overwriting — and the database triggers from sql/008 catch
* anything that still tries to UPDATE.
*
* Uses postgres.js through `RepositorySql`, the same connection type the other
* P0 repositories take (src/repositories/db.ts).
*/
import crypto from "node:crypto";
import { SiaxError } from "../../platform/errors";
import type { RepositorySql } from "../../repositories/db";
import type {
AnalyticsDataset,
DatasetVersion,
DimensionDefinition,
MetricDefinition,
MetricVersion,
PiiClassification,
} from "../domain/entities";
import type { AnalyticsTier } from "../domain/tiers";
/** The verified tenant scope every call must present. */
export interface TenantScope {
tenant_id: string;
project_id: string;
}
function assertScope(scope: TenantScope): void {
if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) {
throw new SiaxError(
"UNAUTHENTICATED",
"0pt.analytics",
"analytics persistence requires a verified tenant_id + project_id",
);
}
}
function iso(v: string | Date | null | undefined): string | undefined {
if (v === null || v === undefined) return undefined;
return v instanceof Date ? v.toISOString() : v;
}
interface DatasetRow {
dataset_id: string;
tenant_id: string;
project_id: string;
name: string;
tier: string;
schema_contract_id: string;
schema_contract_version: string;
owner_id: string;
source_ref: string;
freshness_slo_seconds: number;
quality_rule_ids: string[];
retention_days: number;
classification: string;
status: string;
raw_access_exception: string | null;
description: string | null;
created_at: string | Date;
updated_at: string | Date | null;
created_by: string | null;
}
function toDataset(r: DatasetRow): AnalyticsDataset {
return {
dataset_id: r.dataset_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
name: r.name,
tier: r.tier as AnalyticsTier,
schema_contract_id: r.schema_contract_id,
schema_contract_version: r.schema_contract_version,
owner_id: r.owner_id,
source_ref: r.source_ref,
freshness_slo_seconds: Number(r.freshness_slo_seconds),
quality_rule_ids: r.quality_rule_ids ?? [],
retention_days: Number(r.retention_days),
classification: r.classification as PiiClassification,
status: r.status as AnalyticsDataset["status"],
...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}),
...(r.description ? { description: r.description } : {}),
created_at: iso(r.created_at)!,
...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}),
...(r.created_by ? { created_by: r.created_by } : {}),
};
}
export class AnalyticsDomainRepository {
constructor(private readonly sql: RepositorySql) {}
// ── AnalyticsDataset ─────────────────────────────────────────────────────
async createDataset(scope: TenantScope, input: Omit<AnalyticsDataset, "tenant_id" | "project_id" | "created_at" | "updated_at">): Promise<AnalyticsDataset> {
assertScope(scope);
const rows = await this.sql<DatasetRow[]>`
INSERT INTO opt_analytics_dataset
(dataset_id, tenant_id, project_id, name, tier, schema_contract_id, schema_contract_version,
owner_id, source_ref, freshness_slo_seconds, quality_rule_ids, retention_days,
classification, status, raw_access_exception, description, created_by)
VALUES
(${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.tier},
${input.schema_contract_id}, ${input.schema_contract_version}, ${input.owner_id}, ${input.source_ref},
${input.freshness_slo_seconds}, ${this.sql.array(input.quality_rule_ids ?? [])}, ${input.retention_days},
${input.classification}, ${input.status}, ${input.raw_access_exception ?? null},
${input.description ?? null}, ${input.created_by ?? null})
RETURNING *`;
return toDataset(rows[0]);
}
/** Tenant-scoped read. A foreign dataset_id returns undefined, never a row. */
async getDataset(scope: TenantScope, datasetId: string): Promise<AnalyticsDataset | undefined> {
assertScope(scope);
const rows = await this.sql<DatasetRow[]>`
SELECT * FROM opt_analytics_dataset
WHERE dataset_id = ${datasetId}
AND tenant_id = ${scope.tenant_id}
AND project_id = ${scope.project_id}
LIMIT 1`;
return rows[0] ? toDataset(rows[0]) : undefined;
}
async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise<AnalyticsDataset[]> {
assertScope(scope);
const rows = await this.sql<DatasetRow[]>`
SELECT * FROM opt_analytics_dataset
WHERE tenant_id = ${scope.tenant_id}
AND project_id = ${scope.project_id}
${filter?.tier ? this.sql`AND tier = ${filter.tier}` : this.sql``}
${filter?.status ? this.sql`AND status = ${filter.status}` : this.sql``}
ORDER BY created_at DESC`;
return rows.map(toDataset);
}
// ── DatasetVersion (append-only) ─────────────────────────────────────────
/**
* Mint the next version. The version number is computed inside the INSERT,
* so two racing callers cannot both read "3" and both write it — the loser
* hits UNIQUE(dataset_id, version) and gets a CONFLICT, which is the honest
* outcome.
*/
async createDatasetVersion(
scope: TenantScope,
input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string },
): Promise<DatasetVersion> {
assertScope(scope);
const owner = await this.getDataset(scope, input.dataset_id);
if (!owner) {
throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant");
}
const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`;
try {
const rows = await this.sql<
{
dataset_version_id: string;
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
contract_hash: string;
schema_contract_version: string;
record_count: string | number | null;
created_at: string | Date;
created_by: string | null;
}[]
>`
INSERT INTO opt_analytics_dataset_version
(dataset_version_id, dataset_id, tenant_id, project_id, version,
contract_hash, schema_contract_version, record_count, created_by)
SELECT ${id}, ${input.dataset_id}, ${scope.tenant_id}, ${scope.project_id},
coalesce(max(version), 0) + 1,
${input.contract_hash}, ${input.schema_contract_version},
${input.record_count ?? null}, ${input.created_by ?? null}
FROM opt_analytics_dataset_version
WHERE dataset_id = ${input.dataset_id} AND tenant_id = ${scope.tenant_id}
RETURNING *`;
const r = rows[0];
return {
dataset_version_id: r.dataset_version_id,
dataset_id: r.dataset_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
version: Number(r.version),
contract_hash: r.contract_hash,
schema_contract_version: r.schema_contract_version,
...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}),
created_at: iso(r.created_at)!,
...(r.created_by ? { created_by: r.created_by } : {}),
};
} catch (err) {
if (String(err).includes("duplicate key")) {
throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version");
}
throw err;
}
}
async listDatasetVersions(scope: TenantScope, datasetId: string): Promise<DatasetVersion[]> {
assertScope(scope);
const rows = await this.sql<
{ dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[]
>`
SELECT * FROM opt_analytics_dataset_version
WHERE dataset_id = ${datasetId}
AND tenant_id = ${scope.tenant_id}
AND project_id = ${scope.project_id}
ORDER BY version ASC`;
return rows.map((r) => ({
dataset_version_id: r.dataset_version_id,
dataset_id: r.dataset_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
version: Number(r.version),
contract_hash: r.contract_hash,
schema_contract_version: r.schema_contract_version,
...(r.record_count !== null ? { record_count: Number(r.record_count) } : {}),
created_at: iso(r.created_at)!,
...(r.created_by ? { created_by: r.created_by } : {}),
}));
}
// ── MetricDefinition / MetricVersion ─────────────────────────────────────
async createMetric(
scope: TenantScope,
input: Omit<MetricDefinition, "tenant_id" | "project_id" | "created_at" | "updated_at" | "current_version">,
): Promise<MetricDefinition> {
assertScope(scope);
const rows = await this.sql<
{ metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[]
>`
INSERT INTO opt_analytics_metric_definition
(metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field,
denominator_field, allowed_dimension_ids, current_version, owner_id, description)
VALUES
(${input.metric_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id},
${input.aggregation}, ${input.measure_field}, ${input.denominator_field ?? null},
${this.sql.array(input.allowed_dimension_ids ?? [])}, 1, ${input.owner_id}, ${input.description ?? null})
RETURNING *`;
const r = rows[0];
return {
metric_id: r.metric_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
name: r.name,
dataset_id: r.dataset_id,
aggregation: r.aggregation as MetricDefinition["aggregation"],
measure_field: r.measure_field,
...(r.denominator_field ? { denominator_field: r.denominator_field } : {}),
allowed_dimension_ids: r.allowed_dimension_ids ?? [],
current_version: Number(r.current_version),
owner_id: r.owner_id,
...(r.description ? { description: r.description } : {}),
created_at: iso(r.created_at)!,
...(r.updated_at ? { updated_at: iso(r.updated_at) } : {}),
};
}
/** Append-only. `change_reason` is required for version > 1 (DB CHECK). */
async createMetricVersion(
scope: TenantScope,
input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string },
): Promise<MetricVersion> {
assertScope(scope);
const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex");
const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`;
const rows = await this.sql<
{ metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[]
>`
INSERT INTO opt_analytics_metric_version
(metric_version_id, metric_id, tenant_id, project_id, version, definition_json,
definition_hash, change_reason, created_by)
SELECT ${id}, ${input.metric_id}, ${scope.tenant_id}, ${scope.project_id},
coalesce(max(version), 0) + 1, ${input.definition_json}, ${hash},
${input.change_reason ?? null}, ${input.created_by ?? null}
FROM opt_analytics_metric_version
WHERE metric_id = ${input.metric_id} AND tenant_id = ${scope.tenant_id}
RETURNING *`;
const r = rows[0];
return {
metric_version_id: r.metric_version_id,
metric_id: r.metric_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
version: Number(r.version),
definition_json: r.definition_json,
definition_hash: r.definition_hash,
...(r.change_reason ? { change_reason: r.change_reason } : {}),
created_at: iso(r.created_at)!,
...(r.created_by ? { created_by: r.created_by } : {}),
};
}
// ── DimensionDefinition (the query compiler's allow-list) ────────────────
async createDimension(
scope: TenantScope,
input: Omit<DimensionDefinition, "tenant_id" | "project_id" | "created_at">,
): Promise<DimensionDefinition> {
assertScope(scope);
const rows = await this.sql<
{ dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[]
>`
INSERT INTO opt_analytics_dimension_definition
(dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description)
VALUES
(${input.dimension_id}, ${scope.tenant_id}, ${scope.project_id}, ${input.name}, ${input.dataset_id},
${input.field_name}, ${input.field_type}, ${input.classification}, ${input.description ?? null})
RETURNING *`;
const r = rows[0];
return {
dimension_id: r.dimension_id,
tenant_id: r.tenant_id,
project_id: r.project_id,
name: r.name,
dataset_id: r.dataset_id,
field_name: r.field_name,
field_type: r.field_type as DimensionDefinition["field_type"],
classification: r.classification as PiiClassification,
...(r.description ? { description: r.description } : {}),
created_at: iso(r.created_at)!,
};
}
/**
* The dimension allow-list the compiler resolves through, for one tenant.
* Returns caller-facing name -> physical field name. A tenant with no
* declared dimensions gets an EMPTY map, which means every dimension
* reference is refused — fail closed, not "allow everything".
*/
async dimensionCatalog(scope: TenantScope, datasetId?: string): Promise<Record<string, string>> {
assertScope(scope);
const rows = await this.sql<{ name: string; field_name: string }[]>`
SELECT name, field_name FROM opt_analytics_dimension_definition
WHERE tenant_id = ${scope.tenant_id}
AND project_id = ${scope.project_id}
${datasetId ? this.sql`AND dataset_id = ${datasetId}` : this.sql``}`;
const out: Record<string, string> = Object.create(null);
for (const r of rows) out[r.name] = r.field_name;
return out;
}
}
+511
View File
@@ -0,0 +1,511 @@
/**
* src/analytics/slo/bi-slo.ts
* ============================
* BI-P103 — BI SLO. DoD: "measured baselines exist".
*
* An SLO you cannot measure from real telemetry is a wish, so every objective
* here declares WHERE its number comes from, and the evaluator refuses to
* produce a number it does not have. Three ways that refusal shows up, each of
* which is a lie the usual implementation tells:
*
* insufficient_data nothing (or too little) was observed in the window.
* An availability objective evaluated over zero events
* is NOT 100% attainment — it is silence. This is the
* single most common false green in SLO reporting, and
* it is the same shape BI-P025 found in a quality
* check over an empty table.
* not_measurable_here the objective is real but these events cannot
* measure it. Dashboard data FRESHNESS is a property
* of the dataset (BI-P026 / BI-P105 watermarks), not
* of a usage event, and REPORT DELIVERY is N0TIFY's
* fact — aud0's seven types carry no delivery event.
* Saying so beats fabricating a proxy.
* low_confidence a p95 or p99 computed from a handful of samples. The
* number is reported WITH its sample size and marked,
* because a p99 over 12 requests is the 12th-slowest
* request wearing a percentile's clothes.
*
* The measurement path is aud0's seven BI usage event types (BI-P076, merged
* in aud0#34), produced by src/analytics/slo/usage-events.ts. There is
* deliberately no second telemetry path: two measurement systems disagree, and
* the disagreement is discovered during an incident.
*/
import type { Aud0BiEvent } from './usage-events';
export const BI_WORKLOAD_CLASSES = ['interactive_dashboard', 'exploration', 'report'] as const;
export type BiWorkloadClass = (typeof BI_WORKLOAD_CLASSES)[number];
export const SLO_OBJECTIVE_KINDS = [
'availability',
'latency_p95_first',
'latency_p95_cached',
'latency_p95',
'latency_p99',
'error_rate',
'freshness',
'max_result_rows',
'timeout',
'completion_sla',
'delivery_sla',
] as const;
export type SloObjectiveKind = (typeof SLO_OBJECTIVE_KINDS)[number];
/** Where the number comes from. `none` is an admission, not a placeholder. */
export const MEASUREMENT_SOURCES = [
'aud0_bi_usage_events',
'analytics_freshness_watermark',
'n0tify_delivery_receipt',
'engine_configuration',
] as const;
export type MeasurementSource = (typeof MEASUREMENT_SOURCES)[number];
export interface SloObjective {
objective_id: string;
workload: BiWorkloadClass;
kind: SloObjectiveKind;
/** Target value; unit is fixed by kind (ms, ratio, rows, seconds). */
target: number;
unit: 'ms' | 'ratio' | 'rows' | 'seconds';
/** Higher-is-better (availability) vs lower-is-better (latency). */
direction: 'at_least' | 'at_most';
measured_from: MeasurementSource;
/** Fewer samples than this and the verdict carries low_confidence. */
min_samples: number;
rationale: string;
}
/**
* The catalogue, field-for-field with what BI-P103 lists per workload class.
* Targets are STARTING POINTS chosen to be honest about what has been
* measured: they are the thresholds the evaluator compares against, and
* evidence/bi-p103/baseline.json records the first baselines rather than
* pretending the targets were derived from a year of production traffic.
*
* Those baselines are produced by
* tests/integration/bi-p103-slo-baseline.integration.test.ts, which runs 460
* real queries against a live ClickHouse and feeds them through this
* evaluator. Read the file's own caveats before quoting a number from it: the
* cached path measured there is the in-process reference cache rather than
* Valkey, every latency is the engine round trip with no HTTP or
* authorization layer around it, and there is no concurrency. Seven of the
* eleven objectives produced a number; the other four say why they could not,
* and `fully_measured` is false.
*/
export const BI_SLO_CATALOGUE: readonly SloObjective[] = [
// ── interactive dashboard ────────────────────────────────────────────────
{
objective_id: 'dash.availability',
workload: 'interactive_dashboard',
kind: 'availability',
target: 0.995,
unit: 'ratio',
direction: 'at_least',
measured_from: 'aud0_bi_usage_events',
min_samples: 100,
rationale: 'completed / (completed + failed) for dashboard-surface queries.',
},
{
objective_id: 'dash.p95_first_query',
workload: 'interactive_dashboard',
kind: 'latency_p95_first',
target: 3000,
unit: 'ms',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 50,
rationale: 'cache_hit=false: what a user waits for when they open the dashboard cold.',
},
{
objective_id: 'dash.p95_cached',
workload: 'interactive_dashboard',
kind: 'latency_p95_cached',
target: 400,
unit: 'ms',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 50,
rationale:
'cache_hit=true: BI-P020 cache hits, measured separately so a high hit rate cannot hide a slow cold path.',
},
{
objective_id: 'dash.error_rate',
workload: 'interactive_dashboard',
kind: 'error_rate',
target: 0.005,
unit: 'ratio',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 100,
rationale: 'failed / terminal. Cancellations count as failures here: the user did not get an answer.',
},
{
objective_id: 'dash.freshness',
workload: 'interactive_dashboard',
kind: 'freshness',
target: 900,
unit: 'seconds',
direction: 'at_most',
measured_from: 'analytics_freshness_watermark',
min_samples: 1,
rationale:
'Data age is a property of the DATASET (BI-P026 freshness, BI-P105 watermark), not of a usage ' +
'event. Usage events cannot measure it and this evaluator does not pretend to.',
},
// ── exploration ──────────────────────────────────────────────────────────
{
objective_id: 'explore.p95',
workload: 'exploration',
kind: 'latency_p95',
target: 10_000,
unit: 'ms',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 50,
rationale: 'explore-surface queries; an analyst tolerates more than a dashboard viewer.',
},
{
objective_id: 'explore.p99',
workload: 'exploration',
kind: 'latency_p99',
target: 60_000,
unit: 'ms',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 200,
rationale: 'p99 needs a genuinely large sample; below min_samples it is reported as low confidence.',
},
{
objective_id: 'explore.max_result',
workload: 'exploration',
kind: 'max_result_rows',
target: 1_000_000,
unit: 'rows',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 1,
rationale: 'BI-P012 truncates at the engine ceiling; this watches what actually came back.',
},
{
objective_id: 'explore.timeout',
workload: 'exploration',
kind: 'timeout',
target: 300,
unit: 'seconds',
direction: 'at_most',
measured_from: 'engine_configuration',
min_samples: 1,
rationale: 'The configured ceiling (BI-P012 settings profile), not an observation.',
},
// ── reports ──────────────────────────────────────────────────────────────
{
objective_id: 'report.completion_sla',
workload: 'report',
kind: 'completion_sla',
target: 900_000,
unit: 'ms',
direction: 'at_most',
measured_from: 'aud0_bi_usage_events',
min_samples: 20,
rationale: 'aud0.report_generated duration_ms: submitted to produced.',
},
{
objective_id: 'report.delivery_sla',
workload: 'report',
kind: 'delivery_sla',
target: 300_000,
unit: 'ms',
direction: 'at_most',
measured_from: 'n0tify_delivery_receipt',
min_samples: 20,
rationale:
"Produced is not delivered. aud0 has no delivery event; the receipt is N0TIFY's, so this " +
'objective is declared and reported unmeasured rather than proxied by completion.',
},
];
export const SLO_STATUSES = ['met', 'breached', 'insufficient_data', 'not_measurable_here'] as const;
export type SloStatus = (typeof SLO_STATUSES)[number];
export interface SloResult {
objective_id: string;
workload: BiWorkloadClass;
kind: SloObjectiveKind;
status: SloStatus;
target: number;
unit: SloObjective['unit'];
/** The measured value, or undefined when nothing could be measured. */
observed?: number;
sample_size: number;
/** True when a percentile was computed from fewer samples than min_samples. */
low_confidence?: boolean;
detail: string;
}
export interface SloReport {
window_from: string;
window_to: string;
results: SloResult[];
/** Queries that started and never reached a terminal event in the window. */
unresolved_queries: number;
/** True only when EVERY objective produced a real number. */
fully_measured: boolean;
}
interface QueryFact {
query_id: string;
surface?: string;
started_at: number;
outcome?: 'completed' | 'failed';
duration_ms?: number;
row_count?: number;
cache_hit?: boolean;
}
function num(v: unknown): number | undefined {
return typeof v === 'number' && Number.isFinite(v) ? v : undefined;
}
function str(v: unknown): string | undefined {
return typeof v === 'string' ? v : undefined;
}
/**
* Nearest-rank percentile. No interpolation: with small samples an
* interpolated p99 invents a value that no request ever experienced.
*/
export function percentile(sorted: number[], q: number): number | undefined {
if (sorted.length === 0) return undefined;
const rank = Math.ceil(q * sorted.length);
return sorted[Math.min(sorted.length - 1, Math.max(0, rank - 1))];
}
/** Fold the seven event types into per-query facts. */
function foldQueries(events: Aud0BiEvent[]): Map<string, QueryFact> {
const facts = new Map<string, QueryFact>();
for (const e of events) {
const qid = str(e.payload.query_id);
if (!qid) continue;
const at = Date.parse(e.occurred_at);
const existing = facts.get(qid) ?? { query_id: qid, started_at: Number.isNaN(at) ? 0 : at };
if (e.payload.surface !== undefined) existing.surface = str(e.payload.surface);
if (e.event_type === 'aud0.analytics_query_started' && !Number.isNaN(at)) existing.started_at = at;
if (e.event_type === 'aud0.analytics_query_completed') {
existing.outcome = 'completed';
existing.duration_ms = num(e.payload.duration_ms);
existing.row_count = num(e.payload.row_count);
existing.cache_hit = e.payload.cache_hit === true;
}
if (e.event_type === 'aud0.analytics_query_failed') {
existing.outcome = 'failed';
existing.duration_ms = num(e.payload.duration_ms);
}
facts.set(qid, existing);
}
return facts;
}
function result(
o: SloObjective,
status: SloStatus,
sample_size: number,
detail: string,
observed?: number,
low_confidence?: boolean,
): SloResult {
const r: SloResult = {
objective_id: o.objective_id,
workload: o.workload,
kind: o.kind,
status,
target: o.target,
unit: o.unit,
sample_size,
detail,
};
if (observed !== undefined) r.observed = observed;
if (low_confidence) r.low_confidence = true;
return r;
}
function compare(o: SloObjective, observed: number): SloStatus {
return o.direction === 'at_least'
? observed >= o.target
? 'met'
: 'breached'
: observed <= o.target
? 'met'
: 'breached';
}
/**
* Evaluate the catalogue over a window of AUD0 BI usage events.
*
* Events are DATA: they arrive from aud0's read model, they were written by
* other services, and nothing in them is treated as an instruction or as a
* schema. Unknown event types and unknown payload keys are ignored, never
* spread.
*/
export function evaluateBiSlos(args: {
events: Aud0BiEvent[];
from: Date;
to: Date;
objectives?: readonly SloObjective[];
}): SloReport {
const objectives = args.objectives ?? BI_SLO_CATALOGUE;
const fromMs = args.from.getTime();
const toMs = args.to.getTime();
const inWindow = args.events.filter((e) => {
const t = Date.parse(e.occurred_at);
return !Number.isNaN(t) && t >= fromMs && t <= toMs;
});
const queries = [...foldQueries(inWindow).values()];
const dashboardQueries = queries.filter((q) => q.surface === 'dashboard');
const exploreQueries = queries.filter((q) => q.surface === 'explore');
const terminal = (qs: QueryFact[]) => qs.filter((q) => q.outcome !== undefined);
const durations = (qs: QueryFact[]) =>
qs
.map((q) => q.duration_ms)
.filter((d): d is number => d !== undefined)
.sort((a, b) => a - b);
const reportDurations = inWindow
.filter((e) => e.event_type === 'aud0.report_generated')
.map((e) => num(e.payload.duration_ms))
.filter((d): d is number => d !== undefined)
.sort((a, b) => a - b);
const results = objectives.map((o) => {
if (o.measured_from !== 'aud0_bi_usage_events') {
return result(
o,
'not_measurable_here',
0,
`measured from ${o.measured_from}; BI usage events cannot answer it — ${o.rationale}`,
);
}
switch (o.kind) {
case 'availability':
case 'error_rate': {
const t = terminal(dashboardQueries);
if (t.length === 0) {
return result(o, 'insufficient_data', 0, 'no terminal dashboard query events in the window');
}
const failed = t.filter((q) => q.outcome === 'failed').length;
const observed = o.kind === 'availability' ? (t.length - failed) / t.length : failed / t.length;
const status = compare(o, observed);
return result(
o,
t.length < o.min_samples ? 'insufficient_data' : status,
t.length,
t.length < o.min_samples
? `${t.length} terminal queries is below the ${o.min_samples} this objective requires; ` +
`the ratio ${observed.toFixed(4)} is reported but not judged`
: `${failed} failed of ${t.length} terminal queries`,
observed,
t.length < o.min_samples,
);
}
case 'latency_p95_first':
case 'latency_p95_cached': {
const wanted = o.kind === 'latency_p95_cached';
const sample = durations(dashboardQueries.filter((q) => (q.cache_hit === true) === wanted));
const p95 = percentile(sample, 0.95);
if (p95 === undefined) {
return result(
o,
'insufficient_data',
0,
`no ${wanted ? 'cached' : 'uncached'} dashboard durations`,
);
}
const low = sample.length < o.min_samples;
return result(
o,
low ? 'insufficient_data' : compare(o, p95),
sample.length,
low
? `p95 over ${sample.length} samples (needs ${o.min_samples}) — reported, not judged`
: `p95 over ${sample.length} samples`,
p95,
low,
);
}
case 'latency_p95':
case 'latency_p99': {
const sample = durations(exploreQueries);
const q = o.kind === 'latency_p99' ? 0.99 : 0.95;
const value = percentile(sample, q);
if (value === undefined) return result(o, 'insufficient_data', 0, 'no exploration durations');
const low = sample.length < o.min_samples;
return result(
o,
low ? 'insufficient_data' : compare(o, value),
sample.length,
low
? `p${q * 100} over ${sample.length} samples (needs ${o.min_samples}); with this few samples ` +
`it is the ${Math.ceil((1 - q) * sample.length) || 1}-slowest request, not a percentile`
: `p${q * 100} over ${sample.length} samples`,
value,
low,
);
}
case 'max_result_rows': {
const rows = exploreQueries
.map((q) => q.row_count)
.filter((r): r is number => r !== undefined)
.sort((a, b) => b - a);
if (rows.length === 0) return result(o, 'insufficient_data', 0, 'no exploration row counts');
return result(o, compare(o, rows[0]), rows.length, `largest result seen in the window`, rows[0]);
}
case 'completion_sla': {
const p95 = percentile(reportDurations, 0.95);
if (p95 === undefined) return result(o, 'insufficient_data', 0, 'no report_generated events');
const low = reportDurations.length < o.min_samples;
return result(
o,
low ? 'insufficient_data' : compare(o, p95),
reportDurations.length,
low ? `p95 over ${reportDurations.length} reports (needs ${o.min_samples})` : 'p95 report duration',
p95,
low,
);
}
default:
return result(o, 'not_measurable_here', 0, `no evaluator for kind "${o.kind}"`);
}
});
const unresolved = queries.filter((q) => q.outcome === undefined).length;
return {
window_from: args.from.toISOString(),
window_to: args.to.toISOString(),
results,
unresolved_queries: unresolved,
fully_measured: results.every((r) => r.status === 'met' || r.status === 'breached'),
};
}
/**
* One line an operator can read: how many objectives are actually being
* measured. "9 of 11 met" is meaningless if the other two were never
* measured, so the summary counts them separately and never rolls them in.
*/
export function summariseSloReport(report: SloReport): {
met: number;
breached: number;
insufficient_data: number;
not_measurable_here: number;
measured_fraction: number;
} {
const count = (s: SloStatus) => report.results.filter((r) => r.status === s).length;
const measured = count('met') + count('breached');
return {
met: count('met'),
breached: count('breached'),
insufficient_data: count('insufficient_data'),
not_measurable_here: count('not_measurable_here'),
measured_fraction: report.results.length === 0 ? 0 : measured / report.results.length,
};
}
+209
View File
@@ -0,0 +1,209 @@
/**
* src/analytics/slo/usage-events.ts
* ==================================
* BI-P103's measurement path: 0pt emits the AUD0 BI usage events (aud0#34,
* BI-P076) that the SLO evaluator then reads.
*
* aud0 says it plainly in docs/BI_ANALYTICS_USAGE.md: "Zero real producers
* emit these seven types today." An SLO computed from a second, private
* telemetry path would be a number nobody else can audit, so 0pt becomes a
* producer instead of inventing one.
*
* THE PAYLOAD IS AN ALLOWLIST, BUILT FIELD BY NAMED FIELD. AUD0's ingest
* refuses result-bearing payloads with a 400 and its schemas are `.strict()`,
* so a leak here is a 400 rather than a breach — but a producer that has to be
* refused is a producer that is broken. `biUsageEventsForExecution` therefore
* never spreads an execution record; it names each field, and
* tests/analytics/bi-slo.test.ts feeds it a record carrying SQL text, an error
* message, a tenant id and a result sample, then asserts none of it appears in
* the output.
*
* WHAT IS DELIBERATELY NOT SENT: `compiled_query_hash` goes out as
* `query_hash` (aud0 requires a 64-hex digest and nothing else), the statement
* never leaves 0pt at all, `error_code` travels but `error_message` does not,
* and `row_count` travels but no row does.
*/
/** The seven event types, named exactly as aud0 registers them. */
export const AUD0_BI_EVENT_TYPES = [
'aud0.analytics_query_started',
'aud0.analytics_query_completed',
'aud0.analytics_query_failed',
'aud0.dashboard_viewed',
'aud0.report_generated',
'aud0.dataset_exported',
'aud0.metric_queried',
] as const;
export type Aud0BiEventType = (typeof AUD0_BI_EVENT_TYPES)[number];
/** aud0's closed surface list (BI_SURFACES). */
export const BI_SURFACES = ['dashboard', 'report', 'explore', 'api', 'scheduled', 'notebook'] as const;
export type BiSurface = (typeof BI_SURFACES)[number];
export interface Aud0BiEvent {
event_type: Aud0BiEventType;
/** Bounded, allowlisted metadata. Never a result, a statement or a message. */
payload: Record<string, string | number | boolean | string[]>;
/** Occurrence time, used by the SLO window. */
occurred_at: string;
}
/**
* The 0pt-side view of one execution. Intentionally a SUPERSET of what may be
* emitted — it carries the fields 0pt genuinely has, including the ones that
* must never leave — so that the mapping below is a real filter rather than a
* rename.
*/
export interface ExecutionTelemetry {
query_id: string;
/** 64-hex; anything else is dropped rather than reshaped. */
compiled_query_hash?: string;
/** NEVER emitted. Present here because 0pt has it. */
compiled_sql?: string;
/** NEVER emitted. */
error_message?: string;
error_code?: string;
/** NEVER emitted — a consumer must not be able to name a tenant, BI-P016. */
tenant_id?: string;
dataset_id?: string;
engine?: string;
surface?: BiSurface;
state: 'succeeded' | 'failed' | 'cancelled' | 'running';
started_at: string;
finished_at?: string;
duration_ms?: number;
row_count?: number;
bytes_scanned?: number;
cache_hit?: boolean;
filter_keys?: string[];
/** NEVER emitted: filter VALUES are facts about a subject. */
filter_values?: Record<string, string[]>;
/** NEVER emitted. */
result_sample?: unknown;
metric_name?: string;
dimension_keys?: string[];
granularity?: string;
dashboard_id?: string;
widget_count?: number;
}
const SHA256 = /^[0-9a-f]{64}$/;
const MAX_KEY_NAMES = 50;
const MAX_SHORT_TEXT = 120;
function shortText(v: string | undefined): string | undefined {
if (!v) return undefined;
return v.slice(0, MAX_SHORT_TEXT);
}
function keyNames(v: string[] | undefined): string[] | undefined {
if (!v || v.length === 0) return undefined;
return v.slice(0, MAX_KEY_NAMES).map((k) => k.slice(0, MAX_SHORT_TEXT));
}
function put(
payload: Record<string, string | number | boolean | string[]>,
key: string,
value: string | number | boolean | string[] | undefined,
): void {
if (value === undefined) return;
payload[key] = value;
}
/**
* Map one execution to its AUD0 usage events.
*
* A running execution emits only `started`: a query that has not finished is
* neither a success nor a failure, and the SLO evaluator counts it as
* unresolved rather than quietly as one or the other.
*/
export function biUsageEventsForExecution(t: ExecutionTelemetry): Aud0BiEvent[] {
const base: Record<string, string | number | boolean | string[]> = {};
put(base, 'query_id', t.query_id);
if (t.compiled_query_hash && SHA256.test(t.compiled_query_hash)) {
put(base, 'query_hash', t.compiled_query_hash);
}
put(base, 'dataset_id', t.dataset_id);
put(base, 'engine', shortText(t.engine));
put(base, 'surface', t.surface);
put(base, 'filter_keys', keyNames(t.filter_keys));
const events: Aud0BiEvent[] = [
{ event_type: 'aud0.analytics_query_started', payload: { ...base }, occurred_at: t.started_at },
];
if (t.state === 'succeeded') {
const payload = { ...base };
put(payload, 'duration_ms', t.duration_ms);
put(payload, 'row_count', t.row_count);
put(payload, 'bytes_scanned', t.bytes_scanned);
put(payload, 'cache_hit', t.cache_hit);
events.push({
event_type: 'aud0.analytics_query_completed',
payload,
occurred_at: t.finished_at ?? t.started_at,
});
} else if (t.state === 'failed' || t.state === 'cancelled') {
const payload = { ...base };
put(payload, 'duration_ms', t.duration_ms);
// A CODE. `error_message` is never mapped, at any length.
put(
payload,
'error_code',
shortText(t.error_code ?? (t.state === 'cancelled' ? 'CANCELLED' : undefined)),
);
events.push({
event_type: 'aud0.analytics_query_failed',
payload,
occurred_at: t.finished_at ?? t.started_at,
});
}
if (t.metric_name) {
const payload: Record<string, string | number | boolean | string[]> = {};
put(payload, 'metric_name', shortText(t.metric_name));
put(payload, 'dataset_id', t.dataset_id);
put(payload, 'surface', t.surface);
put(payload, 'engine', shortText(t.engine));
put(payload, 'dimension_keys', keyNames(t.dimension_keys));
put(payload, 'granularity', shortText(t.granularity));
put(payload, 'row_count', t.row_count);
put(payload, 'duration_ms', t.duration_ms);
events.push({
event_type: 'aud0.metric_queried',
payload,
occurred_at: t.finished_at ?? t.started_at,
});
}
if (t.dashboard_id) {
const payload: Record<string, string | number | boolean | string[]> = {};
put(payload, 'dashboard_id', t.dashboard_id);
put(payload, 'widget_count', t.widget_count);
put(payload, 'duration_ms', t.duration_ms);
put(payload, 'surface', t.surface ?? 'dashboard');
events.push({ event_type: 'aud0.dashboard_viewed', payload, occurred_at: t.started_at });
}
return events;
}
/** Every payload key this producer can ever emit. Pinned by test. */
export const EMITTABLE_PAYLOAD_KEYS = [
'query_id',
'query_hash',
'dataset_id',
'engine',
'surface',
'filter_keys',
'duration_ms',
'row_count',
'bytes_scanned',
'cache_hit',
'error_code',
'metric_name',
'dimension_keys',
'granularity',
'dashboard_id',
'widget_count',
] as const;
+376
View File
@@ -0,0 +1,376 @@
/**
* src/analytics/sql/analyst-sql-session.ts
* =========================================
* BI-P017 — ANALYST SQL CONTRACT.
*
* "For privileged analyst role only. Implement separate AnalyticsSQLSession
* using Trino/read-only endpoints. Security: role-gated; query timeout;
* row/byte limits; read only; tenant/project catalogs; audit; statement
* history."
*
* SEPARATE is the operative word, and it is why this is its own class rather
* than a method on `SiaxAnalyticsEngine`. `executeMetric` has no raw-SQL
* surface at all; if analyst SQL lived on the same object, the closed DSL
* would be one refactor away from growing one. `ClickHouseAnalyticsEngine
* .executeQuery` stays a documented refusal.
*
* The four layers are described in statement-guard.ts. This class owns layers
* 1 (role) and the enforcement of every named limit; the guard is layer 2;
* tenant-catalog.ts renders layers 3 and 4.
*
* AUDIT AND STATEMENT HISTORY — the deliberate split, because BI-P017 requires
* statement history and BI-P018 requires NOT storing sensitive raw SQL:
*
* * the BI-P018 execution record stores a HASH of the compiled statement
* and nothing else;
* * the analyst statement history stores the NORMALIZED statement (every
* literal replaced by `?`) always — that is the searchable, non-PII form;
* * the RAW statement is retained only when `retainRawStatements` is on, and
* the entry is classified so a retention job can find it. It is on by
* default for analyst sessions, because a privileged human executing
* unreviewed SQL against tenant data is exactly the case where "who ran
* what" must survive, and a masked statement cannot answer it.
*
* A caller who needs the non-PII form gets `normalized`; nothing forces raw
* SQL into a downstream sink.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { AnalyticsIdentity } from '../engine/siax-analytics-engine';
import type { ResourceBudget } from '../engine/resource-governance';
import { applyQuota, BUDGET_TIERS, DEFAULT_BUDGETS } from '../engine/resource-governance';
import { guardAnalystStatement, type GuardedStatement } from './statement-guard';
/** The CL0UD role BI-P017 gates on. Verified upstream; never self-asserted. */
export const ANALYST_SQL_ROLE = 'analytics.sql.analyst';
/**
* An identity carrying CL0UD-verified roles. Roles are absent by default and
* absence is a denial — a session must never open because a field was missing.
*/
export interface AnalystIdentity extends AnalyticsIdentity {
roles?: readonly string[];
}
export interface AnalystStatementRecord {
statement_id: string;
tenant_id: string;
project_id?: string;
principal_id: string;
/** Literals masked. Always present. */
normalized: string;
/** Present only when the session retains raw statements. */
raw?: string;
statement_hash: string;
referenced_objects: string[];
submitted_at: string;
finished_at?: string;
duration_ms?: number;
rows_returned?: number;
bytes_read?: number;
rows_read?: number;
outcome: 'admitted' | 'refused' | 'succeeded' | 'failed' | 'cancelled';
/** SiaxErrorCode on refusal/failure. Never an engine body. */
error_code?: string;
/** Short operator-facing reason. Never echoes the statement. */
refusal_reason?: string;
correlation_id?: string;
}
/** Where the history goes. AUD0 is the system of record; this is the port. */
export interface AnalystAuditSink {
record(entry: AnalystStatementRecord): Promise<void> | void;
}
/** The read-only endpoint the session executes against. */
export interface ReadOnlySqlEndpoint {
/** "clickhouse" | "trino" — recorded on every execution. */
readonly engine: string;
/**
* Execute an ALREADY-GUARDED statement as the tenant's restricted user.
* The implementation must not add privileges, and must apply `settings`.
*/
execute(args: {
sql: string;
queryId: string;
settings: Record<string, string>;
signal: AbortSignal;
}): Promise<SqlEndpointResult>;
/** Stop a running statement server-side. Returns true when it is gone. */
cancel(queryId: string): Promise<boolean>;
}
export interface SqlEndpointResult {
columns: string[];
rows: unknown[][];
rows_read?: number;
bytes_read?: number;
}
export interface AnalystSqlSessionOptions {
identity: AnalystIdentity;
endpoint: ReadOnlySqlEndpoint;
/** The tenant's own catalog of pre-filtered views (BI-P017 layer 4). */
catalog: string;
/** Objects inside it the analyst may read. */
objects: readonly string[];
audit: AnalystAuditSink;
/**
* Budget tier. ANALYTICAL is the default for a human analyst; the tier is
* still only ever TIGHTENED by a CL0UD quota, never loosened (BI-P012).
*/
tier?: (typeof BUDGET_TIERS)[number];
retainRawStatements?: boolean;
/** Injected for deterministic tests. */
now?: () => number;
}
export interface AnalystQueryOutcome {
statement_id: string;
query_id: string;
columns: string[];
rows: unknown[][];
duration_ms: number;
rows_read?: number;
bytes_read?: number;
/** Objects the statement was permitted to read, as resolved by the guard. */
referenced_objects: string[];
truncated: boolean;
}
export class AnalyticsSQLSession {
private readonly identity: AnalystIdentity;
private readonly endpoint: ReadOnlySqlEndpoint;
private readonly catalog: string;
private readonly objects: readonly string[];
private readonly audit: AnalystAuditSink;
private readonly budget: ResourceBudget;
private readonly retainRaw: boolean;
private readonly now: () => number;
private readonly running = new Map<string, AbortController>();
private closed = false;
/**
* Opening a session is itself the role gate. A caller without the analyst
* role never obtains an object that can run SQL — as opposed to obtaining
* one whose methods happen to check a flag.
*/
constructor(opts: AnalystSqlSessionOptions) {
const id = opts.identity;
if (!id?.tenant_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.sql',
'analyst SQL requires a verified tenant identity',
);
}
if (!id.principal_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics.sql',
'analyst SQL requires a verified principal',
);
}
const roles = id.roles ?? [];
if (!roles.includes(ANALYST_SQL_ROLE)) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.sql',
`analyst SQL requires the ${ANALYST_SQL_ROLE} role`,
);
}
if (opts.objects.length === 0) {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', 'session exposes no readable objects');
}
this.identity = id;
this.endpoint = opts.endpoint;
this.catalog = opts.catalog;
this.objects = opts.objects;
this.audit = opts.audit;
this.budget = applyQuota(DEFAULT_BUDGETS[opts.tier ?? 'ANALYTICAL'], id.quota);
this.retainRaw = opts.retainRawStatements ?? true;
this.now = opts.now ?? Date.now;
}
get resourceBudget(): ResourceBudget {
return { ...this.budget };
}
/**
* Guard a statement WITHOUT running it. Exposed so a UI can validate as the
* analyst types, and so the adversarial test corpus has a pure entry point.
*/
guard(sql: string): GuardedStatement {
return guardAnalystStatement(sql, { allowedCatalog: this.catalog, allowedObjects: this.objects });
}
async query(sql: string): Promise<AnalystQueryOutcome> {
if (this.closed) {
throw new SiaxError('CONFLICT', '0pt.analytics.sql', 'session is closed');
}
const statementId = crypto.randomUUID();
const submittedAt = new Date(this.now()).toISOString();
const base = {
statement_id: statementId,
tenant_id: this.identity.tenant_id,
project_id: this.identity.project_id,
principal_id: this.identity.principal_id,
correlation_id: this.identity.correlation_id,
submitted_at: submittedAt,
};
let guarded: GuardedStatement;
try {
guarded = this.guard(sql);
} catch (e) {
const err = e as SiaxError;
// A REFUSAL is audited as loudly as an execution. A guard whose
// rejections are invisible cannot be reviewed, and an analyst probing
// the boundary is exactly what an operator needs to see.
await this.audit.record({
...base,
normalized: safeNormalize(sql),
statement_hash: hashStatement(sql),
referenced_objects: [],
outcome: 'refused',
error_code: err.code ?? 'PERMANENT_FAILURE',
// `message` is already the sanitised "[CODE] source: detail" form; it
// never contains the caller's statement, only the guard's reason.
refusal_reason: err.message ?? 'refused',
});
throw e;
}
const queryId = this.newQueryId();
const controller = new AbortController();
this.running.set(queryId, controller);
const startedMs = this.now();
const record: AnalystStatementRecord = {
...base,
normalized: guarded.normalized,
...(this.retainRaw ? { raw: guarded.raw } : {}),
statement_hash: hashStatement(guarded.raw),
referenced_objects: guarded.referencedObjects,
outcome: 'admitted',
};
try {
const result = await this.endpoint.execute({
// The statement is passed THROUGH unchanged. This class does not
// rewrite SQL: a rewriter must be a correct parser of the dialect to
// be safe, and layers 3+4 do not need it to be one.
sql: guarded.raw,
queryId,
settings: this.executionSettings(),
signal: controller.signal,
});
const duration = this.now() - startedMs;
const truncated = result.rows.length >= this.budget.maxResultRows;
await this.audit.record({
...record,
outcome: 'succeeded',
finished_at: new Date(this.now()).toISOString(),
duration_ms: duration,
rows_returned: result.rows.length,
rows_read: result.rows_read,
bytes_read: result.bytes_read,
});
return {
statement_id: statementId,
query_id: queryId,
columns: result.columns,
rows: result.rows,
duration_ms: duration,
rows_read: result.rows_read,
bytes_read: result.bytes_read,
referenced_objects: guarded.referencedObjects,
truncated,
};
} catch (e) {
const err = e as SiaxError;
await this.audit.record({
...record,
outcome: controller.signal.aborted ? 'cancelled' : 'failed',
finished_at: new Date(this.now()).toISOString(),
duration_ms: this.now() - startedMs,
error_code: err.code ?? 'PERMANENT_FAILURE',
});
throw e;
} finally {
this.running.delete(queryId);
}
}
/** Cancel a statement this session started. Never another session's. */
async cancel(queryId: string): Promise<boolean> {
if (!queryId.startsWith(this.queryIdPrefix())) {
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.sql',
'statement does not belong to this session',
);
}
this.running.get(queryId)?.abort();
return this.endpoint.cancel(queryId);
}
/** Abort everything in flight. Idempotent. */
async close(): Promise<void> {
this.closed = true;
const ids = [...this.running.keys()];
for (const id of ids) {
this.running.get(id)?.abort();
try {
await this.endpoint.cancel(id);
} catch {
/* best effort; the abort already stopped us waiting */
}
this.running.delete(id);
}
}
/**
* Every limit BI-P017 names, as engine settings. `readonly` is `1`, not `2`:
* `readonly = 2` permits SETTINGS changes, which would let a statement raise
* its own ceilings.
*/
private executionSettings(): Record<string, string> {
return {
readonly: '1',
max_execution_time: String(this.budget.maxExecutionTimeSec),
timeout_overflow_mode: 'throw',
max_result_rows: String(this.budget.maxResultRows),
result_overflow_mode: 'throw',
max_rows_to_read: String(this.budget.maxRowsToRead),
max_bytes_to_read: String(this.budget.maxBytesToRead),
read_overflow_mode: 'throw',
max_memory_usage: String(this.budget.maxMemoryUsageBytes),
};
}
private queryIdPrefix(): string {
return `sql_${crypto.createHash('sha256').update(this.identity.tenant_id).digest('hex').slice(0, 12)}_`;
}
private newQueryId(): string {
return `${this.queryIdPrefix()}${crypto.randomUUID().replace(/-/g, '')}`;
}
}
export function hashStatement(sql: string): string {
return crypto.createHash('sha256').update(sql.trim()).digest('hex');
}
/**
* Mask literals in a statement that FAILED the guard, so a refusal can be
* audited without copying whatever the caller pasted. Best-effort by design:
* the statement did not parse cleanly, so neither will this.
*/
function safeNormalize(sql: string): string {
return sql
.replace(/'(?:[^'\\]|\\.|'')*'/g, '?')
.replace(/\s+/g, ' ')
.trim()
.slice(0, 2000);
}
+516
View File
@@ -0,0 +1,516 @@
/**
* src/analytics/sql/statement-guard.ts
* =====================================
* BI-P017 — ANALYST SQL CONTRACT, layer 2 (statement admission).
*
* READ THIS BEFORE TRUSTING IT.
*
* This module is a GUARD, not a proof. It is the second of four layers, and
* it is the WEAKEST of the four, because it reasons about SQL text without
* being ClickHouse's parser. A guard that reads a dialect it does not
* implement can always be wrong about some string. The layers below it are
* the ones that hold when it is:
*
* 1. ROLE — only a CL0UD-verified analyst role reaches a session
* at all (analyst-sql-session.ts).
* 2. THIS GUARD — statement admission: one read-only statement, no
* banned construct, every table reference resolved
* into the tenant's own catalog.
* 3. GRANTS — the session authenticates as a per-tenant ClickHouse
* user with `readonly = 1` and SELECT granted ONLY on
* that tenant's catalog. A statement this guard wrongly
* admits still cannot read a table the user cannot see.
* 4. VIEW PREDICATE — the tenant catalog contains VIEWS, never base tables,
* and each view bakes `WHERE tenant_id = '<tenant>'`
* into its own definition. A row policy on the base
* tables repeats it server-side.
*
* tests/integration/analyst-sql-clickhouse.integration.test.ts runs the
* adversarial corpus against a live ClickHouse with layer 2 REMOVED, and
* requires layers 3 and 4 to refuse it anyway. A defence-in-depth claim that
* has never been tested with a layer removed is not evidence — this estate
* has shipped several.
*
* DESIGN NOTES that are load-bearing:
*
* * Identifiers must resolve to an ALLOW-LIST. The deny-lists below are a
* second opinion and a better error message, never the mechanism. Every
* table reference is checked positively against the tenant's catalog.
* * Outside string literals the statement must be ASCII. Homoglyph and
* zero-width tricks against keyword matching stop being expressible.
* * The FROM/JOIN scan is CONSERVATIVE where it is unsure, and that costs
* something worth naming: ClickHouse's `ARRAY JOIN arr` is a legitimate
* construct whose target is a COLUMN, not a table, and this guard refuses
* it because the token after JOIN is neither a CTE nor a qualified object.
* A false refusal an analyst can see and report is the right side of that
* trade; a false ADMISSION is not. Supporting it means teaching the scan
* which JOIN flavours take a non-table target, which is the direction that
* grows holes, so it stays refused until someone needs it and can add it
* WITH a corpus entry proving the narrowing did not open anything else.
* * Comments and string literals are removed BEFORE any keyword scan, using
* ClickHouse's own lexical rules (`--` and `#` to end of line, `/* *\/`
* non-nesting, `'` with backslash and doubled-quote escapes, `"` and
* backtick as identifier quotes).
*/
import { SiaxError } from '../../platform/errors';
export interface StatementGuardOptions {
/**
* The ONLY database name a table reference may carry. This is the tenant's
* own catalog of pre-filtered views, not the shared analytics database.
*/
allowedCatalog: string;
/** Object names inside that catalog the analyst may read. */
allowedObjects: readonly string[];
/** Hard ceiling on statement length; a megabyte of SQL is an attack, not a query. */
maxLength?: number;
}
export interface GuardedStatement {
/** The statement as submitted, unchanged. */
raw: string;
/** Comments and literals removed — what the keyword scan actually saw. */
stripped: string;
/** Literals replaced by `?`, for statement history that is not a PII dump. */
normalized: string;
/** Fully-qualified objects the statement reads, resolved against the catalog. */
referencedObjects: string[];
/** Common-table-expression names defined by the statement itself. */
cteNames: string[];
}
const DEFAULT_MAX_LENGTH = 20_000;
/**
* Statement-level constructs that make a statement not-a-read. Matched as
* whole words against the stripped text.
*
* `SETTINGS` is here because a trailing `SETTINGS max_rows_to_read = …` is the
* obvious way to try to lift the governance ceiling from inside the query.
* `FORMAT` is here because the session appends its own; a caller-supplied one
* is either a duplicate or an attempt at `FORMAT ...` side effects.
*/
const BANNED_KEYWORDS = [
'insert',
'update',
'delete',
'drop',
'alter',
'create',
'truncate',
'rename',
'attach',
'detach',
'replace',
'optimize',
'grant',
'revoke',
'kill',
'system',
'use',
'set',
'settings',
'exchange',
'move',
'undrop',
'backup',
'restore',
'outfile',
'infile',
'format',
'watch',
'check',
'explain',
'show',
'exists',
'call',
'execute',
'declare',
'begin',
'commit',
'rollback',
] as const;
/**
* Functions that read something other than a table the grants control:
* the filesystem, the network, another server, a dictionary, an executable.
* ClickHouse table functions are the whole reason an analyst SQL surface is
* dangerous — `SELECT * FROM url('http://…')` needs no table privilege at all.
*/
const BANNED_FUNCTIONS = [
'file',
'url',
'urlcluster',
's3',
's3cluster',
'gcs',
'azureblobstorage',
'remote',
'remotesecure',
'cluster',
'clusterallreplicas',
'hdfs',
'hdfscluster',
'mysql',
'postgresql',
'mongodb',
'jdbc',
'odbc',
'sqlite',
'redis',
'deltalake',
'hudi',
'iceberg',
'executable',
'input',
'generaterandom',
'numbers_mt',
'merge',
'view',
'viewifpermitted',
'loop',
'fuzzjson',
'fuzzquery',
'dictget',
'dictgetordefault',
'dictgetornull',
'dictgetstring',
'dictgetuint64',
'dictgethierarchy',
'dictisin',
'dicthas',
'dictgetchildren',
'dictgetdescendants',
'filesystemavailable',
'filesystemfree',
'filesystemcapacity',
'currentdatabase',
'currentuser',
'hostname',
'fqdn',
'uptime',
'version',
'getsetting',
'getmacro',
'shardnum',
'shardcount',
'buildid',
'servertimezone',
'readwkt',
'structurefromjson',
's3queue',
'timeseries',
'mergetreeindex',
] as const;
/**
* Constructs that a whole-word keyword ban gets WRONG, so they get a pattern.
*
* `DESC` is the case that proves the point: banning the word breaks every
* `ORDER BY x DESC` an analyst will ever write, and the test corpus caught it
* immediately. What is actually dangerous is `DESCRIBE`/`DESC` used as a
* STATEMENT — including inside a subquery, where ClickHouse accepts
* `SELECT * FROM (DESCRIBE TABLE x)` and the FROM-scan sees only a `(`.
*/
const BANNED_PATTERNS: Array<{ re: RegExp; reason: string }> = [
{ re: /\bdesc(ribe)?\s+(table|\()/i, reason: 'DESCRIBE is not permitted in a read-only analyst session' },
{ re: /\bcurrent_user\b/i, reason: 'CURRENT_USER is not permitted in an analyst session' },
{ re: /\binto\s+outfile\b/i, reason: 'INTO OUTFILE is not permitted in an analyst session' },
];
/** Names that are catalogs an analyst may never reach, whatever the grants say. */
const BANNED_CATALOGS = [
'system',
'information_schema',
'INFORMATION_SCHEMA',
'default',
'opt_analytics',
'opt_scraped',
];
const IDENT = '[A-Za-z_][A-Za-z0-9_]*';
const QUALIFIED_RE = new RegExp(`^${IDENT}\\.${IDENT}$`);
function deny(detail: string): never {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.sql', detail);
}
function invalid(detail: string): never {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', detail);
}
interface StripResult {
stripped: string;
normalized: string;
/** Identifier tokens that were quoted, kept so they can still be resolved. */
hadQuotedIdentifier: boolean;
}
/**
* Remove comments and string literals using ClickHouse's lexical rules.
*
* Two outputs, deliberately: `stripped` keeps the literal DELIMITERS so the
* token stream still parses (`'x'` becomes `''`), and `normalized` replaces
* each literal with `?` so a statement history is searchable without being a
* copy of the tenant's data.
*
* An unterminated literal or comment is a REJECTION. Silently treating the
* rest of the statement as a string is exactly how a guard and an engine come
* to disagree about where the query ends.
*/
export function stripSqlText(sql: string): StripResult {
let stripped = '';
let normalized = '';
let hadQuotedIdentifier = false;
let i = 0;
const n = sql.length;
while (i < n) {
const c = sql[i];
const next = sql[i + 1];
// -- line comment
if (c === '-' && next === '-') {
while (i < n && sql[i] !== '\n') i += 1;
stripped += ' ';
normalized += ' ';
continue;
}
// # line comment (ClickHouse accepts `#` and `#!`)
if (c === '#') {
while (i < n && sql[i] !== '\n') i += 1;
stripped += ' ';
normalized += ' ';
continue;
}
// /* block comment */ — ClickHouse does NOT nest these
if (c === '/' && next === '*') {
const end = sql.indexOf('*/', i + 2);
if (end === -1) invalid('unterminated block comment');
i = end + 2;
stripped += ' ';
normalized += ' ';
continue;
}
// 'string literal'
if (c === "'") {
i += 1;
while (i < n) {
if (sql[i] === '\\') {
i += 2;
continue;
}
if (sql[i] === "'") {
if (sql[i + 1] === "'") {
i += 2;
continue;
}
break;
}
i += 1;
}
if (i >= n) invalid('unterminated string literal');
i += 1; // closing quote
stripped += "''";
normalized += '?';
continue;
}
// "quoted identifier" / `quoted identifier`
if (c === '"' || c === '`') {
const quote = c;
let ident = '';
i += 1;
while (i < n) {
if (sql[i] === '\\') {
ident += sql[i + 1] ?? '';
i += 2;
continue;
}
if (sql[i] === quote) {
if (sql[i + 1] === quote) {
ident += quote;
i += 2;
continue;
}
break;
}
ident += sql[i];
i += 1;
}
if (i >= n) invalid('unterminated quoted identifier');
i += 1;
hadQuotedIdentifier = true;
// The identifier is put back UNQUOTED so the table-reference scan sees
// it exactly as it would see a bare one. A quoted identifier that does
// not survive that scan is refused like any other.
if (!new RegExp(`^${IDENT}$`).test(ident)) {
deny(`quoted identifier "${ident.slice(0, 32)}" is not a plain name`);
}
stripped += ident;
normalized += ident;
continue;
}
stripped += c;
normalized += c;
i += 1;
}
return { stripped, normalized, hadQuotedIdentifier };
}
/**
* Admit a statement, or refuse it.
*
* Returns the parsed view of what was admitted; the caller (the session) is
* responsible for actually running it under the restricted user.
*/
export function guardAnalystStatement(sql: string, opts: StatementGuardOptions): GuardedStatement {
const maxLength = opts.maxLength ?? DEFAULT_MAX_LENGTH;
if (typeof sql !== 'string') invalid('statement must be a string');
const raw = sql.trim();
if (!raw) invalid('statement is empty');
if (raw.length > maxLength) invalid(`statement exceeds ${maxLength} characters`);
if (!opts.allowedCatalog || !new RegExp(`^${IDENT}$`).test(opts.allowedCatalog)) {
// A misconfigured session must not degrade to "any catalog".
deny('session has no valid tenant catalog');
}
if (BANNED_CATALOGS.includes(opts.allowedCatalog)) {
deny(`session catalog "${opts.allowedCatalog}" is a reserved catalog`);
}
if (opts.allowedObjects.length === 0) {
deny('session exposes no readable objects');
}
const { stripped, normalized } = stripSqlText(raw);
// ── ASCII-only outside literals ────────────────────────────────────────
// Homoglyphs, zero-width joiners and RTL overrides stop being expressible
// against the keyword scan. Values are unaffected: they live in literals,
// which have already been removed at this point.
// Written as a code-point scan rather than a character class: a regex of
// control characters is unreadable, trips `no-control-regex`, and is the
// kind of expression a reviewer nods at without checking. Permitted: tab,
// LF, CR and printable ASCII. Everything else, NUL included, is refused.
for (let i = 0; i < stripped.length; i += 1) {
const code = stripped.charCodeAt(i);
const printable = code >= 0x20 && code <= 0x7e;
const allowedWhitespace = code === 0x09 || code === 0x0a || code === 0x0d;
if (!printable && !allowedWhitespace) {
deny(
`statement contains a disallowed character (U+${code.toString(16).toUpperCase().padStart(4, '0')}) ` +
`outside a string literal`,
);
}
}
// ── exactly one statement ──────────────────────────────────────────────
const withoutTrailingSemi = stripped.replace(/;\s*$/, '');
if (withoutTrailingSemi.includes(';')) {
deny('multiple statements are not allowed in an analyst SQL session');
}
const lower = ` ${withoutTrailingSemi.toLowerCase().replace(/\s+/g, ' ').trim()} `;
// ── must be a read ─────────────────────────────────────────────────────
if (!/^\s*(select|with)\b/i.test(withoutTrailingSemi.trim())) {
deny('analyst SQL sessions accept SELECT and WITH statements only');
}
for (const kw of BANNED_KEYWORDS) {
if (new RegExp(`(^|[^A-Za-z0-9_])${kw}([^A-Za-z0-9_]|$)`).test(lower)) {
deny(`"${kw.toUpperCase()}" is not permitted in a read-only analyst session`);
}
}
for (const { re, reason } of BANNED_PATTERNS) {
if (re.test(lower)) deny(reason);
}
// ── banned functions, matched as `name(` ───────────────────────────────
const calls = [...lower.matchAll(new RegExp(`([a-z_][a-z0-9_]*)\\s*\\(`, 'g'))].map((m) => m[1]);
for (const call of calls) {
if ((BANNED_FUNCTIONS as readonly string[]).includes(call)) {
deny(`function "${call}" is not permitted in an analyst session`);
}
}
// ── table references: positively resolved against the catalog ──────────
const cteNames = collectCteNames(withoutTrailingSemi);
const referencedObjects = collectTableReferences(withoutTrailingSemi, cteNames, opts);
if (referencedObjects.length === 0) {
// A SELECT with no FROM (`SELECT 1`) is harmless, but so is refusing it,
// and "no table reference" is also what a successful parser evasion looks
// like. Refuse: an analyst session exists to read tables.
deny('statement references no readable object in the tenant catalog');
}
return { raw, stripped, normalized, referencedObjects, cteNames };
}
/** `WITH x AS (…), y AS (…)` — names the statement defines for itself. */
function collectCteNames(stripped: string): string[] {
const names: string[] = [];
const re = new RegExp(`(?:^|[,\\s(])(${IDENT})\\s+as\\s*\\(`, 'gi');
for (const m of stripped.matchAll(re)) names.push(m[1].toLowerCase());
return names;
}
/**
* Every `FROM` / `JOIN` target, resolved.
*
* Rules, all fail-closed:
* * `(` after FROM/JOIN is a subquery — fine, its own FROM is scanned too.
* * `name(` after FROM/JOIN is a TABLE FUNCTION — refused outright, even if
* the name is not on the deny-list, because an unknown table function is
* precisely the case the deny-list cannot cover.
* * a bare `name` must be a CTE this statement defined.
* * a `db.name` must have `db === allowedCatalog` and `name` in the
* allow-list. Three-part `a.b.c` names are refused.
*/
function collectTableReferences(stripped: string, cteNames: string[], opts: StatementGuardOptions): string[] {
const allowed = new Set(opts.allowedObjects.map((o) => o.toLowerCase()));
const ctes = new Set(cteNames);
const found = new Set<string>();
const re = /(^|[^A-Za-z0-9_])(from|join)([^A-Za-z0-9_]|$)/gi;
let m: RegExpExecArray | null;
while ((m = re.exec(stripped)) !== null) {
const after = stripped.slice(m.index + m[0].length - 1);
const target = /^\s*([A-Za-z_][A-Za-z0-9_.]*|\()/.exec(after);
if (!target) deny('could not resolve a FROM/JOIN target');
const token = target[1];
if (token === '(') continue; // subquery; its own FROM is scanned by this loop
const rest = after.slice(target[0].length);
if (/^\s*\(/.test(rest)) {
deny(`table function "${token}" is not permitted in an analyst session`);
}
const parts = token.split('.');
if (parts.length === 1) {
if (!ctes.has(parts[0].toLowerCase())) {
deny(
`unqualified table "${parts[0]}" — an analyst statement must name ` +
`${opts.allowedCatalog}.<object>, or a CTE it defined itself`,
);
}
continue;
}
if (parts.length !== 2 || !QUALIFIED_RE.test(token)) {
deny(`"${token}" is not a valid <catalog>.<object> reference`);
}
const [catalog, object] = parts;
if (catalog.toLowerCase() !== opts.allowedCatalog.toLowerCase()) {
deny(`catalog "${catalog}" is not readable by this session`);
}
if (!allowed.has(object.toLowerCase())) {
deny(`object "${catalog}.${object}" is not readable by this session`);
}
found.add(`${catalog}.${object}`);
}
return [...found];
}
/** Exported for the test corpus and for operator documentation. */
export const GUARD_BANNED_KEYWORDS = BANNED_KEYWORDS;
export const GUARD_BANNED_FUNCTIONS = BANNED_FUNCTIONS;
+238
View File
@@ -0,0 +1,238 @@
/**
* src/analytics/sql/tenant-catalog.ts
* ====================================
* BI-P017 — ANALYST SQL CONTRACT, layers 3 and 4.
*
* The analyst never touches a base table. Each tenant gets its OWN ClickHouse
* database containing nothing but VIEWS, and every view has the BI-P011 forced
* predicate baked into its own definition:
*
* CREATE VIEW t_alpha_sql.opt_analytics_event AS
* SELECT … FROM opt_analytics.opt_analytics_event
* WHERE tenant_id = 't_alpha'
*
* That is the difference between "the guard rewrites your SQL to add a WHERE"
* (fragile — it depends on the guard being a correct SQL parser) and "there is
* no unfiltered object in reach" (structural — it depends only on the grants).
*
* The session's ClickHouse user is created with `readonly = 1` and is granted
* SELECT on that database ALONE. `GRANT SELECT ON opt_analytics.*` is never
* issued, so a statement the guard wrongly admits still cannot read the base
* table: ClickHouse answers "Not enough privileges", not rows.
*
* The `tenant_id` column is deliberately DROPPED from the view projection.
* An analyst cannot filter on, group by, or select a column that does not
* exist in the object they can see — which removes the whole class of
* "predicate on tenant_id looks legitimate" confusion from the audit trail.
*/
import { SiaxError } from '../../platform/errors';
import { assertSafeIdentifier } from '../engine/query-compiler';
import type { ResourceBudget } from '../engine/resource-governance';
export interface TenantSqlCatalogSpec {
/** Shared analytics database holding the base tables. */
sourceDatabase: string;
/** Verified tenant id. Embedded as a literal — validated strictly first. */
tenantId: string;
/** The tenant's own catalog name, e.g. `t_alpha_sql`. */
catalog: string;
/** Base tables to expose, with the columns each view projects. */
objects: ReadonlyArray<{ table: string; columns: readonly string[] }>;
}
export interface TenantSqlUserSpec extends TenantSqlCatalogSpec {
userName: string;
/** From a secret ref. Never a literal in code, never logged. */
password: string;
/**
* BI-P012 ceilings. MEASURED CONSTRAINT, not a style choice: a ClickHouse
* user whose profile carries `readonly = 1` cannot have ANY setting changed
* from the request — the server answers
*
* Code: 164. Cannot modify 'max_rows_to_read' setting in readonly mode.
*
* (measured on 24.8.14.39; see evidence/bi-p017/). So the ceilings cannot be
* client-supplied per request the way `clickhouseSettings()` supplies them
* for the non-readonly engine user. They must be baked into the user's
* SETTINGS PROFILE at provisioning time.
*
* That is strictly BETTER: the ceiling no longer depends on the client
* remembering to send it, and the same `readonly = 1` that forbids raising
* it also forbids lowering it or turning it off.
*/
budget?: ResourceBudget;
/** Explicit profile name; defaults to `<userName>_profile`. */
settingsProfile?: string;
}
const TENANT_ID_RE = /^[A-Za-z0-9_-]{1,64}$/;
function assertTenantId(tenantId: string): string {
if (!TENANT_ID_RE.test(tenantId)) {
// ClickHouse view definitions take no bound parameters, so validation is
// the only protection available here and it is applied strictly.
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', 'tenant id is not catalog-safe');
}
return tenantId;
}
/**
* DDL creating the tenant's read-only catalog of pre-filtered views.
* Idempotent; safe to re-run on every deploy.
*/
export function tenantSqlCatalogDdl(spec: TenantSqlCatalogSpec): string[] {
assertSafeIdentifier(spec.sourceDatabase, 'database');
assertSafeIdentifier(spec.catalog, 'catalog');
assertTenantId(spec.tenantId);
if (spec.catalog === spec.sourceDatabase) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.sql',
'tenant catalog must not be the source database',
);
}
const stmts: string[] = [`CREATE DATABASE IF NOT EXISTS ${spec.catalog}`];
for (const obj of spec.objects) {
assertSafeIdentifier(obj.table, 'table');
if (obj.columns.length === 0) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.sql', `view ${obj.table} projects no columns`);
}
for (const col of obj.columns) {
assertSafeIdentifier(col, 'column');
if (col === 'tenant_id') {
// Not an oversight to guard against — a deliberate refusal. Exposing
// tenant_id inside a single-tenant view invites a reader to believe
// filtering on it means something.
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.sql',
'tenant_id must not be projected into a tenant SQL view',
);
}
}
stmts.push(
`CREATE OR REPLACE VIEW ${spec.catalog}.${obj.table} AS ` +
`SELECT ${obj.columns.join(', ')} FROM ${spec.sourceDatabase}.${obj.table} ` +
`WHERE tenant_id = '${spec.tenantId}'`,
);
}
return stmts;
}
/**
* DDL for the session's ClickHouse user.
*
* Every line here is a layer-3 control:
* * `readonly = 1` — no writes AND no settings changes from inside a query,
* which is what stops `SETTINGS max_rows_to_read = …` lifting the ceiling.
* * `allow_ddl = 0` — belt to readonly's braces.
* * SELECT granted on the tenant catalog ONLY. The base database is never
* granted, so a guard bypass hits a privilege error.
* * A row policy on the BASE tables as well, so even a mistaken grant, or a
* view accidentally created without its predicate, still filters.
*/
export function tenantSqlUserDdl(spec: TenantSqlUserSpec): string[] {
assertSafeIdentifier(spec.userName, 'user');
assertSafeIdentifier(spec.catalog, 'catalog');
assertSafeIdentifier(spec.sourceDatabase, 'database');
assertTenantId(spec.tenantId);
if (spec.password.includes("'") || spec.password.includes('\\')) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.sql',
'password contains characters that cannot be embedded',
);
}
if (spec.settingsProfile) assertSafeIdentifier(spec.settingsProfile, 'settings profile');
const profileName = spec.settingsProfile ?? `${spec.userName}_profile`;
assertSafeIdentifier(profileName, 'settings profile');
const stmts: string[] = [];
if (spec.budget) {
// Every BI-P012 ceiling, plus the overflow modes. `throw` is explicit
// because ClickHouse's default for some of these is `break`, which returns
// a PARTIAL result that looks complete — the analytics equivalent of a
// false green.
// `OR REPLACE`, for the same measured reason as the row policy below:
// `CREATE SETTINGS PROFILE IF NOT EXISTS` is a silent no-op when the
// profile already exists, so a CHANGED budget never lands. Found the hard
// way — a re-provisioning run kept a stale 60-row ceiling and starved
// every legitimate analyst statement in the integration suite, while the
// provisioning DDL reported success on every statement.
stmts.push(
`CREATE SETTINGS PROFILE OR REPLACE ${profileName} SETTINGS ` +
`max_execution_time = ${spec.budget.maxExecutionTimeSec}, ` +
`timeout_overflow_mode = 'throw', ` +
`max_result_rows = ${spec.budget.maxResultRows}, ` +
`result_overflow_mode = 'throw', ` +
`max_rows_to_read = ${spec.budget.maxRowsToRead}, ` +
`max_bytes_to_read = ${spec.budget.maxBytesToRead}, ` +
`read_overflow_mode = 'throw', ` +
`max_memory_usage = ${spec.budget.maxMemoryUsageBytes}, ` +
`readonly = 1`,
);
}
const settings = spec.budget ? `SETTINGS PROFILE ${profileName}` : `SETTINGS readonly = 1, allow_ddl = 0`;
stmts.push(
`CREATE USER IF NOT EXISTS ${spec.userName} IDENTIFIED WITH sha256_password BY '${spec.password}' ${settings}`,
);
// Re-applied on every provisioning run so a rotated budget actually lands on
// an already-existing user; CREATE USER IF NOT EXISTS would silently skip it.
stmts.push(`ALTER USER ${spec.userName} ${settings}`);
stmts.push(`GRANT SELECT ON ${spec.catalog}.* TO ${spec.userName}`);
for (const obj of spec.objects) {
assertSafeIdentifier(obj.table, 'table');
// A view reads its source with the DEFINER's rights in ClickHouse only for
// parameterised/materialised cases; for a plain view the reader still needs
// SELECT on the source. Grant it on the SOURCE TABLE and pin a row policy
// to it, so the grant can never be wider than the tenant.
stmts.push(`GRANT SELECT ON ${spec.sourceDatabase}.${obj.table} TO ${spec.userName}`);
// `OR REPLACE`, never `IF NOT EXISTS`. MEASURED FAILURE, found by a test
// that attempted the bypass:
//
// Dropping and re-creating the user empties the policy's apply-to list
// (ClickHouse removes the dropped user from it). On the next
// provisioning run `CREATE ROW POLICY IF NOT EXISTS` sees the policy
// already exists and does NOTHING — so the policy survives with
// `apply_to_list = []`, applying to nobody, while `system.row_policies`
// still shows the right `select_filter`. An operator reading that table
// sees a policy protecting `opt_analytics.sales` and is wrong: the
// restricted user read 100 rows of 100, both tenants.
//
// This is the "green check that does not gate" shape. `OR REPLACE` makes
// provisioning idempotent in the way that was intended, and
// `rowPolicyVerificationSql` below turns "the policy is applied" from an
// assumption into a query.
stmts.push(
`CREATE ROW POLICY OR REPLACE ${spec.userName}_${obj.table}_rp ON ${spec.sourceDatabase}.${obj.table} ` +
`AS restrictive FOR SELECT USING tenant_id = '${spec.tenantId}' TO ${spec.userName}`,
);
}
return stmts;
}
/**
* The query that PROVES the row policies landed, rather than assuming it.
*
* Returns one row per (table, policy) with `applied_to_user` — which must be
* 1 for every object. Run it after provisioning and fail the deploy if any
* row reads 0; a policy with an empty apply-to list is indistinguishable from
* a working one in `SHOW ROW POLICIES`.
*/
export function rowPolicyVerificationSql(spec: TenantSqlUserSpec): string {
assertSafeIdentifier(spec.userName, 'user');
assertSafeIdentifier(spec.sourceDatabase, 'database');
assertTenantId(spec.tenantId);
return (
`SELECT table, short_name, has(apply_to_list, '${spec.userName}') AS applied_to_user, select_filter ` +
`FROM system.row_policies WHERE database = '${spec.sourceDatabase}' ` +
`AND short_name LIKE '${spec.userName}%' FORMAT JSON`
);
}
/** Object names the guard's allow-list is built from. */
export function catalogObjectNames(spec: TenantSqlCatalogSpec): string[] {
return spec.objects.map((o) => o.table);
}
+374
View File
@@ -0,0 +1,374 @@
/**
* src/analytics/transform/dbt-contract.ts
* ========================================
* BI-P022 — DBT CORE CANONICAL TRANSFORMATION.
*
* ── SCOPE, STATED FIRST ───────────────────────────────────────────────────
* This module defines the CONTRACT. It does not run dbt, and nothing in this
* repo does. Execution belongs to N0D (n0d#38), which consumes this contract
* from the other side. There is deliberately no `DbtExecutor` implementation
* here — only the interface — so that "0pt can run dbt" cannot become true by
* accident, and so the two repos have one shared, versioned shape rather than
* two drifting ones.
*
* ── THE VERSION RULE BI-P022 STATES ───────────────────────────────────────
* "Use stable dbt Core production line AFTER current license/security
* verification. Do NOT use alpha development engine in production unless
* separately certified."
*
* So the pin is recorded together with its certification STATE, and the state
* is `unverified` until someone does the review. `assertDbtRuntimeCertified`
* refuses a production run against an uncertified or alpha runtime — which
* makes the sentence above a gate rather than a paragraph. A version constant
* on its own would be exactly the "documented but unenforced" pattern.
*
* ── THE TIER MAPPING ──────────────────────────────────────────────────────
* BI-P022's DoD names raw → staging → curated → serving. BI-P003 (merged)
* already owns RAW/NORMALIZED/CURATED/SERVING, so the dbt layers map onto it
* rather than inventing a fifth vocabulary:
*
* sources/ -> RAW (declared, never modelled)
* staging/ -> NORMALIZED (rename, cast, deduplicate; no business logic)
* intermediate/ -> CURATED (joins, business logic, conformed entities)
* marts/ -> SERVING (what a dashboard or a metric reads)
*
* A model in the wrong directory for its tier is a rejection, not a warning:
* the tier is what BI-P003's read rule gates on, and a mart that is really a
* staging model is how RAW data reaches a dashboard.
*/
import { SiaxError } from '../../platform/errors';
import type { AnalyticsTier } from '../domain/tiers';
/** dbt layers, in dependency order. */
export const DBT_LAYERS = ['sources', 'staging', 'intermediate', 'marts'] as const;
export type DbtLayer = (typeof DBT_LAYERS)[number];
export const DBT_LAYER_TIERS: Record<DbtLayer, AnalyticsTier> = {
sources: 'RAW',
staging: 'NORMALIZED',
intermediate: 'CURATED',
marts: 'SERVING',
};
/** Directories BI-P022 requires under `analytics/dbt/`. */
export const REQUIRED_DBT_DIRECTORIES = [
'models/sources',
'models/staging',
'models/intermediate',
'models/marts',
'tests',
'snapshots',
] as const;
// ── runtime certification ──────────────────────────────────────────────────
export type DbtReleaseLine = 'stable' | 'prerelease' | 'alpha';
export interface DbtRuntimePin {
/** e.g. "1.10.13". Exact, never a range. */
version: string;
line: DbtReleaseLine;
/** Adapter and its pin, e.g. { name: "dbt-clickhouse", version: "1.9.2" }. */
adapter: { name: string; version: string };
/** Has a licence + supply-chain review been completed FOR THIS PIN? */
licenseReviewed: boolean;
securityReviewed: boolean;
reviewedBy?: string;
reviewedAt?: string;
/** Required by BI-P022 to run an alpha line in production at all. */
separateCertificationRef?: string;
}
/**
* The pin this contract is written against.
*
* BOTH review flags are FALSE. No licence or supply-chain review of dbt Core
* has been performed in this repo, and none is claimed. The constant records
* an intended pin; `assertDbtRuntimeCertified` is what stops it being treated
* as an approval. Flipping these to true without doing the review is the
* failure this shape exists to make visible.
*/
export const DBT_RUNTIME_PIN: DbtRuntimePin = {
version: '1.10.13',
line: 'stable',
adapter: { name: 'dbt-clickhouse', version: '1.9.2' },
licenseReviewed: false,
securityReviewed: false,
};
export type DbtRunEnvironment = 'development' | 'ci' | 'production';
/** BI-P022's version rule, as a gate. */
export function assertDbtRuntimeCertified(pin: DbtRuntimePin, environment: DbtRunEnvironment): void {
if (!/^\d+\.\d+\.\d+$/.test(pin.version)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
'the dbt runtime pin must be an exact version',
);
}
if (environment !== 'production') return;
if (pin.line === 'alpha' && !pin.separateCertificationRef) {
throw new SiaxError(
'ENTITLEMENT_REQUIRED',
'0pt.analytics.dbt',
'BI-P022: an alpha dbt engine may not run in production without a separate certification reference',
);
}
if (!pin.licenseReviewed || !pin.securityReviewed) {
throw new SiaxError(
'ENTITLEMENT_REQUIRED',
'0pt.analytics.dbt',
`BI-P022: dbt ${pin.version} has not completed licence and security verification; ` +
`it may not run in production`,
);
}
}
// ── the job contract N0D executes ──────────────────────────────────────────
export const DBT_COMMANDS = [
'build',
'run',
'test',
'snapshot',
'seed',
'compile',
'source_freshness',
] as const;
export type DbtCommand = (typeof DBT_COMMANDS)[number];
/**
* What 0pt hands N0D. Deliberately narrow: there is no free-form `args` field,
* because a passthrough argument list is a shell surface, and a shell surface
* on a job that holds warehouse credentials is the whole problem.
*/
export interface DbtJobRequest {
/** Verified, from CL0UD. Never read from a request body. */
tenant_id: string;
project_id: string;
/** Which analytics project to run — a registered name, not a path. */
dbt_project: string;
/** Git ref of the analytics repo the models come from. Exact SHA. */
revision: string;
command: DbtCommand;
/** dbt node selectors. Validated; never concatenated into a shell string. */
select?: string[];
exclude?: string[];
/** Target layer, which N0D maps to a credential with matching write scope. */
target_layer: DbtLayer;
environment: DbtRunEnvironment;
runtime: DbtRuntimePin;
/** Idempotency key, per the standard N0D job contract. */
idempotency_key: string;
correlation_id?: string;
/** Fail the run if it would write outside this tenant's schema. */
expected_target_schema: string;
}
/** The artefacts N0D must return. dbt produces all of them; they are not optional. */
export const REQUIRED_DBT_ARTIFACTS = ['manifest.json', 'run_results.json'] as const;
export interface DbtJobResult {
tenant_id: string;
dbt_project: string;
revision: string;
command: DbtCommand;
status: 'success' | 'error' | 'partial';
started_at: string;
finished_at: string;
/** ST0RE artifact refs, one per REQUIRED_DBT_ARTIFACTS entry. */
artifacts: Record<string, string>;
/** Per-node outcomes, the input to BI-P025's DbtQualityProvider. */
nodes: DbtNodeResult[];
/** dbt's own version, as reported by the runtime that actually ran. */
observed_dbt_version: string;
}
export interface DbtNodeResult {
unique_id: string;
resource_type: 'model' | 'test' | 'snapshot' | 'seed' | 'source';
status: 'success' | 'error' | 'skipped' | 'pass' | 'fail' | 'warn';
rows_affected?: number;
execution_time_s?: number;
message?: string;
}
const SELECTOR_RE = /^[A-Za-z0-9_.:*+@ -]{1,200}$/;
const NAME_RE = /^[a-z][a-z0-9_]{1,62}$/;
const SHA_RE = /^[0-9a-f]{40}$/;
export function assertValidDbtJobRequest(req: DbtJobRequest): void {
if (!req.tenant_id?.trim()) {
throw new SiaxError('UNAUTHENTICATED', '0pt.analytics.dbt', 'a dbt job requires a verified tenant');
}
if (!NAME_RE.test(req.dbt_project)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
'dbt_project must be a registered snake_case name',
);
}
if (!SHA_RE.test(req.revision)) {
// A branch name is not reproducible, and BI-P022's DoD is about
// reproducibility. "main" today and "main" tomorrow are different projects.
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
'revision must be an exact 40-character commit SHA — a branch name is not reproducible',
);
}
if (!(DBT_COMMANDS as readonly string[]).includes(req.command)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unsupported dbt command "${req.command}"`);
}
if (!(DBT_LAYERS as readonly string[]).includes(req.target_layer)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `unknown target layer "${req.target_layer}"`);
}
if (req.target_layer === 'sources') {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
'sources are DECLARED, never built — a job cannot target the sources layer',
);
}
for (const selector of [...(req.select ?? []), ...(req.exclude ?? [])]) {
if (!SELECTOR_RE.test(selector)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
`dbt selector "${selector.slice(0, 40)}" contains characters that are not selector syntax`,
);
}
}
if (!req.idempotency_key?.trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
'the N0D job contract requires an idempotency key',
);
}
if (!req.expected_target_schema.includes(req.tenant_id)) {
// The target schema must be derivable from the tenant. A job that can name
// an arbitrary schema is a cross-tenant write waiting to happen.
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics.dbt',
"expected_target_schema must be scoped to the job's tenant",
);
}
assertDbtRuntimeCertified(req.runtime, req.environment);
}
/**
* Validate what came BACK. A result is untrusted input: it arrives from
* another repo's executor, and the estate has been burned by trusting a green
* status that meant nothing.
*/
export function assertValidDbtJobResult(result: DbtJobResult, request: DbtJobRequest): void {
if (result.tenant_id !== request.tenant_id) {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics.dbt', 'dbt result is for a different tenant');
}
if (result.revision !== request.revision) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics.dbt',
'dbt result is for a different revision than was requested — the run is not reproducible',
);
}
for (const artifact of REQUIRED_DBT_ARTIFACTS) {
if (!result.artifacts[artifact]) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dbt',
`dbt result is missing "${artifact}" — without it the run cannot be audited or its tests ingested`,
);
}
}
if (result.observed_dbt_version !== request.runtime.version) {
// A runtime that is not the pinned one silently changes what the models
// mean. Catching it here is the difference between a pin and a wish.
throw new SiaxError(
'CONFLICT',
'0pt.analytics.dbt',
`dbt ran as ${result.observed_dbt_version} but ${request.runtime.version} was pinned`,
);
}
if (result.status === 'success' && result.nodes.length === 0) {
// A "successful" run that built nothing is the dbt-shaped false green: a
// selector that matched no models exits 0.
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dbt',
'dbt reported success but executed zero nodes — a selector that matches nothing exits 0',
);
}
const failed = result.nodes.filter((n) => n.status === 'error' || n.status === 'fail');
if (result.status === 'success' && failed.length > 0) {
throw new SiaxError(
'PERMANENT_FAILURE',
'0pt.analytics.dbt',
`dbt reported success while ${failed.length} node(s) failed`,
);
}
}
/**
* The executor N0D implements. Declared so both repos compile against one
* shape; deliberately NOT implemented here.
*/
export interface DbtExecutor {
readonly runsIn: 'n0d';
execute(request: DbtJobRequest): Promise<DbtJobResult>;
}
/** Which tier a model path belongs to, or a rejection. */
export function tierForModelPath(modelPath: string): AnalyticsTier {
const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(modelPath);
if (!match) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
`model "${modelPath}" is not under one of ${DBT_LAYERS.join('/')} — BI-P003 cannot assign it a tier`,
);
}
return DBT_LAYER_TIERS[match[1] as DbtLayer];
}
/**
* A staging model must not reach past sources, a mart must not read a source
* directly, and nothing may skip a layer. Enforced on the dependency graph
* from `manifest.json`, because "we follow the convention" is not a check.
*/
const LAYER_ORDER: DbtLayer[] = ['sources', 'staging', 'intermediate', 'marts'];
export function assertLayerDependenciesLegal(edges: Array<{ from: string; to: string }>): void {
for (const edge of edges) {
const fromLayer = layerOf(edge.from);
const toLayer = layerOf(edge.to);
const fromIdx = LAYER_ORDER.indexOf(fromLayer);
const toIdx = LAYER_ORDER.indexOf(toLayer);
if (toIdx > fromIdx) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
`"${edge.from}" (${fromLayer}) depends on "${edge.to}" (${toLayer}) — a model may not read a later layer`,
);
}
if (fromIdx - toIdx > 1) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.dbt',
`"${edge.from}" (${fromLayer}) skips a layer to read "${edge.to}" (${toLayer}) — ` +
`raw data reaching a mart directly is what BI-P003's tier rule exists to prevent`,
);
}
}
}
function layerOf(path: string): DbtLayer {
const match = /(?:^|\/)models\/(sources|staging|intermediate|marts)\//.exec(path);
if (!match) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.dbt', `model "${path}" is not in a known layer`);
}
return match[1] as DbtLayer;
}