diff --git a/evidence/bi-p014/duckdb-sandbox.json b/evidence/bi-p014/duckdb-sandbox.json index e8a6078..7366789 100644 --- a/evidence/bi-p014/duckdb-sandbox.json +++ b/evidence/bi-p014/duckdb-sandbox.json @@ -1,6 +1,6 @@ { "duckdb_version": "1.5.5", - "recorded_at": "2026-09-07T01:07:03.476Z", + "recorded_at": "2026-09-07T01:15:40.775Z", "parquet": { "SE": 394.5, "NO": 105.5 diff --git a/evidence/bi-p017/analyst-sql-live-bypass.json b/evidence/bi-p017/analyst-sql-live-bypass.json index e99840f..b92328a 100644 --- a/evidence/bi-p017/analyst-sql-live-bypass.json +++ b/evidence/bi-p017/analyst-sql-live-bypass.json @@ -2,7 +2,7 @@ "prompt": "BI-P017", "claim": "Every bypass attempt, executed directly against a live ClickHouse as the tenant's restricted user WITH THE STATEMENT GUARD REMOVED, fails to return a foreign-tenant row, an ungranted-table row, or server metadata.", "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:07.196Z", + "recorded_at": "2026-09-07T01:15:43.364Z", "base_table_rows_both_tenants": 100, "tenant_view_rows_alpha": 50, "attempts_total": 64, @@ -308,7 +308,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"generaterandom\" is not permitted in an analyst session", - "engine_response_head": "Code: 158. DB::Exception: Limit for rows or bytes to read exceeded, max rows: 100.00 thousand, current rows: 157.89 thousand: While executing GenerateRandom. (T" + "engine_response_head": "Code: 158. DB::Exception: Limit for rows or bytes to read exceeded, max rows: 100.00 thousand, current rows: 315.79 thousand: While executing GenerateRandom. (T" }, { "id": "tf-merge", @@ -572,7 +572,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", - "engine_response_head": "events sales events sales ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 A" + "engine_response_head": "ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 ALPHA_ROW_11 ALPHA_ROW_12 A" }, { "id": "unterminated-comment", @@ -656,7 +656,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", - "engine_response_head": "opt_analytics events 30087fa6-7696-4216-abc7-507d8ab01e10 MergeTree 0 ['/var/lib/clickhouse/store/300/30087fa6-7696-4216-abc7-507d8ab01e10/'] /var/lib/clickhous" + "engine_response_head": "opt_analytics events 204f5e8f-9134-417f-b7f5-569500c15059 MergeTree 0 ['/var/lib/clickhouse/store/204/204f5e8f-9134-417f-b7f5-569500c15059/'] /var/lib/clickhous" }, { "id": "describe-subquery", diff --git a/evidence/bi-p019-p021/cancellation-and-budget.json b/evidence/bi-p019-p021/cancellation-and-budget.json index e8324a6..809350f 100644 --- a/evidence/bi-p019-p021/cancellation-and-budget.json +++ b/evidence/bi-p019-p021/cancellation-and-budget.json @@ -1,20 +1,20 @@ { "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:02.373Z", + "recorded_at": "2026-09-07T01:15:40.234Z", "counterfactual_still_running_after_2s": true, "cancellation": { - "rows_read_before_cancel": 4955255022, - "elapsed_ms_to_verified_stop": 110, + "rows_read_before_cancel": 4227318261, + "elapsed_ms_to_verified_stop": 162, "verified_stopped": true, "clickhouse_query_log_type": "ExceptionWhileProcessing", "clickhouse_exception_code": 394 }, "budget_in_flight": { "budget_max_rows_scanned": 1000000, - "observed_rows_read_at_cancel": 215980518, - "observed_elapsed_ms_at_cancel": 60, + "observed_rows_read_at_cancel": 153318696, + "observed_elapsed_ms_at_cancel": 55, "polls_before_breach": 1, - "overshoot_factor": 216, + "overshoot_factor": 153, "overshoot_note": "in-flight enforcement is granular to the poll interval; the query overshot the window by this factor before the poll that caught it. A bound, not a hard cap.", "cancellation_verified": true, "clickhouse_exception_code": 394, diff --git a/evidence/bi-p025/native-quality-live.json b/evidence/bi-p025/native-quality-live.json index 67a17a3..c2d296b 100644 --- a/evidence/bi-p025/native-quality-live.json +++ b/evidence/bi-p025/native-quality-live.json @@ -1,6 +1,6 @@ { "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:17.084Z", + "recorded_at": "2026-09-07T01:15:45.657Z", "not_null": { "evaluated": 118, "violations": 5 diff --git a/src/analytics/arrow/flatbuffer-builder.ts b/src/analytics/arrow/flatbuffer-builder.ts index 46d3695..3185803 100644 --- a/src/analytics/arrow/flatbuffer-builder.ts +++ b/src/analytics/arrow/flatbuffer-builder.ts @@ -148,14 +148,14 @@ export class FlatBufferBuilder { } startObject(numFields: number): void { - if (this.inObject) throw new Error('nested startObject'); + if (this.inObject) throw new Error("nested startObject"); this.vtable = new Array(numFields).fill(0); this.objectStart = this.used; this.inObject = true; } endObject(): Offset { - if (!this.inObject) throw new Error('endObject outside an object'); + if (!this.inObject) throw new Error("endObject outside an object"); this.prep(SIZEOF_INT); this.writeInt32(0); // soffset placeholder, patched below const objectOffset = this.used; diff --git a/src/analytics/arrow/ipc.ts b/src/analytics/arrow/ipc.ts index 2288a7f..d4a693c 100644 --- a/src/analytics/arrow/ipc.ts +++ b/src/analytics/arrow/ipc.ts @@ -29,8 +29,8 @@ * padded so the body starts 8-byte aligned, then the body buffers. * Terminated by 0xFFFFFFFF 0x00000000. */ -import { FlatBufferBuilder, type Offset } from './flatbuffer-builder'; -import { SiaxError } from '../../platform/errors'; +import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder"; +import { SiaxError } from "../../platform/errors"; // ── FlatBuffer enum values from org.apache.arrow.flatbuf ──────────────────── @@ -51,13 +51,13 @@ const PRECISION_DOUBLE = 2; // every 64-bit constant is built with BigInt(...) instead. Behaviour is // identical — this is a compiler-target accommodation, not a semantic choice. const BIG_ZERO = BigInt(0); -const INT64_MAX = BigInt('9223372036854775807'); -const INT64_MIN = BigInt('-9223372036854775808'); +const INT64_MAX = BigInt("9223372036854775807"); +const INT64_MIN = BigInt("-9223372036854775808"); const TIME_UNIT_MILLISECOND = 1; // ── public column model ──────────────────────────────────────────────────── -export type ArrowColumnType = 'int64' | 'float64' | 'utf8' | 'bool' | 'timestamp_ms'; +export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms"; export interface ArrowColumn { name: string; @@ -78,39 +78,39 @@ export const ARROW_TYPE_SUPPORT: { note: string; }[] = [ { - clickhouse: 'Int64 / UInt32 / Int32', - arrow: 'int64', - duckdb: 'BIGINT', - python: 'int', - note: 'exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped', + clickhouse: "Int64 / UInt32 / Int32", + arrow: "int64", + duckdb: "BIGINT", + python: "int", + note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped", }, { - clickhouse: 'Float64', - arrow: 'float64', - duckdb: 'DOUBLE', - python: 'float', - note: 'exact IEEE-754 round trip', + clickhouse: "Float64", + arrow: "float64", + duckdb: "DOUBLE", + python: "float", + note: "exact IEEE-754 round trip", }, { - clickhouse: 'String / LowCardinality(String)', - arrow: 'utf8', - duckdb: 'VARCHAR', - python: 'str', - note: 'LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not', + clickhouse: "String / LowCardinality(String)", + arrow: "utf8", + duckdb: "VARCHAR", + python: "str", + note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not", }, { - clickhouse: 'Bool / UInt8(0|1)', - arrow: 'bool', - duckdb: 'BOOLEAN', - python: 'bool', - note: 'bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64', + clickhouse: "Bool / UInt8(0|1)", + arrow: "bool", + duckdb: "BOOLEAN", + python: "bool", + note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64", }, { clickhouse: "DateTime64(3,'UTC')", - arrow: 'timestamp_ms', - duckdb: 'TIMESTAMP', - python: 'datetime', - note: 'millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set', + arrow: "timestamp_ms", + duckdb: "TIMESTAMP", + python: "datetime", + note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set", }, ]; @@ -118,7 +118,7 @@ export const ARROW_TYPE_SUPPORT: { /** ClickHouse output format producing an Arrow IPC *stream*. */ export function clickhouseArrowStreamFormat(): string { - return 'ArrowStream'; + return "ArrowStream"; } /** @@ -129,8 +129,8 @@ export function clickhouseArrowStreamFormat(): string { */ export function arrowStreamSettings(): Record { return { - output_format_arrow_string_as_string: '1', - output_format_arrow_low_cardinality_as_dictionary: '0', + output_format_arrow_string_as_string: "1", + output_format_arrow_low_cardinality_as_dictionary: "0", }; } @@ -165,24 +165,20 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: const n = col.values.length; switch (col.type) { - case 'int64': { + case "int64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; - const b = typeof v === 'bigint' ? v : BigInt(Math.trunc(Number(v))); + const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v))); if (b > INT64_MAX || b < INT64_MIN) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `value out of int64 range in column "${col.name}"`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`); } dv.setBigInt64(i * 8, b, true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'float64': { + case "float64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { @@ -191,17 +187,17 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'timestamp_ms': { + case "timestamp_ms": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; - const ms = v instanceof Date ? v.getTime() : typeof v === 'bigint' ? Number(v) : Number(v); + const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v); dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'bool': { + case "bool": { const byteLen = Math.ceil(n / 8); const data = new Uint8Array(alignTo8(byteLen)); col.values.forEach((v, i) => { @@ -210,7 +206,7 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: }); return { buffers: [validity, { bytes: data, length: byteLen }], nullCount }; } - case 'utf8': { + case "utf8": { const encoded = col.values.map((v) => v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)), ); @@ -232,36 +228,36 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: } default: { const never: never = col.type; - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } { switch (type) { - case 'int64': { + case "int64": { b.startObject(2); b.addInt32(0, 64, 0); // bitWidth b.addInt8(1, 1, 0); // is_signed return { typeType: TYPE_INT, offset: b.endObject() }; } - case 'float64': { + case "float64": { b.startObject(1); b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() }; } - case 'utf8': { + case "utf8": { b.startObject(0); return { typeType: TYPE_UTF8, offset: b.endObject() }; } - case 'bool': { + case "bool": { b.startObject(0); return { typeType: TYPE_BOOL, offset: b.endObject() }; } - case 'timestamp_ms': { + case "timestamp_ms": { // Build the timezone string BEFORE startObject — FlatBuffers forbids // creating a nested object while a table is open. - const tz = b.createString('UTC'); + const tz = b.createString("UTC"); b.startObject(2); b.addInt16(0, TIME_UNIT_MILLISECOND, 0); b.addOffset(1, tz, 0); @@ -269,7 +265,7 @@ function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType } default: { const never: never = type; - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } @@ -365,14 +361,14 @@ function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array { */ export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array { if (columns.length === 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'an Arrow stream needs at least one column'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column"); } const rowCount = columns[0].values.length; for (const c of columns) { if (c.values.length !== rowCount) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `column "${c.name}" has ${c.values.length} values, expected ${rowCount}`, ); } @@ -500,11 +496,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { while (pos + 8 <= bytes.byteLength) { const continuation = dv.getUint32(pos, true); if (continuation !== 0xffffffff) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'not an Arrow IPC stream (missing continuation marker)', - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)"); } const metaLen = dv.getInt32(pos + 4, true); if (metaLen === 0) break; // end of stream @@ -521,7 +513,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { fields = []; for (let i = 0; i < n; i++) { const f = schema.vectorElementTable(1, i); - fields.push({ name: f.readString(0) ?? '', typeType: f.readInt8(2, 0) }); + fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) }); } } } else if (headerType === MESSAGE_HEADER_RECORD_BATCH) { @@ -546,8 +538,8 @@ export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: const info = decodeArrowStream(bytes); if (info.rowCount > maxResultRows) { throw new SiaxError( - 'QUOTA_EXCEEDED', - '0pt.analytics', + "QUOTA_EXCEEDED", + "0pt.analytics", `Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`, ); } diff --git a/src/analytics/clickhouse-analytics.ts b/src/analytics/clickhouse-analytics.ts index 26fb574..c4fafc2 100644 --- a/src/analytics/clickhouse-analytics.ts +++ b/src/analytics/clickhouse-analytics.ts @@ -25,7 +25,7 @@ * * Retention: tables are created WITH TTL (event_time + INTERVAL n DAY). */ -import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../platform/errors'; +import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors"; export interface ClickHouseAnalyticsOptions { baseUrl: string; @@ -52,7 +52,7 @@ export interface MetricQuery { tenant_id: string; dataset_id?: string; metric: string; - aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count'; + aggregate: "sum" | "avg" | "min" | "max" | "count"; /** Group by one dimension key (validated identifier). */ group_by?: string; from_time?: string; @@ -77,9 +77,9 @@ export class ClickHouseAnalytics { private readonly maxResultRows: number; constructor(opts: ClickHouseAnalyticsOptions) { - this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); - this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined; - this.database = opts.database ?? 'opt_analytics'; + this.baseUrl = opts.baseUrl.replace(/\/+$/, ""); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined; + this.database = opts.database ?? "opt_analytics"; this.timeoutMs = opts.timeoutMs ?? 30_000; this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10; this.maxResultRows = opts.maxResultRows ?? 10_000; @@ -100,7 +100,7 @@ export class ClickHouseAnalytics { ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (tenant_id, metric, dataset_id, event_time) - TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`, + TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY` ); } @@ -109,14 +109,10 @@ export class ClickHouseAnalytics { if (events.length === 0) return { ingested: 0 }; for (const e of events) { if (!e.tenant_id?.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every analytics event needs tenant_id'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id"); } if (!Number.isFinite(e.value)) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `metric value for "${e.metric}" must be finite`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`); } } const rows = events @@ -126,11 +122,11 @@ export class ClickHouseAnalytics { dataset_id: e.dataset_id, metric: e.metric, value: e.value, - event_time: (e.event_time ?? new Date().toISOString()).replace('T', ' ').replace(/Z$/, ''), + event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""), dimensions: e.dimensions ?? {}, - }), + }) ) - .join('\n'); + .join("\n"); await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows); return { ingested: events.length }; } @@ -138,40 +134,41 @@ export class ClickHouseAnalytics { /** Closed metric-query DSL — compiled with a FORCED tenant predicate. */ async queryMetric(q: MetricQuery): Promise { if (!q.tenant_id?.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every metric query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query"); } if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid group_by identifier`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`); } - const agg = q.aggregate === 'count' ? 'count()' : `${q.aggregate}(value)`; + const agg = + q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`; const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined; const limit = Math.min(q.limit ?? 1000, this.maxResultRows); // Parameterized query — values travel as HTTP params, never spliced. - const conditions = ['tenant_id = {tenant:String}', 'metric = {metric:String}']; + const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"]; const params: Record = { param_tenant: q.tenant_id, param_metric: q.metric }; if (q.dataset_id) { - conditions.push('dataset_id = {dataset:String}'); + conditions.push("dataset_id = {dataset:String}"); params.param_dataset = q.dataset_id; } if (q.from_time) { - conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})'); + conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})"); params.param_fromTime = q.from_time; } if (q.to_time) { - conditions.push('event_time < parseDateTime64BestEffort({toTime:String})'); + conditions.push("event_time < parseDateTime64BestEffort({toTime:String})"); params.param_toTime = q.to_time; } const sql = - `SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ''}${agg} AS value ` + + `SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` + `FROM ${this.database}.opt_metric_events ` + - `WHERE ${conditions.join(' AND ')} ` + - (groupExpr ? `GROUP BY grp ORDER BY value DESC ` : '') + + `WHERE ${conditions.join(" AND ")} ` + + (groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") + `LIMIT ${limit} FORMAT JSON`; const json = (await this.exec(sql, undefined, { - readonly: '1', + readonly: "1", max_execution_time: String(this.maxExecutionTimeSec), max_result_rows: String(this.maxResultRows), ...params, @@ -185,7 +182,7 @@ export class ClickHouseAnalytics { async health(): Promise { try { - await this.exec('SELECT 1 FORMAT JSON'); + await this.exec("SELECT 1 FORMAT JSON"); return true; } catch { return false; @@ -193,24 +190,28 @@ export class ClickHouseAnalytics { } // ----------------------------------------------------------------------- - private async exec(sql: string, body?: string, settings?: Record): Promise { + private async exec( + sql: string, + body?: string, + settings?: Record + ): Promise { const url = new URL(this.baseUrl); - url.searchParams.set('query', sql); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v); const headers: Record = {}; - if (this.authHeader) headers['Authorization'] = this.authHeader; + if (this.authHeader) headers["Authorization"] = this.authHeader; let res: Response; try { res = await fetch(url.toString(), { - method: 'POST', + method: "POST", headers, - body: body ?? '', + body: body ?? "", signal: AbortSignal.timeout(this.timeoutMs), }); } catch (cause) { - throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause }); } const text = await res.text(); if (!res.ok) { @@ -218,8 +219,8 @@ export class ClickHouseAnalytics { const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status); throw new SiaxError( siaxErrorCodeForUpstreamStatus(res.status), - 'clickhouse', - `query failed (CH code ${code})`, + "clickhouse", + `query failed (CH code ${code})` ); } if (!text) return undefined; diff --git a/src/analytics/domain/contract.ts b/src/analytics/domain/contract.ts index 7453c31..0a3ba10 100644 --- a/src/analytics/domain/contract.ts +++ b/src/analytics/domain/contract.ts @@ -18,10 +18,10 @@ * shipped under a minor bump is exactly the failure this gate exists to * catch, so it is reported as a violation rather than tolerated. */ -import crypto from 'node:crypto'; -import { SiaxError } from '../../platform/errors'; -import type { AnalyticsFieldType, FieldRole, PiiClassification } from './entities'; -import type { AnalyticsTier } from './tiers'; +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities"; +import type { AnalyticsTier } from "./tiers"; export interface ContractField { name: string; @@ -38,7 +38,7 @@ export interface ContractQualityRule { rule_id: string; kind: string; field_name?: string; - severity: 'warn' | 'block'; + severity: "warn" | "block"; } /** BI-P004's named contents, one field each. */ @@ -77,18 +77,14 @@ const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/; export function parseSemVer(v: string): SemVer { const m = SEMVER_RE.exec(v); if (!m) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `contract version "${v}" is not major.minor.patch`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`); } return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) }; } // ── change classification ────────────────────────────────────────────────── -export type ChangeSeverity = 'compatible' | 'breaking'; +export type ChangeSeverity = "compatible" | "breaking"; export interface ContractChange { severity: ChangeSeverity; @@ -104,7 +100,7 @@ export interface ContractChange { * Everything absent from this table is breaking. */ const SAFE_TYPE_WIDENING: Partial> = { - date: ['timestamp'], + date: ["timestamp"], }; function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean { @@ -136,9 +132,9 @@ function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] { if (previous.dataset_id !== next.dataset_id) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'contract compatibility is only defined between versions of the SAME dataset', + "INVALID_REQUEST", + "0pt.analytics", + "contract compatibility is only defined between versions of the SAME dataset", ); } const changes: ContractChange[] = []; @@ -148,49 +144,44 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): for (const [name, pf] of prevFields) { const nf = nextFields.get(name); if (!nf) { - changes.push({ - severity: 'breaking', - code: 'field_removed', - field: name, - detail: `field "${name}" removed`, - }); + changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` }); continue; } if (!typeChangeIsCompatible(pf.type, nf.type)) { changes.push({ - severity: 'breaking', - code: 'field_type_changed', + severity: "breaking", + code: "field_type_changed", field: name, detail: `field "${name}" type ${pf.type} -> ${nf.type}`, }); } else if (pf.type !== nf.type) { changes.push({ - severity: 'compatible', - code: 'field_type_widened', + severity: "compatible", + code: "field_type_widened", field: name, detail: `field "${name}" widened ${pf.type} -> ${nf.type}`, }); } if (!pf.nullable && nf.nullable) { changes.push({ - severity: 'breaking', - code: 'field_became_nullable', + severity: "breaking", + code: "field_became_nullable", field: name, detail: `field "${name}" became nullable`, }); } if (pf.nullable && !nf.nullable) { changes.push({ - severity: 'compatible', - code: 'field_became_required', + severity: "compatible", + code: "field_became_required", field: name, detail: `field "${name}" is no longer nullable`, }); } if (pf.role !== nf.role) { changes.push({ - severity: 'breaking', - code: 'field_role_changed', + severity: "breaking", + code: "field_role_changed", field: name, detail: `field "${name}" role ${pf.role} -> ${nf.role}`, }); @@ -201,8 +192,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): if (prevFields.has(name)) continue; if (nf.nullable) { changes.push({ - severity: 'compatible', - code: 'nullable_field_added', + severity: "compatible", + code: "nullable_field_added", field: name, detail: `nullable field "${name}" added`, }); @@ -210,8 +201,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): // A required new field breaks producers, and breaks readers that // reconstruct rows (they now have no value to supply). changes.push({ - severity: 'breaking', - code: 'required_field_added', + severity: "breaking", + code: "required_field_added", field: name, detail: `required (non-nullable) field "${name}" added`, }); @@ -223,75 +214,51 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v); if (!idsEqual) { changes.push({ - severity: 'breaking', - code: 'primary_identifier_changed', - detail: `primary identifier [${previous.primary_identifiers.join(',')}] -> [${next.primary_identifiers.join(',')}]`, + severity: "breaking", + code: "primary_identifier_changed", + detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`, }); } for (const d of previous.dimensions) { if (!next.dimensions.includes(d)) { - changes.push({ - severity: 'breaking', - code: 'dimension_removed', - field: d, - detail: `dimension "${d}" removed`, - }); + changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` }); } } for (const d of next.dimensions) { if (!previous.dimensions.includes(d)) { - changes.push({ - severity: 'compatible', - code: 'dimension_added', - field: d, - detail: `dimension "${d}" added`, - }); + changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` }); } } for (const m of previous.measures) { if (!next.measures.includes(m)) { - changes.push({ - severity: 'breaking', - code: 'measure_removed', - field: m, - detail: `measure "${m}" removed`, - }); + changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` }); } } for (const m of next.measures) { if (!previous.measures.includes(m)) { - changes.push({ - severity: 'compatible', - code: 'measure_added', - field: m, - detail: `measure "${m}" added`, - }); + changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` }); } } if (next.freshness_slo_seconds > previous.freshness_slo_seconds) { changes.push({ - severity: 'breaking', - code: 'freshness_slo_loosened', + severity: "breaking", + code: "freshness_slo_loosened", detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`, }); } if (next.retention_days < previous.retention_days) { changes.push({ - severity: 'breaking', - code: 'retention_shortened', + severity: "breaking", + code: "retention_shortened", detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`, }); } for (const p of previous.allowed_purposes) { if (!next.allowed_purposes.includes(p)) { - changes.push({ - severity: 'breaking', - code: 'purpose_removed', - detail: `allowed purpose "${p}" removed`, - }); + changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` }); } } @@ -303,8 +270,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): }; if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) { changes.push({ - severity: 'breaking', - code: 'pii_classification_raised', + severity: "breaking", + code: "pii_classification_raised", detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`, }); } @@ -312,16 +279,16 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r])); for (const [id, pr] of prevRules) { const nr = next.quality_rules.find((r) => r.rule_id === id); - if (!nr && pr.severity === 'block') { + if (!nr && pr.severity === "block") { changes.push({ - severity: 'breaking', - code: 'blocking_quality_rule_removed', + severity: "breaking", + code: "blocking_quality_rule_removed", detail: `blocking quality rule "${id}" removed`, }); - } else if (nr && pr.severity === 'block' && nr.severity === 'warn') { + } else if (nr && pr.severity === "block" && nr.severity === "warn") { changes.push({ - severity: 'breaking', - code: 'blocking_quality_rule_downgraded', + severity: "breaking", + code: "blocking_quality_rule_downgraded", detail: `quality rule "${id}" downgraded block -> warn`, }); } @@ -329,8 +296,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): if (previous.tier !== next.tier) { changes.push({ - severity: 'breaking', - code: 'tier_changed', + severity: "breaking", + code: "tier_changed", detail: `tier ${previous.tier} -> ${next.tier}`, }); } @@ -367,7 +334,7 @@ export function checkContractCompatibility( const pv = parseSemVer(previous.version); const nv = parseSemVer(next.version); const changes = diffContracts(previous, next); - const breaking = changes.filter((c) => c.severity === 'breaking'); + const breaking = changes.filter((c) => c.severity === "breaking"); const forward = nv.major > pv.major || @@ -397,7 +364,7 @@ export function checkContractCompatibility( reason: `${breaking.length} breaking change(s) require a MAJOR version bump ` + `(${previous.version} -> ${next.version} is not one): ` + - breaking.map((b) => b.code).join(', '), + breaking.map((b) => b.code).join(", "), }; } @@ -412,17 +379,10 @@ export function checkContractCompatibility( } /** Throwing form for call sites that treat incompatibility as a failure. */ -export function assertContractCompatible( - previous: DatasetContract, - next: DatasetContract, -): CompatibilityVerdict { +export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict { const verdict = checkContractCompatibility(previous, next); if (!verdict.compatible) { - throw new SiaxError( - 'CONFLICT', - '0pt.analytics', - verdict.reason ?? 'dataset contract compatibility gate failed', - ); + throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed"); } return verdict; } @@ -435,28 +395,24 @@ export function assertContractCompatible( export function validateContract(contract: DatasetContract): void { parseSemVer(contract.version); if (!contract.dataset_id.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'contract needs a dataset_id'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id"); } if (contract.fields.length === 0) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `contract ${contract.dataset_id} declares no fields`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`); } const names = new Set(); for (const f of contract.fields) { if (names.has(f.name)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `duplicate field "${f.name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`); } names.add(f.name); } - for (const label of ['primary_identifiers', 'dimensions', 'measures'] as const) { + for (const label of ["primary_identifiers", "dimensions", "measures"] as const) { for (const n of contract[label]) { if (!names.has(n)) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `${label} references unknown field "${n}" in ${contract.dataset_id}`, ); } @@ -464,23 +420,23 @@ export function validateContract(contract: DatasetContract): void { } if (contract.primary_identifiers.length === 0) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `contract ${contract.dataset_id} declares no primary identifier`, ); } if (contract.allowed_purposes.length === 0) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`, ); } if (contract.freshness_slo_seconds <= 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'freshness_slo_seconds must be positive'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive"); } if (contract.retention_days < 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'retention_days must not be negative'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative"); } } @@ -501,7 +457,7 @@ export function contractHash(contract: DatasetContract): string { nullable: f.nullable, role: f.role, primary_identifier: f.primary_identifier ?? false, - classification: f.classification ?? 'none', + classification: f.classification ?? "none", })) .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)), primary_identifiers: contract.primary_identifiers, @@ -514,5 +470,5 @@ export function contractHash(contract: DatasetContract): string { quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)), allowed_purposes: [...contract.allowed_purposes].sort(), }; - return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); + return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); } diff --git a/src/analytics/domain/entities.ts b/src/analytics/domain/entities.ts index b96faba..29d0017 100644 --- a/src/analytics/domain/entities.ts +++ b/src/analytics/domain/entities.ts @@ -35,7 +35,7 @@ * defines its own copies of these tables and never receives engine * credentials. */ -import type { AnalyticsTier } from './tiers'; +import type { AnalyticsTier } from "./tiers"; /** Every tenant-owned analytics record. */ export interface TenantScoped { @@ -46,28 +46,28 @@ export interface TenantScoped { // ── field/typing vocabulary ──────────────────────────────────────────────── export const ANALYTICS_FIELD_TYPES = [ - 'string', - 'int64', - 'float64', - 'bool', - 'date', - 'timestamp', - 'decimal', - 'json', + "string", + "int64", + "float64", + "bool", + "date", + "timestamp", + "decimal", + "json", ] as const; export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number]; /** A field's analytical role. Measures aggregate; dimensions group. */ -export const FIELD_ROLES = ['identifier', 'dimension', 'measure', 'attribute'] as const; +export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const; export type FieldRole = (typeof FIELD_ROLES)[number]; /** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */ -export const PII_CLASSIFICATIONS = ['none', 'pseudonymous', 'personal', 'sensitive'] as const; +export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const; export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number]; // ── 1. AnalyticsDataset ──────────────────────────────────────────────────── -export const DATASET_STATUSES = ['draft', 'active', 'deprecated', 'retired'] as const; +export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const; export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number]; /** @@ -137,7 +137,7 @@ export interface DatasetField extends TenantScoped { // ── 4. DatasetGrant ──────────────────────────────────────────────────────── -export const GRANT_LEVELS = ['read', 'query', 'export', 'admin'] as const; +export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const; export type GrantLevel = (typeof GRANT_LEVELS)[number]; /** @@ -150,7 +150,7 @@ export interface DatasetGrant extends TenantScoped { dataset_id: string; /** Subject: a CL0UD principal, role, or BI application binding id. */ subject_id: string; - subject_type: 'user' | 'role' | 'service' | 'bi_application'; + subject_type: "user" | "role" | "service" | "bi_application"; level: GrantLevel; allowed_purposes: string[]; expires_at?: string; @@ -161,14 +161,14 @@ export interface DatasetGrant extends TenantScoped { // ── 5. MetricDefinition ──────────────────────────────────────────────────── export const METRIC_AGGREGATIONS = [ - 'sum', - 'avg', - 'min', - 'max', - 'count', - 'count_distinct', - 'ratio', - 'derived', + "sum", + "avg", + "min", + "max", + "count", + "count_distinct", + "ratio", + "derived", ] as const; export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number]; @@ -244,7 +244,7 @@ export interface SemanticModel extends TenantScoped { // ── 9. AnalyticsQuery ────────────────────────────────────────────────────── -export const QUERY_SURFACES = ['metric', 'sql', 'projection'] as const; +export const QUERY_SURFACES = ["metric", "sql", "projection"] as const; export type QuerySurface = (typeof QUERY_SURFACES)[number]; /** @@ -268,12 +268,12 @@ export interface AnalyticsQuery extends TenantScoped { // ── 10. QueryExecution ───────────────────────────────────────────────────── export const EXECUTION_STATES = [ - 'queued', - 'running', - 'succeeded', - 'failed', - 'cancelled', - 'rejected', + "queued", + "running", + "succeeded", + "failed", + "cancelled", + "rejected", ] as const; export type ExecutionState = (typeof EXECUTION_STATES)[number]; @@ -299,7 +299,7 @@ export interface QueryExecution extends TenantScoped { // ── 11. QueryResult ──────────────────────────────────────────────────────── -export const RESULT_ENCODINGS = ['json', 'arrow_ipc'] as const; +export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const; export type ResultEncoding = (typeof RESULT_ENCODINGS)[number]; export interface QueryResult extends TenantScoped { @@ -345,7 +345,7 @@ export interface DashboardVersion extends TenantScoped { // ── 14. ReportDefinition ─────────────────────────────────────────────────── -export const REPORT_FORMATS = ['pdf', 'csv', 'xlsx', 'parquet', 'arrow_ipc'] as const; +export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const; export type ReportFormat = (typeof REPORT_FORMATS)[number]; export interface ReportDefinition extends TenantScoped { @@ -378,18 +378,18 @@ export interface ReportRun extends TenantScoped { // ── 16. DataQualityRule ──────────────────────────────────────────────────── export const QUALITY_RULE_KINDS = [ - 'not_null', - 'unique', - 'accepted_values', - 'range', - 'freshness', - 'row_count', - 'referential', - 'custom', + "not_null", + "unique", + "accepted_values", + "range", + "freshness", + "row_count", + "referential", + "custom", ] as const; export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number]; -export const QUALITY_SEVERITIES = ['warn', 'block'] as const; +export const QUALITY_SEVERITIES = ["warn", "block"] as const; export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number]; export interface DataQualityRule extends TenantScoped { @@ -423,7 +423,7 @@ export interface DataQualityRun extends TenantScoped { // ── 18. LineageReference ─────────────────────────────────────────────────── -export const LINEAGE_DIRECTIONS = ['input', 'output'] as const; +export const LINEAGE_DIRECTIONS = ["input", "output"] as const; export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number]; /** @@ -438,7 +438,7 @@ export interface LineageReference extends TenantScoped { job_name: string; direction: LineageDirection; /** The analytics entity this edge touches. */ - entity_type: 'dataset' | 'metric' | 'dashboard' | 'report'; + entity_type: "dataset" | "metric" | "dashboard" | "report"; entity_id: string; /** Namespaced dataset name on the other side of the edge. */ counterpart_ref: string; @@ -447,7 +447,7 @@ export interface LineageReference extends TenantScoped { // ── 19. BIApplicationBinding ─────────────────────────────────────────────── -export const BI_APPLICATION_KINDS = ['superset', 'evidence', 'des1gn', 'notebook', 'custom'] as const; +export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const; export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number]; /** @@ -482,55 +482,55 @@ export interface BIApplicationBinding extends TenantScoped { * or omission is a test failure rather than drift. */ export const ANALYTICS_DOMAIN_ENTITIES = [ - 'AnalyticsDataset', - 'DatasetVersion', - 'DatasetField', - 'DatasetGrant', - 'MetricDefinition', - 'MetricVersion', - 'DimensionDefinition', - 'SemanticModel', - 'AnalyticsQuery', - 'QueryExecution', - 'QueryResult', - 'DashboardDefinition', - 'DashboardVersion', - 'ReportDefinition', - 'ReportRun', - 'DataQualityRule', - 'DataQualityRun', - 'LineageReference', - 'BIApplicationBinding', + "AnalyticsDataset", + "DatasetVersion", + "DatasetField", + "DatasetGrant", + "MetricDefinition", + "MetricVersion", + "DimensionDefinition", + "SemanticModel", + "AnalyticsQuery", + "QueryExecution", + "QueryResult", + "DashboardDefinition", + "DashboardVersion", + "ReportDefinition", + "ReportRun", + "DataQualityRule", + "DataQualityRun", + "LineageReference", + "BIApplicationBinding", ] as const; export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number]; /** Entity -> physical table in sql/008_analytics_domain_model.sql. */ export const ANALYTICS_ENTITY_TABLES: Record = { - AnalyticsDataset: 'opt_analytics_dataset', - DatasetVersion: 'opt_analytics_dataset_version', - DatasetField: 'opt_analytics_dataset_field', - DatasetGrant: 'opt_analytics_dataset_grant', - MetricDefinition: 'opt_analytics_metric_definition', - MetricVersion: 'opt_analytics_metric_version', - DimensionDefinition: 'opt_analytics_dimension_definition', - SemanticModel: 'opt_analytics_semantic_model', - AnalyticsQuery: 'opt_analytics_query', - QueryExecution: 'opt_analytics_query_execution', - QueryResult: 'opt_analytics_query_result', - DashboardDefinition: 'opt_analytics_dashboard_definition', - DashboardVersion: 'opt_analytics_dashboard_version', - ReportDefinition: 'opt_analytics_report_definition', - ReportRun: 'opt_analytics_report_run', - DataQualityRule: 'opt_analytics_data_quality_rule', - DataQualityRun: 'opt_analytics_data_quality_run', - LineageReference: 'opt_analytics_lineage_reference', - BIApplicationBinding: 'opt_analytics_bi_application_binding', + AnalyticsDataset: "opt_analytics_dataset", + DatasetVersion: "opt_analytics_dataset_version", + DatasetField: "opt_analytics_dataset_field", + DatasetGrant: "opt_analytics_dataset_grant", + MetricDefinition: "opt_analytics_metric_definition", + MetricVersion: "opt_analytics_metric_version", + DimensionDefinition: "opt_analytics_dimension_definition", + SemanticModel: "opt_analytics_semantic_model", + AnalyticsQuery: "opt_analytics_query", + QueryExecution: "opt_analytics_query_execution", + QueryResult: "opt_analytics_query_result", + DashboardDefinition: "opt_analytics_dashboard_definition", + DashboardVersion: "opt_analytics_dashboard_version", + ReportDefinition: "opt_analytics_report_definition", + ReportRun: "opt_analytics_report_run", + DataQualityRule: "opt_analytics_data_quality_rule", + DataQualityRun: "opt_analytics_data_quality_run", + LineageReference: "opt_analytics_lineage_reference", + BIApplicationBinding: "opt_analytics_bi_application_binding", }; /** The four entities BI-P002 requires to be immutable once written. */ export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [ - 'DatasetVersion', - 'MetricVersion', - 'SemanticModel', - 'DashboardVersion', + "DatasetVersion", + "MetricVersion", + "SemanticModel", + "DashboardVersion", ]; diff --git a/src/analytics/domain/tiers.ts b/src/analytics/domain/tiers.ts index bc0b4a2..d67e590 100644 --- a/src/analytics/domain/tiers.ts +++ b/src/analytics/domain/tiers.ts @@ -21,24 +21,24 @@ * ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product * repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers. */ -import { SiaxError } from '../../platform/errors'; +import { SiaxError } from "../../platform/errors"; -export const ANALYTICS_TIERS = ['RAW', 'NORMALIZED', 'CURATED', 'SERVING'] as const; +export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const; export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number]; /** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */ export const TIER_PHYSICAL_HOME: Record = { - RAW: 'st0re:iceberg|parquet', - NORMALIZED: 'st0re:iceberg', - CURATED: 'st0re:iceberg', - SERVING: 'clickhouse:projection', + RAW: "st0re:iceberg|parquet", + NORMALIZED: "st0re:iceberg", + CURATED: "st0re:iceberg", + SERVING: "clickhouse:projection", }; /** Tiers a BI presentation surface may read without an explicit exception. */ -export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ['SERVING', 'CURATED']; +export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"]; export function isAnalyticsTier(v: unknown): v is AnalyticsTier { - return typeof v === 'string' && (ANALYTICS_TIERS as readonly string[]).includes(v); + return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v); } /** @@ -64,8 +64,8 @@ export function assertReadableByBiSurface( if (BI_READABLE_TIERS.includes(dataset.tier)) return; if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return; throw new SiaxError( - 'AUTHORIZATION_DENIED', - '0pt.analytics', + "AUTHORIZATION_DENIED", + "0pt.analytics", `BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` + `without a recorded raw_access_exception (BI-P003)`, ); @@ -75,8 +75,8 @@ export function assertReadableByBiSurface( export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void { if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`, ); } diff --git a/src/analytics/engine/clickhouse-engine.ts b/src/analytics/engine/clickhouse-engine.ts index e2e27d0..7753f24 100644 --- a/src/analytics/engine/clickhouse-engine.ts +++ b/src/analytics/engine/clickhouse-engine.ts @@ -26,17 +26,17 @@ * LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts * ClickHouse pipelines, never quarantined market data. */ -import crypto from 'node:crypto'; -import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../../platform/errors'; -import { ClickHouseAnalytics } from '../clickhouse-analytics'; +import crypto from "node:crypto"; +import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors"; +import { ClickHouseAnalytics } from "../clickhouse-analytics"; import { compileMetricQuery, compileProjectionQuery, dimensionCatalogFrom, FORCED_TENANT_PREDICATE, type DimensionCatalog, -} from './query-compiler'; -import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from './schema'; +} from "./query-compiler"; +import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema"; import { applyQuota, clickhouseSettings, @@ -44,7 +44,7 @@ import { DEFAULT_BUDGETS, planningSettings, type ResourceBudget, -} from './resource-governance'; +} from "./resource-governance"; import type { AnalystQueryRequest, AnalyticsIdentity, @@ -56,7 +56,7 @@ import type { QueryEstimate, QueryExplanation, SiaxAnalyticsEngine, -} from './siax-analytics-engine'; +} from "./siax-analytics-engine"; export interface ClickHouseEngineOptions { baseUrl: string; @@ -66,13 +66,13 @@ export interface ClickHouseEngineOptions { timeoutMs?: number; /** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */ dimensionCatalogs?: Record>; - schema?: Omit; + schema?: Omit; } const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined }; export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { - readonly provider = 'clickhouse'; + readonly provider = "clickhouse"; /** PRESERVED pre-existing implementation, composed not replaced. */ private readonly legacy: ClickHouseAnalytics; @@ -81,15 +81,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { private readonly database: string; private readonly timeoutMs: number; private readonly dimensionCatalogs: Record>; - private readonly schemaOpts: Omit; + private readonly schemaOpts: Omit; private readonly gate: ConcurrencyGate; /** query_id -> AbortController, so cancelQuery can stop the client side too. */ private readonly inFlight = new Map(); constructor(opts: ClickHouseEngineOptions) { - this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); - this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined; - this.database = opts.database ?? 'opt_analytics'; + this.baseUrl = opts.baseUrl.replace(/\/+$/, ""); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined; + this.database = opts.database ?? "opt_analytics"; this.timeoutMs = opts.timeoutMs ?? 30_000; this.dimensionCatalogs = opts.dimensionCatalogs ?? {}; this.schemaOpts = opts.schema ?? {}; @@ -112,7 +112,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { } private budgetFor(identity: AnalyticsIdentity): ResourceBudget { - const tier = identity.budget_tier ?? 'INTERACTIVE'; + const tier = identity.budget_tier ?? "INTERACTIVE"; return applyQuota(DEFAULT_BUDGETS[tier], identity.quota); } @@ -125,13 +125,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { private assertIdentity(identity: AnalyticsIdentity): void { if (!identity?.tenant_id || !identity.tenant_id.trim()) { throw new SiaxError( - 'UNAUTHENTICATED', - '0pt.analytics', - 'analytics requires a verified CL0UD identity carrying tenant_id', + "UNAUTHENTICATED", + "0pt.analytics", + "analytics requires a verified CL0UD identity carrying tenant_id", ); } if (!identity.principal_id || !identity.principal_id.trim()) { - throw new SiaxError('UNAUTHENTICATED', '0pt.analytics', 'analytics requires a verified principal_id'); + throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id"); } } @@ -150,16 +150,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { async health(): Promise { try { const legacyOk = await this.legacy.health(); - if (!legacyOk) - return { healthy: false, provider: this.provider, detail: 'legacy health probe returned false' }; - const v = await this.raw('SELECT version() AS v FORMAT JSON', {}, {}); + if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" }; + const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {}); const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v; return { healthy: true, provider: this.provider, version }; } catch (err) { return { healthy: false, provider: this.provider, - detail: err instanceof SiaxError ? err.code : 'unreachable', + detail: err instanceof SiaxError ? err.code : "unreachable", }; } } @@ -192,13 +191,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { budget.maxResultRows, ); const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as - ClickHouseJsonResponse | undefined; + | ClickHouseJsonResponse + | undefined; const groupNames = request.group_by ?? []; const rows = (json?.data ?? []).map((r) => { const groups: Record = {}; groupNames.forEach((name, i) => { const v = (r as Record)[`grp${i}`]; - groups[name] = v === undefined || v === null ? '' : String(v); + groups[name] = v === undefined || v === null ? "" : String(v); }); return { ...(groupNames.length ? { groups } : {}), @@ -230,15 +230,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { */ async executeLegacyMetric( identity: AnalyticsIdentity, - request: { - metric: string; - aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count'; - dataset_id?: string; - group_by?: string; - from_time?: string; - to_time?: string; - limit?: number; - }, + request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number }, ): Promise { this.assertIdentity(identity); const started = Date.now(); @@ -272,9 +264,9 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // BI-P033 (analytics authorization) exist would create precisely the // "raw SQL attempt" hole BI-P011's adversarial list enumerates. throw new SiaxError( - 'CAPABILITY_UNAVAILABLE', - '0pt.analytics', - 'analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land', + "CAPABILITY_UNAVAILABLE", + "0pt.analytics", + "analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land", ); } @@ -304,7 +296,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read // AFTER primary-key pruning — which is why tenant_id leads every ORDER BY // in schema.ts: the estimate for one tenant does not include everyone. - const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, ''); + const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, ""); let rowsToRead = 0; let partsRead = 0; try { @@ -321,7 +313,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // An estimate that cannot be produced must not read as "cheap". throw err instanceof SiaxError ? err - : new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'could not estimate query'); + : new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query"); } // Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes. // 128 B/row is a deliberately PESSIMISTIC constant for these tables @@ -355,7 +347,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise { this.assertIdentity(identity); if (!queryId.startsWith(this.queryIdPrefix(identity))) { - throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this tenant'); + throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant"); } this.inFlight.get(queryId)?.abort(); this.inFlight.delete(queryId); @@ -365,7 +357,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { await this.raw( `KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`, { param_killId: queryId }, - { max_execution_time: '10' }, + { max_execution_time: "10" }, ); return true; } catch { @@ -396,14 +388,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { this.catalogFor(identity), budget.maxResultRows, ); - const sql = compiled.sql.replace(/ FORMAT JSON$/, ''); + const sql = compiled.sql.replace(/ FORMAT JSON$/, ""); const res = (await this.raw( `EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`, compiled.params, planningSettings(budget), )) as string | undefined; return { - plan: typeof res === 'string' ? res : JSON.stringify(res ?? ''), + plan: typeof res === "string" ? res : JSON.stringify(res ?? ""), forced_tenant_predicate: FORCED_TENANT_PREDICATE, table: compiled.table, }; @@ -422,22 +414,22 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { const body = request.rows .map((r) => { if (!r.entity_key || !r.entity_key.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every projection row needs an entity_key'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key"); } return JSON.stringify({ // tenant/project come from the VERIFIED identity, never the row — // a row-supplied tenant_id is the INF0 Wave-2 bug class. tenant_id: identity.tenant_id, - project_id: identity.project_id ?? '', + project_id: identity.project_id ?? "", dataset_id: request.dataset_id, entity_key: r.entity_key, - as_of: (r.as_of ?? now.toISOString()).replace('T', ' ').replace(/Z$/, ''), + as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""), version: r.version ?? version, attributes: r.attributes ?? {}, measures: r.measures ?? {}, }); }) - .join('\n'); + .join("\n"); await this.raw( `INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`, {}, @@ -463,7 +455,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { await this.raw( `OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`, { param_tenant: identity.tenant_id }, - { max_execution_time: '120' }, + { max_execution_time: "120" }, ); const compiled = compileProjectionQuery( { tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 }, @@ -474,7 +466,8 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { `SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` + `WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`; const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as - { data?: { value: number | string }[] } | undefined; + | { data?: { value: number | string }[] } + | undefined; return { dataset_id, row_count: Number(res?.data?.[0]?.value ?? 0), @@ -485,16 +478,16 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // ── transport ──────────────────────────────────────────────────────────── private queryIdPrefix(identity: AnalyticsIdentity): string { - return `opt_${crypto.createHash('sha256').update(identity.tenant_id).digest('hex').slice(0, 12)}_`; + return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`; } private newQueryId(identity: AnalyticsIdentity): string { - return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, '')}`; + return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`; } /** DDL path: no readonly, no result parsing beyond errors. */ private async ddl(sql: string): Promise { - await this.raw(sql, {}, { max_execution_time: '60' }); + await this.raw(sql, {}, { max_execution_time: "60" }); } private async raw( @@ -505,13 +498,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { body?: string, ): Promise { const url = new URL(this.baseUrl); - url.searchParams.set('query', sql); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); - if (queryId) url.searchParams.set('query_id', queryId); + if (queryId) url.searchParams.set("query_id", queryId); const headers: Record = {}; - if (this.authHeader) headers['Authorization'] = this.authHeader; + if (this.authHeader) headers["Authorization"] = this.authHeader; const controller = new AbortController(); if (queryId) this.inFlight.set(queryId, controller); @@ -520,13 +513,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { let res: Response; try { res = await fetch(url.toString(), { - method: 'POST', + method: "POST", headers, - body: body ?? '', + body: body ?? "", signal: controller.signal, }); } catch (cause) { - throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause }); } finally { clearTimeout(timeout); } @@ -537,7 +530,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status); throw new SiaxError( siaxErrorCodeForUpstreamStatus(res.status), - 'clickhouse', + "clickhouse", `query failed (CH code ${code})`, ); } @@ -555,10 +548,10 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // have already been flushed, the JSON body can carry an `exception` key // under a 200 — a truncated answer indistinguishable from a whole one. // Treat any body carrying an exception as a failure, whatever the status. - if (parsed && typeof parsed === 'object' && 'exception' in parsed) { - const ex = String((parsed as { exception?: unknown }).exception ?? ''); - const code = /Code: (\d+)/.exec(ex)?.[1] ?? 'unknown'; - throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `query failed mid-stream (CH code ${code})`); + if (parsed && typeof parsed === "object" && "exception" in parsed) { + const ex = String((parsed as { exception?: unknown }).exception ?? ""); + const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown"; + throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`); } return parsed; } diff --git a/src/analytics/engine/query-compiler.ts b/src/analytics/engine/query-compiler.ts index 3a11c54..ffb6aa2 100644 --- a/src/analytics/engine/query-compiler.ts +++ b/src/analytics/engine/query-compiler.ts @@ -27,20 +27,20 @@ * the guarantee rather than replacing it, and * tests/analytics/tenant-predicate-regression.test.ts asserts BOTH. */ -import { SiaxError } from '../../platform/errors'; -import type { AnalyticsTable } from './schema'; +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsTable } from "./schema"; /** A physical identifier that has been VALIDATED, not merely received. */ const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; export function assertSafeIdentifier(value: string, what: string): string { if (!IDENT_RE.test(value)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid ${what} identifier`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`); } return value; } -export const METRIC_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const; +export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const; export type MetricAggregate = (typeof METRIC_AGGREGATES)[number]; /** @@ -97,15 +97,15 @@ export interface CompiledQuery { * forcing, the test fails on a missing substring rather than on a subtle * behavioural difference that a mocked engine would happily hide. */ -export const FORCED_TENANT_PREDICATE = 'tenant_id = {tenant:String}'; +export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}"; const AGG_SQL: Record = { - sum: 'sum(value)', - avg: 'avg(value)', - min: 'min(value)', - max: 'max(value)', - count: 'count()', - count_distinct: 'uniqExact(event_id)', + sum: "sum(value)", + avg: "avg(value)", + min: "min(value)", + max: "max(value)", + count: "count()", + count_distinct: "uniqExact(event_id)", }; const MAX_GROUP_BY = 8; @@ -126,13 +126,13 @@ export function compileMetricQuery( ): CompiledQuery { if (!req.tenant_id || !req.tenant_id.trim()) { // Fail closed. A blank tenant must never compile to "no predicate". - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); } - assertSafeIdentifier(db, 'database'); - const table: AnalyticsTable = req.table ?? 'opt_analytics_event'; - assertSafeIdentifier(table, 'table'); + assertSafeIdentifier(db, "database"); + const table: AnalyticsTable = req.table ?? "opt_analytics_event"; + assertSafeIdentifier(table, "table"); if (!METRIC_AGGREGATES.includes(req.aggregate)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'unsupported aggregate'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate"); } const params: Record = { @@ -141,22 +141,22 @@ export function compileMetricQuery( }; // ── the forced predicate. First condition, always, no branch. ───────── - const conditions: string[] = [FORCED_TENANT_PREDICATE, 'metric = {metric:String}']; + const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"]; if (req.project_id) { - conditions.push('project_id = {project:String}'); + conditions.push("project_id = {project:String}"); params.param_project = req.project_id; } if (req.dataset_id) { - conditions.push('dataset_id = {dataset:String}'); + conditions.push("dataset_id = {dataset:String}"); params.param_dataset = req.dataset_id; } if (req.from_time) { - conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})'); + conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})"); params.param_fromTime = req.from_time; } if (req.to_time) { - conditions.push('event_time < parseDateTime64BestEffort({toTime:String})'); + conditions.push("event_time < parseDateTime64BestEffort({toTime:String})"); params.param_toTime = req.to_time; } @@ -165,14 +165,16 @@ export function compileMetricQuery( for (const [name, value] of Object.entries(req.filters ?? {})) { const physical = catalog.resolve(name); if (physical === undefined) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown filter dimension "${name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`); } // `physical` came from the catalog, but validate anyway: a corrupted or // hostile catalog row must not become an injection vector either. - assertSafeIdentifier(physical, 'dimension'); + assertSafeIdentifier(physical, "dimension"); const nameKey = `f${filterIdx}n`; const valKey = `f${filterIdx}v`; - conditions.push(`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`); + conditions.push( + `dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`, + ); params[`param_${nameKey}`] = physical; params[`param_${valKey}`] = value; filterIdx += 1; @@ -182,8 +184,8 @@ export function compileMetricQuery( const groupBy = req.group_by ?? []; if (groupBy.length > MAX_GROUP_BY) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`, ); } @@ -192,9 +194,9 @@ export function compileMetricQuery( groupBy.forEach((name, i) => { const physical = catalog.resolve(name); if (physical === undefined) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown group_by dimension "${name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`); } - assertSafeIdentifier(physical, 'dimension'); + assertSafeIdentifier(physical, "dimension"); const key = `g${i}`; // The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text. selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`); @@ -205,10 +207,10 @@ export function compileMetricQuery( const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); const sql = - `SELECT ${selectParts.length ? selectParts.join(', ') + ', ' : ''}${AGG_SQL[req.aggregate]} AS value ` + + `SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` + `FROM ${db}.${table} ` + - `WHERE ${conditions.join(' AND ')} ` + - (groupParts.length ? `GROUP BY ${groupParts.join(', ')} ORDER BY value DESC ` : '') + + `WHERE ${conditions.join(" AND ")} ` + + (groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") + `LIMIT ${limit} FORMAT JSON`; return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table }; @@ -224,29 +226,29 @@ export function compileProjectionQuery( maxResultRows: number, ): CompiledQuery { if (!req.tenant_id || !req.tenant_id.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); } - assertSafeIdentifier(db, 'database'); + assertSafeIdentifier(db, "database"); const params: Record = { param_tenant: req.tenant_id, param_dataset: req.dataset_id, }; - const conditions = [FORCED_TENANT_PREDICATE, 'dataset_id = {dataset:String}']; + const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"]; if (req.project_id) { - conditions.push('project_id = {project:String}'); + conditions.push("project_id = {project:String}"); params.param_project = req.project_id; } if (req.entity_keys && req.entity_keys.length > 0) { - conditions.push('has({entityKeys:Array(String)}, entity_key)'); + conditions.push("has({entityKeys:Array(String)}, entity_key)"); params.param_entityKeys = JSON.stringify(req.entity_keys); } const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); const sql = `SELECT entity_key, as_of, version, attributes, measures ` + `FROM ${db}.opt_dataset_projection ` + - `WHERE ${conditions.join(' AND ')} ` + + `WHERE ${conditions.join(" AND ")} ` + `ORDER BY entity_key LIMIT ${limit} FORMAT JSON`; - return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: 'opt_dataset_projection' }; + return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" }; } /** @@ -273,29 +275,25 @@ export function restrictedTenantUserDdl(args: { tables: readonly string[]; settingsProfile?: string; }): string[] { - assertSafeIdentifier(args.db, 'database'); - assertSafeIdentifier(args.userName, 'user'); + assertSafeIdentifier(args.db, "database"); + assertSafeIdentifier(args.userName, "user"); // A tenant id is `t_` in this estate; enforce that shape strictly // rather than trusting the caller. if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant id is not policy-safe'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe"); } - if (args.password.includes("'") || args.password.includes('\\')) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'password contains characters that cannot be embedded', - ); + if (args.password.includes("'") || args.password.includes("\\")) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded"); } const profile = args.settingsProfile; - if (profile) assertSafeIdentifier(profile, 'settings profile'); + if (profile) assertSafeIdentifier(profile, "settings profile"); const stmts: string[] = [ `CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` + (profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`), ]; for (const table of args.tables) { - assertSafeIdentifier(table, 'table'); + assertSafeIdentifier(table, "table"); stmts.push( `CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` + `FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`, diff --git a/src/analytics/engine/resource-governance.ts b/src/analytics/engine/resource-governance.ts index fe7cb6c..4d2e184 100644 --- a/src/analytics/engine/resource-governance.ts +++ b/src/analytics/engine/resource-governance.ts @@ -30,7 +30,7 @@ * subsidises a query bomb. `INTERACTIVE` is the dashboard default and is * deliberately mean. */ -import { SiaxError } from '../../platform/errors'; +import { SiaxError } from "../../platform/errors"; export interface ResourceBudget { /** Seconds. ClickHouse `max_execution_time`. */ @@ -47,7 +47,7 @@ export interface ResourceBudget { maxConcurrentQueries: number; } -export const BUDGET_TIERS = ['INTERACTIVE', 'ANALYTICAL', 'BATCH'] as const; +export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const; export type BudgetTier = (typeof BUDGET_TIERS)[number]; /** @@ -91,7 +91,7 @@ export const DEFAULT_BUDGETS: Record = { export function applyQuota(base: ResourceBudget, quota: Partial | undefined): ResourceBudget { if (!quota) return { ...base }; const pick = (b: number, q: number | undefined): number => - typeof q === 'number' && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b; + typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b; return { maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec), maxResultRows: pick(base.maxResultRows, quota.maxResultRows), @@ -114,14 +114,14 @@ export function applyQuota(base: ResourceBudget, quota: Partial */ export function clickhouseSettings(budget: ResourceBudget): Record { return { - readonly: '1', + readonly: "1", max_execution_time: String(budget.maxExecutionTimeSec), - timeout_overflow_mode: 'throw', + timeout_overflow_mode: "throw", max_result_rows: String(budget.maxResultRows), - result_overflow_mode: 'throw', + result_overflow_mode: "throw", max_rows_to_read: String(budget.maxRowsToRead), max_bytes_to_read: String(budget.maxBytesToRead), - read_overflow_mode: 'throw', + read_overflow_mode: "throw", max_memory_usage: String(budget.maxMemoryUsageBytes), }; } @@ -139,21 +139,21 @@ export function clickhouseSettings(budget: ResourceBudget): Record { return { - readonly: '1', + readonly: "1", max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)), - timeout_overflow_mode: 'throw', + timeout_overflow_mode: "throw", max_memory_usage: String(budget.maxMemoryUsageBytes), }; } /** Every setting name BI-P012 requires, for the DoD assertion. */ export const REQUIRED_GOVERNANCE_SETTINGS = [ - 'max_execution_time', - 'max_result_rows', - 'max_rows_to_read', - 'max_bytes_to_read', - 'max_memory_usage', - 'max_concurrent_queries', + "max_execution_time", + "max_result_rows", + "max_rows_to_read", + "max_bytes_to_read", + "max_memory_usage", + "max_concurrent_queries", ] as const; /** @@ -192,8 +192,8 @@ export class ConcurrencyGate { const current = this.inFlight.get(tenantId) ?? 0; if (current >= limit) { throw new SiaxError( - 'RATE_LIMITED', - '0pt.analytics', + "RATE_LIMITED", + "0pt.analytics", `tenant already has ${current} analytics queries in flight (limit ${limit})`, ); } diff --git a/src/analytics/engine/schema.ts b/src/analytics/engine/schema.ts index 1825a8d..657d1fb 100644 --- a/src/analytics/engine/schema.ts +++ b/src/analytics/engine/schema.ts @@ -55,14 +55,14 @@ export interface AnalyticsSchemaOptions { /** Physical tables this module owns (opt_metric_events is NOT one of them). */ export const ANALYTICS_TABLES = [ - 'opt_analytics_event', - 'opt_metric_daily', - 'opt_dataset_projection', + "opt_analytics_event", + "opt_metric_daily", + "opt_dataset_projection", ] as const; export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number]; /** The materialized view wiring opt_analytics_event -> opt_metric_daily. */ -export const ANALYTICS_MATERIALIZED_VIEWS = ['opt_metric_daily_mv'] as const; +export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const; /** * DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so @@ -163,6 +163,7 @@ export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] { */ export function sortingKeySql(db: string, table: string): string { return ( - `SELECT sorting_key FROM system.tables ` + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` + `SELECT sorting_key FROM system.tables ` + + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` ); } diff --git a/src/analytics/engine/siax-analytics-engine.ts b/src/analytics/engine/siax-analytics-engine.ts index 168dd02..53ac17a 100644 --- a/src/analytics/engine/siax-analytics-engine.ts +++ b/src/analytics/engine/siax-analytics-engine.ts @@ -19,9 +19,9 @@ * (VehIQ) call an API in front of this interface; they never hold a provider * credential and never construct an engine. */ -import type { CompiledQuery, MetricAggregate } from './query-compiler'; -import type { AnalyticsTable } from './schema'; -import type { BudgetTier, ResourceBudget } from './resource-governance'; +import type { CompiledQuery, MetricAggregate } from "./query-compiler"; +import type { AnalyticsTable } from "./schema"; +import type { BudgetTier, ResourceBudget } from "./resource-governance"; /** * The VERIFIED caller. `tenant_id` here is the one the engine forces into @@ -158,24 +158,21 @@ export interface SiaxAnalyticsEngine { explain(identity: AnalyticsIdentity, request: MetricRequest): Promise; - ingestProjection( - identity: AnalyticsIdentity, - request: ProjectionIngestRequest, - ): Promise<{ ingested: number }>; + ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>; refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise; } /** The eight capability names, for the provider contract test. */ export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [ - 'health', - 'executeMetric', - 'executeQuery', - 'estimateQuery', - 'cancelQuery', - 'explain', - 'ingestProjection', - 'refreshProjection', + "health", + "executeMetric", + "executeQuery", + "estimateQuery", + "cancelQuery", + "explain", + "ingestProjection", + "refreshProjection", ] as const; export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number]; diff --git a/src/analytics/repositories/analytics-domain-repository.ts b/src/analytics/repositories/analytics-domain-repository.ts index b7a341e..8b01c37 100644 --- a/src/analytics/repositories/analytics-domain-repository.ts +++ b/src/analytics/repositories/analytics-domain-repository.ts @@ -19,9 +19,9 @@ * Uses postgres.js through `RepositorySql`, the same connection type the other * P0 repositories take (src/repositories/db.ts). */ -import crypto from 'node:crypto'; -import { SiaxError } from '../../platform/errors'; -import type { RepositorySql } from '../../repositories/db'; +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { RepositorySql } from "../../repositories/db"; import type { AnalyticsDataset, DatasetVersion, @@ -29,8 +29,8 @@ import type { MetricDefinition, MetricVersion, PiiClassification, -} from '../domain/entities'; -import type { AnalyticsTier } from '../domain/tiers'; +} from "../domain/entities"; +import type { AnalyticsTier } from "../domain/tiers"; /** The verified tenant scope every call must present. */ export interface TenantScope { @@ -41,9 +41,9 @@ export interface TenantScope { function assertScope(scope: TenantScope): void { if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) { throw new SiaxError( - 'UNAUTHENTICATED', - '0pt.analytics', - 'analytics persistence requires a verified tenant_id + project_id', + "UNAUTHENTICATED", + "0pt.analytics", + "analytics persistence requires a verified tenant_id + project_id", ); } } @@ -90,7 +90,7 @@ function toDataset(r: DatasetRow): AnalyticsDataset { quality_rule_ids: r.quality_rule_ids ?? [], retention_days: Number(r.retention_days), classification: r.classification as PiiClassification, - status: r.status as AnalyticsDataset['status'], + status: r.status as AnalyticsDataset["status"], ...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}), ...(r.description ? { description: r.description } : {}), created_at: iso(r.created_at)!, @@ -104,10 +104,7 @@ export class AnalyticsDomainRepository { // ── AnalyticsDataset ───────────────────────────────────────────────────── - async createDataset( - scope: TenantScope, - input: Omit, - ): Promise { + async createDataset(scope: TenantScope, input: Omit): Promise { assertScope(scope); const rows = await this.sql` INSERT INTO opt_analytics_dataset @@ -136,10 +133,7 @@ export class AnalyticsDomainRepository { return rows[0] ? toDataset(rows[0]) : undefined; } - async listDatasets( - scope: TenantScope, - filter?: { tier?: AnalyticsTier; status?: string }, - ): Promise { + async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise { assertScope(scope); const rows = await this.sql` SELECT * FROM opt_analytics_dataset @@ -161,20 +155,14 @@ export class AnalyticsDomainRepository { */ async createDatasetVersion( scope: TenantScope, - input: { - dataset_id: string; - contract_hash: string; - schema_contract_version: string; - record_count?: number; - created_by?: string; - }, + input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string }, ): Promise { assertScope(scope); const owner = await this.getDataset(scope, input.dataset_id); if (!owner) { - throw new SiaxError('NOT_FOUND', '0pt.analytics', 'dataset not found in this tenant'); + throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant"); } - const id = `dsv_${crypto.randomUUID().replace(/-/g, '')}`; + const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`; try { const rows = await this.sql< { @@ -214,8 +202,8 @@ export class AnalyticsDomainRepository { ...(r.created_by ? { created_by: r.created_by } : {}), }; } catch (err) { - if (String(err).includes('duplicate key')) { - throw new SiaxError('CONFLICT', '0pt.analytics', 'a concurrent writer minted this dataset version'); + if (String(err).includes("duplicate key")) { + throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version"); } throw err; } @@ -224,18 +212,7 @@ export class AnalyticsDomainRepository { async listDatasetVersions(scope: TenantScope, datasetId: string): Promise { assertScope(scope); const rows = await this.sql< - { - dataset_version_id: string; - dataset_id: string; - tenant_id: string; - project_id: string; - version: number; - contract_hash: string; - schema_contract_version: string; - record_count: string | number | null; - created_at: string | Date; - created_by: string | null; - }[] + { dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[] >` SELECT * FROM opt_analytics_dataset_version WHERE dataset_id = ${datasetId} @@ -260,29 +237,11 @@ export class AnalyticsDomainRepository { async createMetric( scope: TenantScope, - input: Omit< - MetricDefinition, - 'tenant_id' | 'project_id' | 'created_at' | 'updated_at' | 'current_version' - >, + input: Omit, ): Promise { assertScope(scope); const rows = await this.sql< - { - metric_id: string; - tenant_id: string; - project_id: string; - name: string; - dataset_id: string; - aggregation: string; - measure_field: string; - denominator_field: string | null; - allowed_dimension_ids: string[]; - current_version: number; - owner_id: string; - description: string | null; - created_at: string | Date; - updated_at: string | Date | null; - }[] + { metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[] >` INSERT INTO opt_analytics_metric_definition (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, @@ -299,7 +258,7 @@ export class AnalyticsDomainRepository { project_id: r.project_id, name: r.name, dataset_id: r.dataset_id, - aggregation: r.aggregation as MetricDefinition['aggregation'], + aggregation: r.aggregation as MetricDefinition["aggregation"], measure_field: r.measure_field, ...(r.denominator_field ? { denominator_field: r.denominator_field } : {}), allowed_dimension_ids: r.allowed_dimension_ids ?? [], @@ -317,21 +276,10 @@ export class AnalyticsDomainRepository { input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string }, ): Promise { assertScope(scope); - const hash = crypto.createHash('sha256').update(input.definition_json).digest('hex'); - const id = `mv_${crypto.randomUUID().replace(/-/g, '')}`; + const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex"); + const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`; const rows = await this.sql< - { - metric_version_id: string; - metric_id: string; - tenant_id: string; - project_id: string; - version: number; - definition_json: string; - definition_hash: string; - change_reason: string | null; - created_at: string | Date; - created_by: string | null; - }[] + { metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[] >` INSERT INTO opt_analytics_metric_version (metric_version_id, metric_id, tenant_id, project_id, version, definition_json, @@ -361,22 +309,11 @@ export class AnalyticsDomainRepository { async createDimension( scope: TenantScope, - input: Omit, + input: Omit, ): Promise { assertScope(scope); const rows = await this.sql< - { - dimension_id: string; - tenant_id: string; - project_id: string; - name: string; - dataset_id: string; - field_name: string; - field_type: string; - classification: string; - description: string | null; - created_at: string | Date; - }[] + { dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[] >` INSERT INTO opt_analytics_dimension_definition (dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description) @@ -392,7 +329,7 @@ export class AnalyticsDomainRepository { name: r.name, dataset_id: r.dataset_id, field_name: r.field_name, - field_type: r.field_type as DimensionDefinition['field_type'], + field_type: r.field_type as DimensionDefinition["field_type"], classification: r.classification as PiiClassification, ...(r.description ? { description: r.description } : {}), created_at: iso(r.created_at)!, diff --git a/tests/analytics/duckdb-provider.test.ts b/tests/analytics/duckdb-provider.test.ts index 9039821..43fcac7 100644 --- a/tests/analytics/duckdb-provider.test.ts +++ b/tests/analytics/duckdb-provider.test.ts @@ -13,6 +13,7 @@ * /etc/passwd. Measured, recorded, and asserted here — because a comment * saying so would rot the moment someone "simplified" the bootstrap. */ +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -307,6 +308,19 @@ describe('BI-P014 — the committed fixtures', () => { }); it('does NOT ship the Iceberg table, because its manifests carry absolute paths', () => { - expect(fs.existsSync(path.join(FIXTURES, 'iceberg'))).toBe(false); + // Asserted against GIT, not the filesystem. The claim is about what is + // COMMITTED; the live suite REBUILDS that table locally, so a filesystem + // check fails whenever both suites run in one pass — it was testing the + // wrong thing rather than catching a real problem. + const tracked = execFileSync('git', ['ls-files', 'tests/fixtures/duckdb/'], { + encoding: 'utf8', + cwd: process.cwd(), + }) + .split('\n') + .filter(Boolean); + // The parquet/csv fixtures ARE tracked, so a version that shipped nothing + // would fail here too rather than passing vacuously. + expect(tracked.length).toBeGreaterThan(0); + expect(tracked.filter((f) => f.includes('/iceberg'))).toEqual([]); }); }); diff --git a/tests/integration/analytics-arrow-clickhouse.integration.test.ts b/tests/integration/analytics-arrow-clickhouse.integration.test.ts index b87e38f..648ed06 100644 --- a/tests/integration/analytics-arrow-clickhouse.integration.test.ts +++ b/tests/integration/analytics-arrow-clickhouse.integration.test.ts @@ -17,13 +17,13 @@ * ran against real pyarrow + DuckDB; that transcript is committed at * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { arrowStreamSettings, clickhouseArrowStreamFormat, decodeArrowStream, encodeArrowStream, -} from '@/src/analytics/arrow/ipc'; +} from "@/src/analytics/arrow/ipc"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -32,29 +32,31 @@ const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_arrow_${Date.now().toString(36)}`; function authHeaders(): Record { - return CLICKHOUSE_AUTH ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } : {}; + return CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` } + : {}; } async function chBytes(sql: string, settings: Record = {}): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); - const res = await fetch(url.toString(), { method: 'POST', headers: authHeaders(), body: '' }); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" }); const buf = new Uint8Array(await res.arrayBuffer()); if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`); return buf; } async function chText(sql: string, body?: BodyInit): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); - const res = await fetch(url.toString(), { method: 'POST', headers: authHeaders(), body: body ?? '' }); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" }); const text = await res.text(); if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`); return text; } -run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { +run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => { beforeAll(async () => { await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`); await chText( @@ -68,7 +70,7 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`); }, 60_000); - it('THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count', async () => { + it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => { const bytes = await chBytes( `SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag, toDateTime64(ts, 3, 'UTC') AS ts @@ -84,38 +86,35 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { ); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(3); - expect(info.fields.map((f) => f.name)).toEqual(['tenant_id', 'n', 'v', 'flag', 'ts']); + expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); // Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys), // Int=2, FloatingPoint=3, Bool=6, Timestamp=10. expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); }); - it('WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents', async () => { + it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => { const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, { - output_format_arrow_string_as_string: '0', + output_format_arrow_string_as_string: "0", }); // Arrow Type union: Binary = 4, Utf8 = 5. expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4); - const asText = await chBytes( - `SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, - arrowStreamSettings(), - ); + const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings()); expect(decodeArrowStream(asText).fields[0].typeType).toBe(5); }); - it('OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged', async () => { + it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => { const bytes = encodeArrowStream([ - { name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', 't_gamma'] }, - { name: 'n', type: 'int64', values: [1, 2, 3] }, - { name: 'v', type: 'float64', values: [1.5, 2.5, 3.25] }, - { name: 'flag', type: 'bool', values: [true, false, true] }, + { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, + { name: "n", type: "int64", values: [1, 2, 3] }, + { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, + { name: "flag", type: "bool", values: [true, false, true] }, { - name: 'ts', - type: 'timestamp_ms', + name: "ts", + type: "timestamp_ms", values: [ - new Date('2026-01-01T00:00:00.000Z'), - new Date('2026-06-15T08:09:10.123Z'), - new Date('2026-09-06T12:34:56.789Z'), + new Date("2026-01-01T00:00:00.000Z"), + new Date("2026-06-15T08:09:10.123Z"), + new Date("2026-09-06T12:34:56.789Z"), ], }, ]); @@ -124,30 +123,22 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { const back = await chText( `SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`, ); - const rows = ( - JSON.parse(back) as { - data: { - tenant_id: string; - n: string | number; - v: number | string; - flag: boolean | number; - ts: string; - }[]; - } - ).data; + const rows = (JSON.parse(back) as { + data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[]; + }).data; expect(rows).toHaveLength(3); - expect(rows.map((r) => r.tenant_id)).toEqual(['t_alpha', 't_beta', 't_gamma']); + expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]); expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]); expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]); expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]); expect(rows.map((r) => r.ts)).toEqual([ - '2026-01-01 00:00:00.000', - '2026-06-15 08:09:10.123', - '2026-09-06 12:34:56.789', + "2026-01-01 00:00:00.000", + "2026-06-15 08:09:10.123", + "2026-09-06 12:34:56.789", ]); }); - it('full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows', async () => { + it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => { const exported = await chBytes( `SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`, arrowStreamSettings(), @@ -167,18 +158,14 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0); }); - it('a large result survives the round trip (batching/offsets are not only correct for three rows)', async () => { + it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => { const n = 20_000; const bytes = encodeArrowStream([ - { name: 'tenant_id', type: 'utf8', values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, - { name: 'n', type: 'int64', values: Array.from({ length: n }, (_, i) => i) }, - { name: 'v', type: 'float64', values: Array.from({ length: n }, (_, i) => i / 3) }, - { name: 'flag', type: 'bool', values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, - { - name: 'ts', - type: 'timestamp_ms', - values: Array.from({ length: n }, () => new Date('2026-09-06T00:00:00Z')), - }, + { name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, + { name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) }, + { name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) }, + { name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, + { name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) }, ]); expect(decodeArrowStream(bytes).rowCount).toBe(n); @@ -187,11 +174,7 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { const agg = await chText( `SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`, ); - const d = ( - JSON.parse(agg) as { - data: { c: string | number; s: string | number; t: string | number; f: string | number }[]; - } - ).data[0]; + const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0]; expect(Number(d.c)).toBe(n); expect(Number(d.s)).toBe((n * (n - 1)) / 2); expect(Number(d.t)).toBe(7); diff --git a/tests/integration/analytics-clickhouse.integration.test.ts b/tests/integration/analytics-clickhouse.integration.test.ts index 64e9f00..8d072cb 100644 --- a/tests/integration/analytics-clickhouse.integration.test.ts +++ b/tests/integration/analytics-clickhouse.integration.test.ts @@ -7,19 +7,19 @@ * JSONEachRow ingest, the closed metric-query DSL, and — the §69-relevant * part — that the FORCED tenant predicate isolates tenants on real data. */ -import { describe, it, expect, beforeAll } from 'vitest'; -import { ClickHouseAnalytics } from '@/src/analytics/clickhouse-analytics'; +import { describe, it, expect, beforeAll } from "vitest"; +import { ClickHouseAnalytics } from "@/src/analytics/clickhouse-analytics"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_itest_${Date.now().toString(36)}`; -run('Analytics capability against a LIVE ClickHouse', () => { +run("Analytics capability against a LIVE ClickHouse", () => { // The skip-variant still evaluates this callback body, so the constructor // must tolerate the env-less case (the tests themselves never run then). const analytics = new ClickHouseAnalytics({ - baseUrl: CLICKHOUSE_URL ?? 'http://skipped.invalid', + baseUrl: CLICKHOUSE_URL ?? "http://skipped.invalid", basicAuth: process.env.CLICKHOUSE_AUTH, database: DB, }); @@ -31,90 +31,66 @@ run('Analytics capability against a LIVE ClickHouse', () => { await analytics.ensureSchema(30); }, 60_000); - it('ingests events and answers aggregate metric queries', async () => { + it("ingests events and answers aggregate metric queries", async () => { const res = await analytics.ingest([ - { tenant_id: 't_alpha', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 100 }, - { tenant_id: 't_alpha', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 50 }, - { tenant_id: 't_alpha', dataset_id: 'ds_vehicles', metric: 'rows_ingested', value: 7 }, - { tenant_id: 't_beta', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 999 }, + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 100 }, + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 50 }, + { tenant_id: "t_alpha", dataset_id: "ds_vehicles", metric: "rows_ingested", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_companies", metric: "rows_ingested", value: 999 }, ]); expect(res.ingested).toBe(4); const total = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'rows_ingested', - aggregate: 'sum', + tenant_id: "t_alpha", + metric: "rows_ingested", + aggregate: "sum", }); expect(total).toEqual([{ value: 157 }]); const perDataset = await analytics.queryMetric({ - tenant_id: 't_alpha', - dataset_id: 'ds_companies', - metric: 'rows_ingested', - aggregate: 'sum', + tenant_id: "t_alpha", + dataset_id: "ds_companies", + metric: "rows_ingested", + aggregate: "sum", }); expect(perDataset).toEqual([{ value: 150 }]); }, 30_000); it("REAL tenant isolation: t_beta's numbers never leak into t_alpha aggregates", async () => { - const alpha = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'rows_ingested', - aggregate: 'max', - }); + const alpha = await analytics.queryMetric({ tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "max" }); expect(alpha[0].value).toBe(100); // NOT 999 - const beta = await analytics.queryMetric({ - tenant_id: 't_beta', - metric: 'rows_ingested', - aggregate: 'sum', - }); + const beta = await analytics.queryMetric({ tenant_id: "t_beta", metric: "rows_ingested", aggregate: "sum" }); expect(beta[0].value).toBe(999); - const ghost = await analytics.queryMetric({ - tenant_id: 't_gamma', - metric: 'rows_ingested', - aggregate: 'count', - }); + const ghost = await analytics.queryMetric({ tenant_id: "t_gamma", metric: "rows_ingested", aggregate: "count" }); expect(ghost[0].value).toBe(0); }); - it('group-by dimension queries work through the closed DSL', async () => { + it("group-by dimension queries work through the closed DSL", async () => { await analytics.ingest([ - { - tenant_id: 't_alpha', - dataset_id: 'ds_x', - metric: 'dq_score', - value: 90, - dimensions: { severity: 'warning' }, - }, - { - tenant_id: 't_alpha', - dataset_id: 'ds_x', - metric: 'dq_score', - value: 40, - dimensions: { severity: 'critical' }, - }, + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 90, dimensions: { severity: "warning" } }, + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 40, dimensions: { severity: "critical" } }, ]); const rows = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'dq_score', - aggregate: 'avg', - group_by: 'severity', + tenant_id: "t_alpha", + metric: "dq_score", + aggregate: "avg", + group_by: "severity", }); expect(rows).toHaveLength(2); - expect(rows.find((r) => r.group === 'warning')?.value).toBe(90); + expect(rows.find((r) => r.group === "warning")?.value).toBe(90); }); - it('rejects tenant-less queries and invalid group_by identifiers (injection surface closed)', async () => { + it("rejects tenant-less queries and invalid group_by identifiers (injection surface closed)", async () => { await expect( - analytics.queryMetric({ tenant_id: ' ', metric: 'x', aggregate: 'sum' }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + analytics.queryMetric({ tenant_id: " ", metric: "x", aggregate: "sum" }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); await expect( analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'x', - aggregate: 'sum', + tenant_id: "t_alpha", + metric: "x", + aggregate: "sum", group_by: "sev'; DROP TABLE opt_metric_events; --", - }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); }); }); diff --git a/tests/integration/analytics-engine-clickhouse.integration.test.ts b/tests/integration/analytics-engine-clickhouse.integration.test.ts index db88ff5..d798ca9 100644 --- a/tests/integration/analytics-engine-clickhouse.integration.test.ts +++ b/tests/integration/analytics-engine-clickhouse.integration.test.ts @@ -12,38 +12,38 @@ * Gated on CLICKHOUSE_URL; skips honestly when unset. CI's `integration` job * supplies it and hard-fails on a skipped suite. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; -import { ANALYTICS_MATERIALIZED_VIEWS, ANALYTICS_TABLES } from '@/src/analytics/engine/schema'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { ANALYTICS_MATERIALIZED_VIEWS, ANALYTICS_TABLES } from "@/src/analytics/engine/schema"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_engine_${Date.now().toString(36)}`; -const ID: AnalyticsIdentity = { tenant_id: 't_alpha', project_id: 'p_main', principal_id: 'usr_alpha' }; +const ID: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; let engine: ClickHouseAnalyticsEngine; async function raw(sql: string, body?: string): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); const headers: Record = {}; - if (CLICKHOUSE_AUTH) headers['Authorization'] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); const text = await res.text(); if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 300)}`); return text; } -run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { +run("BI-P009/BI-P010 ClickHouse provider against a LIVE server", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_alpha: { severity: 'severity', make: 'vehicle_make' } }, + dimensionCatalogs: { t_alpha: { severity: "severity", make: "vehicle_make" } }, }); await engine.ensureSchema(); await engine.ensureSchema(); // idempotent, as the DDL claims @@ -58,67 +58,67 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { it("health() reports the live server's version", async () => { const h = await engine.health(); expect(h.healthy).toBe(true); - expect(h.provider).toBe('clickhouse'); + expect(h.provider).toBe("clickhouse"); expect(h.version).toMatch(/^\d+\.\d+/); }); // ── BI-P010: the schema is what it claims to be ───────────────────────── - it('BI-P010: four purpose-built tables exist, NOT one giant table', async () => { + it("BI-P010: four purpose-built tables exist, NOT one giant table", async () => { const listed = await raw( `SELECT name FROM system.tables WHERE database = '${DB}' AND engine != 'MaterializedView' ORDER BY name FORMAT TSV`, ); - const names = listed.trim().split('\n').filter(Boolean); + const names = listed.trim().split("\n").filter(Boolean); // The three this module owns, PLUS the preserved opt_metric_events. for (const t of ANALYTICS_TABLES) expect(names).toContain(t); - expect(names).toContain('opt_metric_events'); + expect(names).toContain("opt_metric_events"); expect(names.length).toBeGreaterThanOrEqual(4); }); - it('BI-P010: the tables use different engines and different sort keys — the split is real', async () => { + it("BI-P010: the tables use different engines and different sort keys — the split is real", async () => { const meta = await raw( `SELECT name, engine, sorting_key FROM system.tables WHERE database = '${DB}' ORDER BY name FORMAT JSON`, ); const rows = (JSON.parse(meta) as { data: { name: string; engine: string; sorting_key: string }[] }).data; const by = Object.fromEntries(rows.map((r) => [r.name, r])); - expect(by['opt_analytics_event'].engine).toBe('MergeTree'); - expect(by['opt_metric_daily'].engine).toBe('AggregatingMergeTree'); - expect(by['opt_dataset_projection'].engine).toBe('ReplacingMergeTree'); + expect(by["opt_analytics_event"].engine).toBe("MergeTree"); + expect(by["opt_metric_daily"].engine).toBe("AggregatingMergeTree"); + expect(by["opt_dataset_projection"].engine).toBe("ReplacingMergeTree"); // Different ORDER BY keys — the reason one table cannot serve both jobs. - expect(by['opt_analytics_event'].sorting_key).not.toBe(by['opt_dataset_projection'].sorting_key); + expect(by["opt_analytics_event"].sorting_key).not.toBe(by["opt_dataset_projection"].sorting_key); // Every one still leads with tenant_id. - for (const t of ANALYTICS_TABLES) expect(by[t].sorting_key.startsWith('tenant_id')).toBe(true); + for (const t of ANALYTICS_TABLES) expect(by[t].sorting_key.startsWith("tenant_id")).toBe(true); }); - it('BI-P010: TTL and LowCardinality are actually applied, not just written in a comment', async () => { + it("BI-P010: TTL and LowCardinality are actually applied, not just written in a comment", async () => { const ddl = await raw(`SHOW CREATE TABLE ${DB}.opt_analytics_event FORMAT TSVRaw`); - expect(ddl).toContain('TTL'); - expect(ddl).toContain('LowCardinality(String)'); - expect(ddl).toContain('PARTITION BY toYYYYMM(event_time)'); + expect(ddl).toContain("TTL"); + expect(ddl).toContain("LowCardinality(String)"); + expect(ddl).toContain("PARTITION BY toYYYYMM(event_time)"); const rollup = await raw(`SHOW CREATE TABLE ${DB}.opt_metric_daily FORMAT TSVRaw`); - expect(rollup).toContain('AggregateFunction(sum, Float64)'); + expect(rollup).toContain("AggregateFunction(sum, Float64)"); }); - it('BI-P010: the materialized view actually populates the rollup', async () => { + it("BI-P010: the materialized view actually populates the rollup", async () => { const views = await raw( `SELECT name FROM system.tables WHERE database='${DB}' AND engine='MaterializedView' FORMAT TSV`, ); for (const v of ANALYTICS_MATERIALIZED_VIEWS) expect(views).toContain(v); - await engine.ingestProjection(ID, { dataset_id: 'warmup', rows: [{ entity_key: 'w1' }] }); + await engine.ingestProjection(ID, { dataset_id: "warmup", rows: [{ entity_key: "w1" }] }); const rows = Array.from({ length: 20 }, (_, i) => JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_mv', - metric: 'mv_metric', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_mv", + metric: "mv_metric", event_id: `mv${i}`, - event_time: '2026-09-03 12:00:00.000', + event_time: "2026-09-03 12:00:00.000", value: 2, dim_names: [], dim_values: [], }), - ).join('\n'); + ).join("\n"); await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows); const rolled = await raw( @@ -132,75 +132,72 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { // ── capability 2: executeMetric (already covered by the isolation suite) ─ - it('executeMetric returns grouped rows keyed by the CALLER-FACING dimension name', async () => { - const rows = ['warning', 'critical', 'critical'].map((sev, i) => + it("executeMetric returns grouped rows keyed by the CALLER-FACING dimension name", async () => { + const rows = ["warning", "critical", "critical"].map((sev, i) => JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_g', - metric: 'grouped', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_g", + metric: "grouped", event_id: `g${i}`, - event_time: '2026-09-04 09:00:00.000', + event_time: "2026-09-04 09:00:00.000", value: 5, - dim_names: ['severity'], + dim_names: ["severity"], dim_values: [sev], }), ); - await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows.join('\n')); + await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows.join("\n")); const result = await engine.executeMetric(ID, { - metric: 'grouped', - aggregate: 'sum', - group_by: ['severity'], + metric: "grouped", + aggregate: "sum", + group_by: ["severity"], }); - const critical = result.rows.find((r) => r.groups?.severity === 'critical'); + const critical = result.rows.find((r) => r.groups?.severity === "critical"); expect(critical?.value).toBe(10); expect(result.stats.result_rows).toBe(2); - expect(result.query_id.startsWith('opt_')).toBe(true); + expect(result.query_id.startsWith("opt_")).toBe(true); }); // ── capability 4: estimateQuery ───────────────────────────────────────── - it('estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget', async () => { - const est = await engine.estimateQuery(ID, { metric: 'grouped', aggregate: 'sum' }); + it("estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget", async () => { + const est = await engine.estimateQuery(ID, { metric: "grouped", aggregate: "sum" }); expect(est.estimated_rows_to_read).toBeGreaterThan(0); expect(est.within_budget).toBe(true); expect(est.budget.maxRowsToRead).toBeGreaterThan(0); }); - it('estimateQuery REFUSES a query whose estimate exceeds a tightened quota', async () => { + it("estimateQuery REFUSES a query whose estimate exceeds a tightened quota", async () => { const tiny: AnalyticsIdentity = { ...ID, quota: { maxRowsToRead: 1 } }; - const est = await engine.estimateQuery(tiny, { metric: 'grouped', aggregate: 'sum' }); + const est = await engine.estimateQuery(tiny, { metric: "grouped", aggregate: "sum" }); expect(est.within_budget).toBe(false); expect(est.reason).toMatch(/over budget/); }); // ── capability 6: explain ─────────────────────────────────────────────── - it('explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts', async () => { - const ex = await engine.explain(ID, { metric: 'grouped', aggregate: 'sum', group_by: ['severity'] }); - expect(ex.forced_tenant_predicate).toBe('tenant_id = {tenant:String}'); - expect(ex.table).toBe('opt_analytics_event'); + it("explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts", async () => { + const ex = await engine.explain(ID, { metric: "grouped", aggregate: "sum", group_by: ["severity"] }); + expect(ex.forced_tenant_predicate).toBe("tenant_id = {tenant:String}"); + expect(ex.table).toBe("opt_analytics_event"); // This is the payoff of BI-P010's tenant-first ORDER BY: the server // reports tenant_id as a PrimaryKey key with a condition on it, i.e. the // isolation predicate is an index range scan, not a post-read filter. - expect(ex.plan).toContain('ReadFromMergeTree'); - expect(ex.plan).toContain('PrimaryKey'); + expect(ex.plan).toContain("ReadFromMergeTree"); + expect(ex.plan).toContain("PrimaryKey"); expect(ex.plan).toMatch(/Keys:[\s\S]*tenant_id/); expect(ex.plan).toMatch(/tenant_id in \['t_alpha', 't_alpha'\]/); // The plan describes only the CALLER's own tenant; no other tenant id // may appear in a plan handed back to them. - expect(ex.plan).not.toContain('t_beta'); + expect(ex.plan).not.toContain("t_beta"); }); // ── capability 5: cancelQuery ─────────────────────────────────────────── - it('cancelQuery issues a real KILL QUERY the server accepts', async () => { + it("cancelQuery issues a real KILL QUERY the server accepts", async () => { // Nothing is running under this id, so the honest outcome is "the KILL // statement was valid and matched nothing" -> true, with no exception. - const ok = await engine.cancelQuery( - ID, - `${(await engine.executeMetric(ID, { metric: 'grouped', aggregate: 'count' })).query_id}`, - ); + const ok = await engine.cancelQuery(ID, `${(await engine.executeMetric(ID, { metric: "grouped", aggregate: "count" })).query_id}`); expect(ok).toBe(true); }); @@ -208,10 +205,10 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { it("ingestProjection writes the SERVING tier with the identity's tenant, ignoring any row-level claim", async () => { const res = await engine.ingestProjection(ID, { - dataset_id: 'ds_vehicles', + dataset_id: "ds_vehicles", rows: [ - { entity_key: 'VIN001', attributes: { make: 'volvo' }, measures: { price: 189000 } }, - { entity_key: 'VIN002', attributes: { make: 'saab' }, measures: { price: 42000 } }, + { entity_key: "VIN001", attributes: { make: "volvo" }, measures: { price: 189000 } }, + { entity_key: "VIN002", attributes: { make: "saab" }, measures: { price: 42000 } }, ], }); expect(res.ingested).toBe(2); @@ -222,18 +219,18 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { expect(Number((JSON.parse(stored) as { data: { t: number | string }[] }).data[0].t)).toBe(1); }); - it('refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL)', async () => { + it("refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL)", async () => { // Re-ingest the SAME entity keys at a higher version. await engine.ingestProjection(ID, { - dataset_id: 'ds_vehicles', + dataset_id: "ds_vehicles", rows: [ - { entity_key: 'VIN001', version: 9_999_999_999_999, measures: { price: 175000 } }, - { entity_key: 'VIN002', version: 9_999_999_999_999, measures: { price: 41000 } }, + { entity_key: "VIN001", version: 9_999_999_999_999, measures: { price: 175000 } }, + { entity_key: "VIN002", version: 9_999_999_999_999, measures: { price: 41000 } }, ], }); - const refreshed = await engine.refreshProjection(ID, 'ds_vehicles'); + const refreshed = await engine.refreshProjection(ID, "ds_vehicles"); expect(refreshed.row_count).toBe(2); // 2 entities, not 4 rows - expect(refreshed.dataset_id).toBe('ds_vehicles'); + expect(refreshed.dataset_id).toBe("ds_vehicles"); const winner = await raw( `SELECT measures['price'] AS p FROM ${DB}.opt_dataset_projection FINAL ` + @@ -244,12 +241,12 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { // ── the PRESERVED path still works through the engine ─────────────────── - it('the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine', async () => { + it("the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine", async () => { await engine.legacyAnalytics.ingest([ - { tenant_id: 't_alpha', dataset_id: 'ds_legacy', metric: 'legacy_metric', value: 7 }, - { tenant_id: 't_beta', dataset_id: 'ds_legacy', metric: 'legacy_metric', value: 700 }, + { tenant_id: "t_alpha", dataset_id: "ds_legacy", metric: "legacy_metric", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_legacy", metric: "legacy_metric", value: 700 }, ]); - const res = await engine.executeLegacyMetric(ID, { metric: 'legacy_metric', aggregate: 'sum' }); + const res = await engine.executeLegacyMetric(ID, { metric: "legacy_metric", aggregate: "sum" }); expect(res.rows[0].value).toBe(7); // NOT 707 }); }); diff --git a/tests/integration/analytics-resource-governance.integration.test.ts b/tests/integration/analytics-resource-governance.integration.test.ts index a4451da..a0b5094 100644 --- a/tests/integration/analytics-resource-governance.integration.test.ts +++ b/tests/integration/analytics-resource-governance.integration.test.ts @@ -14,10 +14,10 @@ * timed WHILE the bomb is in flight and compared against a declared SLO, * not merely observed to "still work". */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; -import { DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -37,7 +37,7 @@ const DB = `opt_gov_${Date.now().toString(36)}`; */ const UNRELATED_TENANT_SLO_MS = 5000; -const VICTIM: AnalyticsIdentity = { tenant_id: 't_victim', project_id: 'p_main', principal_id: 'usr_v' }; +const VICTIM: AnalyticsIdentity = { tenant_id: "t_victim", project_id: "p_main", principal_id: "usr_v" }; let engine: ClickHouseAnalyticsEngine; @@ -47,40 +47,40 @@ async function chRun( settings: Record = {}, body?: string, ): Promise<{ ok: boolean; code?: string; text: string; ms: number }> { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); const headers: Record = {}; - if (CLICKHOUSE_AUTH) headers['Authorization'] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; const t0 = Date.now(); - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); const text = await res.text(); return { ok: res.ok, code: /Code: (\d+)/.exec(text)?.[1], text, ms: Date.now() - t0 }; } -run('BI-P012 resource governance against a LIVE ClickHouse', () => { +run("BI-P012 resource governance against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_victim: { severity: 'severity' } }, + dimensionCatalogs: { t_victim: { severity: "severity" } }, }); await engine.ensureSchema(); // A small, real dataset for the unrelated tenant. const rows = Array.from({ length: 500 }, (_, i) => JSON.stringify({ - tenant_id: 't_victim', - project_id: 'p_main', - dataset_id: 'ds_v', - metric: 'victim_metric', + tenant_id: "t_victim", + project_id: "p_main", + dataset_id: "ds_v", + metric: "victim_metric", event_id: `v${i}`, - event_time: '2026-09-05 08:00:00.000', + event_time: "2026-09-05 08:00:00.000", value: 1, - dim_names: ['severity'], - dim_values: ['warning'], + dim_names: ["severity"], + dim_values: ["warning"], }), - ).join('\n'); + ).join("\n"); const ins = await chRun(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, {}, rows); expect(ins.ok, ins.text.slice(0, 200)).toBe(true); }, 120_000); @@ -91,51 +91,51 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // ── each named limit, fired individually ──────────────────────────────── - it('max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)', async () => { - const r = await chRun('SELECT count() FROM numbers(1000000000) FORMAT JSON', { - readonly: '1', - max_rows_to_read: '1000', - read_overflow_mode: 'throw', + it("max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)", async () => { + const r = await chRun("SELECT count() FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_rows_to_read: "1000", + read_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('158'); + expect(r.code).toBe("158"); }); - it('max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)', async () => { - const r = await chRun('SELECT sum(number) FROM numbers(1000000000) FORMAT JSON', { - readonly: '1', - max_bytes_to_read: '1024', - read_overflow_mode: 'throw', + it("max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)", async () => { + const r = await chRun("SELECT sum(number) FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_bytes_to_read: "1024", + read_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('307'); + expect(r.code).toBe("307"); }); - it('max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)', async () => { - const r = await chRun('SELECT count() FROM numbers(50000000000) FORMAT JSON', { - readonly: '1', - max_execution_time: '1', - timeout_overflow_mode: 'throw', + it("max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)", async () => { + const r = await chRun("SELECT count() FROM numbers(50000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "1", + timeout_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('159'); + expect(r.code).toBe("159"); expect(r.ms).toBeLessThan(20_000); }); - it('max_result_rows terminates a result bomb and does NOT return a partial result', async () => { - const r = await chRun('SELECT number FROM numbers(100000) FORMAT JSON', { - readonly: '1', - max_result_rows: '10', - result_overflow_mode: 'throw', + it("max_result_rows terminates a result bomb and does NOT return a partial result", async () => { + const r = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('396'); // TOO_MANY_ROWS_OR_BYTES + expect(r.code).toBe("396"); // TOO_MANY_ROWS_OR_BYTES // The dangerous alternative: result_overflow_mode=break returns 10 rows // with HTTP 200 and no indication of truncation. - const broken = await chRun('SELECT number FROM numbers(100000) FORMAT JSON', { - readonly: '1', - max_result_rows: '10', - result_overflow_mode: 'break', + const broken = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "break", }); // MEASURED on ClickHouse 24.8: break mode answers HTTP 200 with 65409 of // 100000 rows (it stops at a block boundary, not at the stated 10) and @@ -147,38 +147,37 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { expect(brokenRows).toBeLessThan(100_000); }); - it('max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)', async () => { - const r = await chRun('SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON', { - readonly: '1', - max_memory_usage: '1000000', - max_execution_time: '30', - }); + it("max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)", async () => { + const r = await chRun( + "SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON", + { readonly: "1", max_memory_usage: "1000000", max_execution_time: "30" }, + ); expect(r.ok).toBe(false); - expect(r.code).toBe('241'); + expect(r.code).toBe("241"); }); - it('readonly=1 means the query text cannot re-raise its own limits', async () => { + it("readonly=1 means the query text cannot re-raise its own limits", async () => { const r = await chRun( - 'SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON', - { readonly: '1', max_rows_to_read: '1000', read_overflow_mode: 'throw' }, + "SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON", + { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw" }, ); expect(r.ok).toBe(false); // 164 = READONLY (changing settings is forbidden); 158 = the limit still // applied. Either proves the caller did not win — but NOT a 200. - expect(['164', '158']).toContain(r.code); + expect(["164", "158"]).toContain(r.code); }); - it('readonly=1 blocks writes through the query surface', async () => { - const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: '1' }); + it("readonly=1 blocks writes through the query surface", async () => { + const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: "1" }); expect(r.ok).toBe(false); }); // ── the half that is usually skipped ──────────────────────────────────── - it('an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight', async () => { + it("an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight", async () => { // Baseline first, so "fast enough" is measured against this server, not // against a number someone hoped for. - const baseline = await engine.executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }); + const baseline = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(baseline.rows[0].value).toBe(500); const baselineMs = baseline.stats.duration_ms; @@ -187,23 +186,23 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // numbers(20e9) in ~5s because it can skip the values entirely. Hashing // every row cannot be short-circuited, so this one really does burn CPU // until the limit kills it. - const bomb = chRun('SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON', { - readonly: '1', - max_execution_time: '8', - timeout_overflow_mode: 'throw', + const bomb = chRun("SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "8", + timeout_overflow_mode: "throw", }); // While it burns, run the unrelated tenant's interactive query repeatedly. const samples: number[] = []; for (let i = 0; i < 5; i++) { - const r = await engine.executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }); + const r = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(r.rows[0].value).toBe(500); // still CORRECT, not just fast samples.push(r.stats.duration_ms); } const bombResult = await bomb; - expect(bombResult.ok, 'the bomb was NOT terminated').toBe(false); - expect(bombResult.code).toBe('159'); + expect(bombResult.ok, "the bomb was NOT terminated").toBe(false); + expect(bombResult.code).toBe("159"); const worst = Math.max(...samples); expect( @@ -214,24 +213,24 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // ── the local admission gate, on the real engine ──────────────────────── - it('the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only', async () => { + it("the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only", async () => { const limit = DEFAULT_BUDGETS.INTERACTIVE.maxConcurrentQueries; // Fire limit+4 concurrently; some must be refused with RATE_LIMITED // rather than all of them reaching the server. const attempts = Array.from({ length: limit + 4 }, () => engine - .executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }) - .then(() => 'ok' as const) - .catch((e: { code?: string }) => e.code ?? 'error'), + .executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }) + .then(() => "ok" as const) + .catch((e: { code?: string }) => e.code ?? "error"), ); const outcomes = await Promise.all(attempts); - expect(outcomes.filter((o) => o === 'RATE_LIMITED').length).toBeGreaterThan(0); - expect(outcomes.filter((o) => o === 'ok').length).toBeGreaterThan(0); + expect(outcomes.filter((o) => o === "RATE_LIMITED").length).toBeGreaterThan(0); + expect(outcomes.filter((o) => o === "ok").length).toBeGreaterThan(0); // A different tenant is unaffected by the noisy one's exhausted slots. const other = await engine.executeMetric( - { tenant_id: 't_quiet', project_id: 'p_main', principal_id: 'usr_q' }, - { metric: 'victim_metric', aggregate: 'count' }, + { tenant_id: "t_quiet", project_id: "p_main", principal_id: "usr_q" }, + { metric: "victim_metric", aggregate: "count" }, ); expect(other.rows[0].value).toBe(0); // its own (empty) data, and no error }, 60_000); diff --git a/tests/integration/analytics-tenant-isolation.integration.test.ts b/tests/integration/analytics-tenant-isolation.integration.test.ts index 4d1f60f..3a1f9c9 100644 --- a/tests/integration/analytics-tenant-isolation.integration.test.ts +++ b/tests/integration/analytics-tenant-isolation.integration.test.ts @@ -26,16 +26,16 @@ * `integration` job sets it from a service container and hard-fails if the * suites skip, so this is not a self-skipping green in practice. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; import { compileMetricQuery, dimensionCatalogFrom, restrictedTenantUserDdl, -} from '@/src/analytics/engine/query-compiler'; -import { ANALYTICS_TABLES, sortingKeySql } from '@/src/analytics/engine/schema'; -import { DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +} from "@/src/analytics/engine/query-compiler"; +import { ANALYTICS_TABLES, sortingKeySql } from "@/src/analytics/engine/schema"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -45,10 +45,10 @@ const DB = `opt_isolation_${Date.now().toString(36)}`; const RESTRICTED_USER = `opt_iso_${Date.now().toString(36)}`; const RESTRICTED_PW = `iso_pw_${Math.random().toString(36).slice(2, 12)}`; -const ALPHA: AnalyticsIdentity = { tenant_id: 't_alpha', project_id: 'p_main', principal_id: 'usr_alpha' }; -const BETA: AnalyticsIdentity = { tenant_id: 't_beta', project_id: 'p_main', principal_id: 'usr_beta' }; +const ALPHA: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; +const BETA: AnalyticsIdentity = { tenant_id: "t_beta", project_id: "p_main", principal_id: "usr_beta" }; -const catalog = dimensionCatalogFrom({ severity: 'severity' }); +const catalog = dimensionCatalogFrom({ severity: "severity" }); const ALL_TABLES: readonly string[] = ANALYTICS_TABLES; /** Raw HTTP against the live server, as a named user. */ @@ -59,37 +59,34 @@ async function chQuery( settings: Record = {}, body?: string, ): Promise<{ ok: boolean; status: number; text: string }> { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); const headers: Record = {}; - if (auth) headers['Authorization'] = `Basic ${Buffer.from(auth).toString('base64')}`; - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (auth) headers["Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); return { ok: res.ok, status: res.status, text: await res.text() }; } function rowsOf(text: string): { grp?: string; value: number }[] { if (!text.trim()) return []; const parsed = JSON.parse(text) as { data?: { grp?: string; value: number | string }[] }; - return (parsed.data ?? []).map((r) => ({ - ...(r.grp !== undefined ? { grp: r.grp } : {}), - value: Number(r.value), - })); + return (parsed.data ?? []).map((r) => ({ ...(r.grp !== undefined ? { grp: r.grp } : {}), value: Number(r.value) })); } let engine: ClickHouseAnalyticsEngine; -run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { +run("BI-P011 tenant isolation against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_alpha: { severity: 'severity' }, t_beta: { severity: 'severity' } }, + dimensionCatalogs: { t_alpha: { severity: "severity" }, t_beta: { severity: "severity" } }, }); const health = await engine.health(); - expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ''}`).toBe(true); + expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ""}`).toBe(true); await engine.ensureSchema(); // Two tenants, deliberately different magnitudes so a leak is unmistakable @@ -98,28 +95,28 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { for (let i = 0; i < 50; i++) { rows.push( JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_shared', - metric: 'rows_ingested', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", event_id: `a${i}`, - event_time: '2026-09-01 10:00:00.000', + event_time: "2026-09-01 10:00:00.000", value: 1, - dim_names: ['severity'], - dim_values: [i % 2 === 0 ? 'warning' : 'critical'], + dim_names: ["severity"], + dim_values: [i % 2 === 0 ? "warning" : "critical"], }), ); rows.push( JSON.stringify({ - tenant_id: 't_beta', - project_id: 'p_main', - dataset_id: 'ds_shared', - metric: 'rows_ingested', + tenant_id: "t_beta", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", event_id: `b${i}`, - event_time: '2026-09-01 10:00:00.000', + event_time: "2026-09-01 10:00:00.000", value: 1000, - dim_names: ['severity'], - dim_values: ['critical'], + dim_names: ["severity"], + dim_values: ["critical"], }), ); } @@ -128,7 +125,7 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { CLICKHOUSE_AUTH, {}, {}, - rows.join('\n'), + rows.join("\n"), ); expect(ins.ok, `seed insert failed: ${ins.text.slice(0, 200)}`).toBe(true); }, 120_000); @@ -137,10 +134,7 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { if (!CLICKHOUSE_URL) return; // Disposable by construction: drop the user, its policies, and the DB. for (const table of ANALYTICS_TABLES) { - await chQuery( - `DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, - CLICKHOUSE_AUTH, - ); + await chQuery(`DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, CLICKHOUSE_AUTH); } await chQuery(`DROP USER IF EXISTS ${RESTRICTED_USER}`, CLICKHOUSE_AUTH); await chQuery(`DROP DATABASE IF EXISTS ${DB}`, CLICKHOUSE_AUTH); @@ -155,89 +149,89 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { const unsafeSql = `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event ` + `WHERE metric = {metric:String} LIMIT 1 FORMAT JSON`; - const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: 'rows_ingested' }); + const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: "rows_ingested" }); expect(res.ok).toBe(true); // 50*1 (alpha) + 50*1000 (beta) = 50050 — both tenants, one result. expect(rowsOf(res.text)[0].value).toBe(50050); }); it("CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000", async () => { - const result = await engine.executeMetric(ALPHA, { metric: 'rows_ingested', aggregate: 'sum' }); + const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum" }); expect(result.rows[0].value).toBe(50); - const beta = await engine.executeMetric(BETA, { metric: 'rows_ingested', aggregate: 'sum' }); + const beta = await engine.executeMetric(BETA, { metric: "rows_ingested", aggregate: "sum" }); expect(beta.rows[0].value).toBe(50_000); }); it("tenant A querying tenant B's dataset id gets ZERO rows, not B's rows", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - dataset_id: 'ds_shared', + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_shared", }); expect(result.rows[0].value).toBe(50); // only alpha's 50, though beta shares the dataset id }); - it('a forged dataset id yields an empty result, never an error that leaks existence', async () => { + it("a forged dataset id yields an empty result, never an error that leaks existence", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - dataset_id: 'ds_that_does_not_exist_or_belongs_to_someone_else', + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_that_does_not_exist_or_belongs_to_someone_else", }); expect(result.rows[0].value).toBe(0); }); it("a tenant with no data at all sees zero, not everyone's aggregate", async () => { - const ghost: AnalyticsIdentity = { tenant_id: 't_gamma', project_id: 'p_main', principal_id: 'usr_g' }; - const result = await engine.executeMetric(ghost, { metric: 'rows_ingested', aggregate: 'count' }); + const ghost: AnalyticsIdentity = { tenant_id: "t_gamma", project_id: "p_main", principal_id: "usr_g" }; + const result = await engine.executeMetric(ghost, { metric: "rows_ingested", aggregate: "count" }); expect(result.rows[0].value).toBe(0); }); it("group-by results carry only this tenant's dimension values", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - group_by: ['severity'], + metric: "rows_ingested", + aggregate: "count", + group_by: ["severity"], }); const total = result.rows.reduce((a, r) => a + r.value, 0); expect(total).toBe(50); // t_beta's rows are all "critical" with value 1000; if any leaked, the // critical bucket would exceed alpha's 25. - const critical = result.rows.find((r) => r.groups?.severity === 'critical'); + const critical = result.rows.find((r) => r.groups?.severity === "critical"); expect(critical?.value).toBe(25); }); - it('SQL/dimension/filter injection through the live path changes nothing', async () => { + it("SQL/dimension/filter injection through the live path changes nothing", async () => { await expect( engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'sum', + metric: "rows_ingested", + aggregate: "sum", group_by: ["severity'; SELECT tenant_id FROM system.tables; --"], }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); // A hostile VALUE reaches the server as a bound parameter and simply // matches nothing. const injected = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', + metric: "rows_ingested", + aggregate: "count", filters: { severity: "warning' OR 1=1 --" }, }); expect(injected.rows[0].value).toBe(0); // And the table is still there afterwards. - const still = await engine.executeMetric(ALPHA, { metric: 'rows_ingested', aggregate: 'count' }); + const still = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count" }); expect(still.rows[0].value).toBe(50); }); - it('raw SQL attempt: the analyst-SQL surface is refused, not silently executed', async () => { + it("raw SQL attempt: the analyst-SQL surface is refused, not silently executed", async () => { await expect( engine.executeQuery(ALPHA, { sql: `SELECT * FROM ${DB}.opt_analytics_event`, dataset_ids: [] }), - ).rejects.toMatchObject({ code: 'CAPABILITY_UNAVAILABLE' }); + ).rejects.toMatchObject({ code: "CAPABILITY_UNAVAILABLE" }); }); // ── layer 3: restricted users + row policies ───────────────────────────── - it('HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant', async () => { + it("HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant", async () => { const res = await chQuery( `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, CLICKHOUSE_AUTH, @@ -246,10 +240,10 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { expect(rowsOf(res.text)[0].value).toBe(2); // t_alpha AND t_beta }); - it('CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant', async () => { + it("CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant", async () => { for (const stmt of restrictedTenantUserDdl({ db: DB, - tenantId: 't_alpha', + tenantId: "t_alpha", userName: RESTRICTED_USER, password: RESTRICTED_PW, tables: ALL_TABLES, @@ -274,24 +268,14 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { expect(rowsOf(sum.text)[0].value).toBe(50); // alpha's total, not 50050 }); - it('the restricted user cannot write, and cannot reach a table it was not granted', async () => { + it("the restricted user cannot write, and cannot reach a table it was not granted", async () => { const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; const write = await chQuery( `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, restrictedAuth, {}, {}, - JSON.stringify({ - tenant_id: 't_beta', - project_id: 'p', - dataset_id: 'd', - metric: 'm', - event_id: 'x', - event_time: '2026-09-01 00:00:00.000', - value: 1, - dim_names: [], - dim_values: [], - }), + JSON.stringify({ tenant_id: "t_beta", project_id: "p", dataset_id: "d", metric: "m", event_id: "x", event_time: "2026-09-01 00:00:00.000", value: 1, dim_names: [], dim_values: [] }), ); expect(write.ok).toBe(false); @@ -305,18 +289,18 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { for (const table of ALL_TABLES) { const res = await chQuery(sortingKeySql(DB, table), CLICKHOUSE_AUTH); expect(res.ok).toBe(true); - expect(res.text.trim().startsWith('tenant_id'), `${table} sorting key: ${res.text.trim()}`).toBe(true); + expect(res.text.trim().startsWith("tenant_id"), `${table} sorting key: ${res.text.trim()}`).toBe(true); } }); - it('the tenant value never appears in the statement text the server receives', async () => { + it("the tenant value never appears in the statement text the server receives", async () => { const compiled = compileMetricQuery( - { tenant_id: 't_alpha', metric: 'rows_ingested', aggregate: 'sum' }, + { tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "sum" }, DB, catalog, DEFAULT_BUDGETS.INTERACTIVE.maxResultRows, ); - expect(compiled.sql).not.toContain('t_alpha'); + expect(compiled.sql).not.toContain("t_alpha"); const res = await chQuery(compiled.sql, CLICKHOUSE_AUTH, compiled.params); expect(res.ok).toBe(true); expect(rowsOf(res.text)[0].value).toBe(50); diff --git a/tests/integration/search-opensearch.integration.test.ts b/tests/integration/search-opensearch.integration.test.ts index 6ab96a4..d8f20e6 100644 --- a/tests/integration/search-opensearch.integration.test.ts +++ b/tests/integration/search-opensearch.integration.test.ts @@ -15,10 +15,10 @@ * - full projection-disaster recovery: delete EVERYTHING, rebuild from * the canonical source, identical results (§68-style, scoped to 0pt) */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { HttpOpenSearchAdapter } from '@/src/search/opensearch-adapter'; -import { SearchService, type DocumentSource } from '@/src/search/search-service'; -import type { SearchDocument } from '@/src/search/search-types'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { HttpOpenSearchAdapter } from "@/src/search/opensearch-adapter"; +import { SearchService, type DocumentSource } from "@/src/search/search-service"; +import type { SearchDocument } from "@/src/search/search-types"; const OPENSEARCH_URL = process.env.OPENSEARCH_URL; const run = OPENSEARCH_URL ? describe : describe.skip; @@ -27,52 +27,52 @@ const INDEX = `itest-${Date.now().toString(36)}`; const DOCS: SearchDocument[] = [ { - doc_id: 'c1', - tenant_id: 't_alpha', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal', 'customer_deliverable'], - source_refs: ['0pt:source:src_bolagsverket'], - payload: { name: 'Volvo Personvagnar AB', org_nr: '5561074967', employees: 20000 }, + doc_id: "c1", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal", "customer_deliverable"], + source_refs: ["0pt:source:src_bolagsverket"], + payload: { name: "Volvo Personvagnar AB", org_nr: "5561074967", employees: 20000 }, }, { - doc_id: 'c2', - tenant_id: 't_alpha', - purpose_tags: ['compliance'], - rights_tags: ['internal'], - payload: { name: 'Volvo Lastvagnar AB', org_nr: '5560135508', employees: 15000 }, + doc_id: "c2", + tenant_id: "t_alpha", + purpose_tags: ["compliance"], + rights_tags: ["internal"], + payload: { name: "Volvo Lastvagnar AB", org_nr: "5560135508", employees: 15000 }, }, { - doc_id: 'c3', - tenant_id: 't_beta', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal'], - payload: { name: 'Volvo Cars Retail', org_nr: '5569999999', employees: 300 }, + doc_id: "c3", + tenant_id: "t_beta", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Volvo Cars Retail", org_nr: "5569999999", employees: 300 }, }, ]; function source(docs: SearchDocument[]): DocumentSource { return { - name: 'itest:canonical-array', + name: "itest:canonical-array", async *iterate() { yield* docs; }, }; } -run('Search capability against a LIVE OpenSearch', () => { +run("Search capability against a LIVE OpenSearch", () => { // The skip-variant still evaluates this callback body, so the constructor // must tolerate the env-less case (the tests themselves never run then). - const adapter = new HttpOpenSearchAdapter({ baseUrl: OPENSEARCH_URL ?? 'http://skipped.invalid' }); + const adapter = new HttpOpenSearchAdapter({ baseUrl: OPENSEARCH_URL ?? "http://skipped.invalid" }); const search = new SearchService(adapter); beforeAll(async () => { const health = await adapter.health(); - expect(['green', 'yellow']).toContain(health.status); + expect(["green", "yellow"]).toContain(health.status); search.registerIndexDefinition({ index_name: INDEX, version: 1, - fields: { name: 'text', org_nr: 'keyword', employees: 'long' }, - default_search_fields: ['name'], + fields: { name: "text", org_nr: "keyword", employees: "long" }, + default_search_fields: ["name"], }); }, 30_000); @@ -82,7 +82,7 @@ run('Search capability against a LIVE OpenSearch', () => { } }); - it('rebuild g1 from canonical source; alias serves it', async () => { + it("rebuild g1 from canonical source; alias serves it", async () => { const result = await search.rebuild(INDEX, source(DOCS)); expect(result.new_generation).toBe(1); expect(result.docs_indexed).toBe(3); @@ -91,65 +91,57 @@ run('Search capability against a LIVE OpenSearch', () => { expect(state.active_generation).toBe(1); }, 60_000); - it('REAL tenant isolation: t_alpha never sees t_beta documents', async () => { + it("REAL tenant isolation: t_alpha never sees t_beta documents", async () => { const alpha = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'volvo', + tenant_id: "t_alpha", + purpose: "company_enrichment", + text: "volvo", }); - expect(alpha.hits.map((h) => h.doc_id)).toEqual(['c1']); // c2 wrong purpose, c3 wrong tenant + expect(alpha.hits.map((h) => h.doc_id)).toEqual(["c1"]); // c2 wrong purpose, c3 wrong tenant const beta = await search.query(INDEX, { - tenant_id: 't_beta', - purpose: 'company_enrichment', - text: 'volvo', + tenant_id: "t_beta", + purpose: "company_enrichment", + text: "volvo", }); - expect(beta.hits.map((h) => h.doc_id)).toEqual(['c3']); + expect(beta.hits.map((h) => h.doc_id)).toEqual(["c3"]); // Cross-tenant attack shape: a filter cannot break out of the tenant term. const attack = await search.query(INDEX, { - tenant_id: 't_beta', - purpose: 'company_enrichment', - filters: { org_nr: '5561074967' }, // t_alpha's org_nr + tenant_id: "t_beta", + purpose: "company_enrichment", + filters: { org_nr: "5561074967" }, // t_alpha's org_nr }); expect(attack.total).toBe(0); }); - it('purpose + rights filtering hold on the real engine', async () => { - const compliance = await search.query(INDEX, { tenant_id: 't_alpha', purpose: 'compliance' }); - expect(compliance.hits.map((h) => h.doc_id)).toEqual(['c2']); + it("purpose + rights filtering hold on the real engine", async () => { + const compliance = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "compliance" }); + expect(compliance.hits.map((h) => h.doc_id)).toEqual(["c2"]); const deliverable = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - rights_tag: 'customer_deliverable', + tenant_id: "t_alpha", + purpose: "company_enrichment", + rights_tag: "customer_deliverable", }); - expect(deliverable.hits.map((h) => h.doc_id)).toEqual(['c1']); + expect(deliverable.hits.map((h) => h.doc_id)).toEqual(["c1"]); }); - it('generation rebuild + switch + rollback (§42)', async () => { + it("generation rebuild + switch + rollback (§42)", async () => { const updated = DOCS.map((d) => - d.doc_id === 'c1' ? { ...d, payload: { ...d.payload, name: 'Volvo Personvagnar AB (uppdaterad)' } } : d, + d.doc_id === "c1" ? { ...d, payload: { ...d.payload, name: "Volvo Personvagnar AB (uppdaterad)" } } : d ); const r2 = await search.rebuild(INDEX, source(updated)); expect(r2.new_generation).toBe(2); - let res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'uppdaterad', - }); + let res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); expect(res.total).toBe(1); // Rollback to g1 — the old generation still exists and serves the alias. await search.switchGeneration(INDEX, 1); - res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'uppdaterad', - }); + res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); expect(res.total).toBe(0); await search.switchGeneration(INDEX, 2); }, 60_000); - it('projection disaster: delete ALL generations, rebuild from canonical, results identical (§68)', async () => { + it("projection disaster: delete ALL generations, rebuild from canonical, results identical (§68)", async () => { for (const idx of await adapter.listIndices(`opt-${INDEX}--g*`)) { await adapter.deleteIndex(idx); } @@ -157,32 +149,24 @@ run('Search capability against a LIVE OpenSearch', () => { const rebuilt = await search.rebuild(INDEX, source(DOCS)); expect(rebuilt.docs_indexed).toBe(3); - const res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'volvo', - }); - expect(res.hits.map((h) => h.doc_id)).toEqual(['c1']); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "volvo" }); + expect(res.hits.map((h) => h.doc_id)).toEqual(["c1"]); }, 60_000); - it('incremental indexDocuments into the active generation', async () => { + it("incremental indexDocuments into the active generation", async () => { const out = await search.indexDocuments(INDEX, [ { - doc_id: 'c4', - tenant_id: 't_alpha', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal'], - payload: { name: 'Polestar Performance AB', org_nr: '5568100000', employees: 2500 }, + doc_id: "c4", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Polestar Performance AB", org_nr: "5568100000", employees: 2500 }, }, ]); expect(out.indexed).toBe(1); const state = await search.generationState(INDEX); await adapter.refresh(search.physicalIndexFor(INDEX, state.active_generation!)); - const res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'polestar', - }); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "polestar" }); expect(res.total).toBe(1); }); });