From c864c8c4177e1356a609eaa6d8a46d4fdac340f4 Mon Sep 17 00:00:00 2001 From: Claude Opus 5 Date: Mon, 7 Sep 2026 03:16:30 +0200 Subject: [PATCH] fix(analytics): revert 18 files reformatted by mistake, and actually apply the Iceberg fix a previous message claimed MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Two corrections, both of my own errors. 1. SCOPE CREEP, NOW REVERTED. Running `prettier --write` on whole directories reformatted 18 files this branch had no business touching — the entire merged BI-P002..P012 core (arrow/, domain/, engine/, repositories/, five analytics integration suites) and, absurdly, search-opensearch.integration.test.ts, which has nothing to do with this work. The changes were purely stylistic quote churn, but they added ~1,400 lines of noise to the diff, made the PR harder to review honestly, and invited conflicts with the four other lanes working in this estate. All 18 are now byte-identical to origin/main again (verified with `git diff origin/main --name-only`, which reports nothing for them). Formatting is now applied only to the files this branch actually owns. 2. A COMMIT MESSAGE THAT DESCRIBED A FIX THAT WAS NOT IN THE CODE. Commit 3ee0230 said the "does NOT ship the Iceberg table" assertion had been moved from a filesystem check to `git ls-files`. It had not. The patch anchor used double quotes, prettier had already converted that file to single quotes, so the replacement silently matched nothing and the script printed its success line anyway. The suite then passed — but only because I had just deleted the fixture directory, so the OLD filesystem assertion happened to be true. That is precisely the failure mode this branch keeps writing tests against: a green run that means something different from what it appears to mean. The fix is now genuinely applied and verified the way it should have been the first time — by running the suite WITH the Iceberg fixture present on disk, which is the case that previously failed. It passes, and it now also asserts that the parquet/csv fixtures ARE tracked, so a version shipping no fixture at all would fail rather than pass vacuously. 957 tests pass with ClickHouse, Postgres and DuckDB live. typecheck, eslint and prettier clean on the files this branch owns. Co-Authored-By: Claude Opus 5 --- evidence/bi-p014/duckdb-sandbox.json | 2 +- evidence/bi-p017/analyst-sql-live-bypass.json | 8 +- .../bi-p019-p021/cancellation-and-budget.json | 12 +- evidence/bi-p025/native-quality-live.json | 2 +- src/analytics/arrow/flatbuffer-builder.ts | 4 +- src/analytics/arrow/ipc.ts | 118 ++++++------ src/analytics/clickhouse-analytics.ts | 71 +++---- src/analytics/domain/contract.ts | 176 +++++++----------- src/analytics/domain/entities.ts | 170 ++++++++--------- src/analytics/domain/tiers.ts | 24 +-- src/analytics/engine/clickhouse-engine.ts | 117 ++++++------ src/analytics/engine/query-compiler.ts | 96 +++++----- src/analytics/engine/resource-governance.ts | 34 ++-- src/analytics/engine/schema.ts | 11 +- src/analytics/engine/siax-analytics-engine.ts | 27 ++- .../analytics-domain-repository.ts | 115 +++--------- tests/analytics/duckdb-provider.test.ts | 16 +- ...ytics-arrow-clickhouse.integration.test.ts | 101 +++++----- .../analytics-clickhouse.integration.test.ts | 94 ++++------ ...tics-engine-clickhouse.integration.test.ts | 151 ++++++++------- ...cs-resource-governance.integration.test.ts | 153 ++++++++------- ...ytics-tenant-isolation.integration.test.ts | 150 +++++++-------- .../search-opensearch.integration.test.ts | 136 ++++++-------- 23 files changed, 800 insertions(+), 988 deletions(-) diff --git a/evidence/bi-p014/duckdb-sandbox.json b/evidence/bi-p014/duckdb-sandbox.json index e8a6078..7366789 100644 --- a/evidence/bi-p014/duckdb-sandbox.json +++ b/evidence/bi-p014/duckdb-sandbox.json @@ -1,6 +1,6 @@ { "duckdb_version": "1.5.5", - "recorded_at": "2026-09-07T01:07:03.476Z", + "recorded_at": "2026-09-07T01:15:40.775Z", "parquet": { "SE": 394.5, "NO": 105.5 diff --git a/evidence/bi-p017/analyst-sql-live-bypass.json b/evidence/bi-p017/analyst-sql-live-bypass.json index e99840f..b92328a 100644 --- a/evidence/bi-p017/analyst-sql-live-bypass.json +++ b/evidence/bi-p017/analyst-sql-live-bypass.json @@ -2,7 +2,7 @@ "prompt": "BI-P017", "claim": "Every bypass attempt, executed directly against a live ClickHouse as the tenant's restricted user WITH THE STATEMENT GUARD REMOVED, fails to return a foreign-tenant row, an ungranted-table row, or server metadata.", "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:07.196Z", + "recorded_at": "2026-09-07T01:15:43.364Z", "base_table_rows_both_tenants": 100, "tenant_view_rows_alpha": 50, "attempts_total": 64, @@ -308,7 +308,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: function \"generaterandom\" is not permitted in an analyst session", - "engine_response_head": "Code: 158. DB::Exception: Limit for rows or bytes to read exceeded, max rows: 100.00 thousand, current rows: 157.89 thousand: While executing GenerateRandom. (T" + "engine_response_head": "Code: 158. DB::Exception: Limit for rows or bytes to read exceeded, max rows: 100.00 thousand, current rows: 315.79 thousand: While executing GenerateRandom. (T" }, { "id": "tf-merge", @@ -572,7 +572,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", - "engine_response_head": "events sales events sales ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 A" + "engine_response_head": "ALPHA_ROW_0 ALPHA_ROW_1 ALPHA_ROW_2 ALPHA_ROW_3 ALPHA_ROW_4 ALPHA_ROW_5 ALPHA_ROW_6 ALPHA_ROW_7 ALPHA_ROW_8 ALPHA_ROW_9 ALPHA_ROW_10 ALPHA_ROW_11 ALPHA_ROW_12 A" }, { "id": "unterminated-comment", @@ -656,7 +656,7 @@ "leaked_ungranted_table_row": false, "leaked_server_metadata": false, "guard_verdict_for_reference": "refused: [AUTHORIZATION_DENIED] 0pt.analytics.sql: \"SYSTEM\" is not permitted in a read-only analyst session", - "engine_response_head": "opt_analytics events 30087fa6-7696-4216-abc7-507d8ab01e10 MergeTree 0 ['/var/lib/clickhouse/store/300/30087fa6-7696-4216-abc7-507d8ab01e10/'] /var/lib/clickhous" + "engine_response_head": "opt_analytics events 204f5e8f-9134-417f-b7f5-569500c15059 MergeTree 0 ['/var/lib/clickhouse/store/204/204f5e8f-9134-417f-b7f5-569500c15059/'] /var/lib/clickhous" }, { "id": "describe-subquery", diff --git a/evidence/bi-p019-p021/cancellation-and-budget.json b/evidence/bi-p019-p021/cancellation-and-budget.json index e8324a6..809350f 100644 --- a/evidence/bi-p019-p021/cancellation-and-budget.json +++ b/evidence/bi-p019-p021/cancellation-and-budget.json @@ -1,20 +1,20 @@ { "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:02.373Z", + "recorded_at": "2026-09-07T01:15:40.234Z", "counterfactual_still_running_after_2s": true, "cancellation": { - "rows_read_before_cancel": 4955255022, - "elapsed_ms_to_verified_stop": 110, + "rows_read_before_cancel": 4227318261, + "elapsed_ms_to_verified_stop": 162, "verified_stopped": true, "clickhouse_query_log_type": "ExceptionWhileProcessing", "clickhouse_exception_code": 394 }, "budget_in_flight": { "budget_max_rows_scanned": 1000000, - "observed_rows_read_at_cancel": 215980518, - "observed_elapsed_ms_at_cancel": 60, + "observed_rows_read_at_cancel": 153318696, + "observed_elapsed_ms_at_cancel": 55, "polls_before_breach": 1, - "overshoot_factor": 216, + "overshoot_factor": 153, "overshoot_note": "in-flight enforcement is granular to the poll interval; the query overshot the window by this factor before the poll that caught it. A bound, not a hard cap.", "cancellation_verified": true, "clickhouse_exception_code": 394, diff --git a/evidence/bi-p025/native-quality-live.json b/evidence/bi-p025/native-quality-live.json index 67a17a3..c2d296b 100644 --- a/evidence/bi-p025/native-quality-live.json +++ b/evidence/bi-p025/native-quality-live.json @@ -1,6 +1,6 @@ { "clickhouse_version": "24.8.14.39", - "recorded_at": "2026-09-07T01:07:17.084Z", + "recorded_at": "2026-09-07T01:15:45.657Z", "not_null": { "evaluated": 118, "violations": 5 diff --git a/src/analytics/arrow/flatbuffer-builder.ts b/src/analytics/arrow/flatbuffer-builder.ts index 46d3695..3185803 100644 --- a/src/analytics/arrow/flatbuffer-builder.ts +++ b/src/analytics/arrow/flatbuffer-builder.ts @@ -148,14 +148,14 @@ export class FlatBufferBuilder { } startObject(numFields: number): void { - if (this.inObject) throw new Error('nested startObject'); + if (this.inObject) throw new Error("nested startObject"); this.vtable = new Array(numFields).fill(0); this.objectStart = this.used; this.inObject = true; } endObject(): Offset { - if (!this.inObject) throw new Error('endObject outside an object'); + if (!this.inObject) throw new Error("endObject outside an object"); this.prep(SIZEOF_INT); this.writeInt32(0); // soffset placeholder, patched below const objectOffset = this.used; diff --git a/src/analytics/arrow/ipc.ts b/src/analytics/arrow/ipc.ts index 2288a7f..d4a693c 100644 --- a/src/analytics/arrow/ipc.ts +++ b/src/analytics/arrow/ipc.ts @@ -29,8 +29,8 @@ * padded so the body starts 8-byte aligned, then the body buffers. * Terminated by 0xFFFFFFFF 0x00000000. */ -import { FlatBufferBuilder, type Offset } from './flatbuffer-builder'; -import { SiaxError } from '../../platform/errors'; +import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder"; +import { SiaxError } from "../../platform/errors"; // ── FlatBuffer enum values from org.apache.arrow.flatbuf ──────────────────── @@ -51,13 +51,13 @@ const PRECISION_DOUBLE = 2; // every 64-bit constant is built with BigInt(...) instead. Behaviour is // identical — this is a compiler-target accommodation, not a semantic choice. const BIG_ZERO = BigInt(0); -const INT64_MAX = BigInt('9223372036854775807'); -const INT64_MIN = BigInt('-9223372036854775808'); +const INT64_MAX = BigInt("9223372036854775807"); +const INT64_MIN = BigInt("-9223372036854775808"); const TIME_UNIT_MILLISECOND = 1; // ── public column model ──────────────────────────────────────────────────── -export type ArrowColumnType = 'int64' | 'float64' | 'utf8' | 'bool' | 'timestamp_ms'; +export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms"; export interface ArrowColumn { name: string; @@ -78,39 +78,39 @@ export const ARROW_TYPE_SUPPORT: { note: string; }[] = [ { - clickhouse: 'Int64 / UInt32 / Int32', - arrow: 'int64', - duckdb: 'BIGINT', - python: 'int', - note: 'exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped', + clickhouse: "Int64 / UInt32 / Int32", + arrow: "int64", + duckdb: "BIGINT", + python: "int", + note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped", }, { - clickhouse: 'Float64', - arrow: 'float64', - duckdb: 'DOUBLE', - python: 'float', - note: 'exact IEEE-754 round trip', + clickhouse: "Float64", + arrow: "float64", + duckdb: "DOUBLE", + python: "float", + note: "exact IEEE-754 round trip", }, { - clickhouse: 'String / LowCardinality(String)', - arrow: 'utf8', - duckdb: 'VARCHAR', - python: 'str', - note: 'LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not', + clickhouse: "String / LowCardinality(String)", + arrow: "utf8", + duckdb: "VARCHAR", + python: "str", + note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not", }, { - clickhouse: 'Bool / UInt8(0|1)', - arrow: 'bool', - duckdb: 'BOOLEAN', - python: 'bool', - note: 'bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64', + clickhouse: "Bool / UInt8(0|1)", + arrow: "bool", + duckdb: "BOOLEAN", + python: "bool", + note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64", }, { clickhouse: "DateTime64(3,'UTC')", - arrow: 'timestamp_ms', - duckdb: 'TIMESTAMP', - python: 'datetime', - note: 'millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set', + arrow: "timestamp_ms", + duckdb: "TIMESTAMP", + python: "datetime", + note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set", }, ]; @@ -118,7 +118,7 @@ export const ARROW_TYPE_SUPPORT: { /** ClickHouse output format producing an Arrow IPC *stream*. */ export function clickhouseArrowStreamFormat(): string { - return 'ArrowStream'; + return "ArrowStream"; } /** @@ -129,8 +129,8 @@ export function clickhouseArrowStreamFormat(): string { */ export function arrowStreamSettings(): Record { return { - output_format_arrow_string_as_string: '1', - output_format_arrow_low_cardinality_as_dictionary: '0', + output_format_arrow_string_as_string: "1", + output_format_arrow_low_cardinality_as_dictionary: "0", }; } @@ -165,24 +165,20 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: const n = col.values.length; switch (col.type) { - case 'int64': { + case "int64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; - const b = typeof v === 'bigint' ? v : BigInt(Math.trunc(Number(v))); + const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v))); if (b > INT64_MAX || b < INT64_MIN) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `value out of int64 range in column "${col.name}"`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`); } dv.setBigInt64(i * 8, b, true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'float64': { + case "float64": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { @@ -191,17 +187,17 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'timestamp_ms': { + case "timestamp_ms": { const data = new Uint8Array(alignTo8(n * 8)); const dv = new DataView(data.buffer); col.values.forEach((v, i) => { if (v === null || v === undefined) return; - const ms = v instanceof Date ? v.getTime() : typeof v === 'bigint' ? Number(v) : Number(v); + const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v); dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true); }); return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount }; } - case 'bool': { + case "bool": { const byteLen = Math.ceil(n / 8); const data = new Uint8Array(alignTo8(byteLen)); col.values.forEach((v, i) => { @@ -210,7 +206,7 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: }); return { buffers: [validity, { bytes: data, length: byteLen }], nullCount }; } - case 'utf8': { + case "utf8": { const encoded = col.values.map((v) => v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)), ); @@ -232,36 +228,36 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount: } default: { const never: never = col.type; - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } { switch (type) { - case 'int64': { + case "int64": { b.startObject(2); b.addInt32(0, 64, 0); // bitWidth b.addInt8(1, 1, 0); // is_signed return { typeType: TYPE_INT, offset: b.endObject() }; } - case 'float64': { + case "float64": { b.startObject(1); b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() }; } - case 'utf8': { + case "utf8": { b.startObject(0); return { typeType: TYPE_UTF8, offset: b.endObject() }; } - case 'bool': { + case "bool": { b.startObject(0); return { typeType: TYPE_BOOL, offset: b.endObject() }; } - case 'timestamp_ms': { + case "timestamp_ms": { // Build the timezone string BEFORE startObject — FlatBuffers forbids // creating a nested object while a table is open. - const tz = b.createString('UTC'); + const tz = b.createString("UTC"); b.startObject(2); b.addInt16(0, TIME_UNIT_MILLISECOND, 0); b.addOffset(1, tz, 0); @@ -269,7 +265,7 @@ function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType } default: { const never: never = type; - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`); } } } @@ -365,14 +361,14 @@ function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array { */ export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array { if (columns.length === 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'an Arrow stream needs at least one column'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column"); } const rowCount = columns[0].values.length; for (const c of columns) { if (c.values.length !== rowCount) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `column "${c.name}" has ${c.values.length} values, expected ${rowCount}`, ); } @@ -500,11 +496,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { while (pos + 8 <= bytes.byteLength) { const continuation = dv.getUint32(pos, true); if (continuation !== 0xffffffff) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'not an Arrow IPC stream (missing continuation marker)', - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)"); } const metaLen = dv.getInt32(pos + 4, true); if (metaLen === 0) break; // end of stream @@ -521,7 +513,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo { fields = []; for (let i = 0; i < n; i++) { const f = schema.vectorElementTable(1, i); - fields.push({ name: f.readString(0) ?? '', typeType: f.readInt8(2, 0) }); + fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) }); } } } else if (headerType === MESSAGE_HEADER_RECORD_BATCH) { @@ -546,8 +538,8 @@ export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows: const info = decodeArrowStream(bytes); if (info.rowCount > maxResultRows) { throw new SiaxError( - 'QUOTA_EXCEEDED', - '0pt.analytics', + "QUOTA_EXCEEDED", + "0pt.analytics", `Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`, ); } diff --git a/src/analytics/clickhouse-analytics.ts b/src/analytics/clickhouse-analytics.ts index 26fb574..c4fafc2 100644 --- a/src/analytics/clickhouse-analytics.ts +++ b/src/analytics/clickhouse-analytics.ts @@ -25,7 +25,7 @@ * * Retention: tables are created WITH TTL (event_time + INTERVAL n DAY). */ -import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../platform/errors'; +import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors"; export interface ClickHouseAnalyticsOptions { baseUrl: string; @@ -52,7 +52,7 @@ export interface MetricQuery { tenant_id: string; dataset_id?: string; metric: string; - aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count'; + aggregate: "sum" | "avg" | "min" | "max" | "count"; /** Group by one dimension key (validated identifier). */ group_by?: string; from_time?: string; @@ -77,9 +77,9 @@ export class ClickHouseAnalytics { private readonly maxResultRows: number; constructor(opts: ClickHouseAnalyticsOptions) { - this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); - this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined; - this.database = opts.database ?? 'opt_analytics'; + this.baseUrl = opts.baseUrl.replace(/\/+$/, ""); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined; + this.database = opts.database ?? "opt_analytics"; this.timeoutMs = opts.timeoutMs ?? 30_000; this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10; this.maxResultRows = opts.maxResultRows ?? 10_000; @@ -100,7 +100,7 @@ export class ClickHouseAnalytics { ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (tenant_id, metric, dataset_id, event_time) - TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`, + TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY` ); } @@ -109,14 +109,10 @@ export class ClickHouseAnalytics { if (events.length === 0) return { ingested: 0 }; for (const e of events) { if (!e.tenant_id?.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every analytics event needs tenant_id'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id"); } if (!Number.isFinite(e.value)) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `metric value for "${e.metric}" must be finite`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`); } } const rows = events @@ -126,11 +122,11 @@ export class ClickHouseAnalytics { dataset_id: e.dataset_id, metric: e.metric, value: e.value, - event_time: (e.event_time ?? new Date().toISOString()).replace('T', ' ').replace(/Z$/, ''), + event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""), dimensions: e.dimensions ?? {}, - }), + }) ) - .join('\n'); + .join("\n"); await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows); return { ingested: events.length }; } @@ -138,40 +134,41 @@ export class ClickHouseAnalytics { /** Closed metric-query DSL — compiled with a FORCED tenant predicate. */ async queryMetric(q: MetricQuery): Promise { if (!q.tenant_id?.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every metric query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query"); } if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid group_by identifier`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`); } - const agg = q.aggregate === 'count' ? 'count()' : `${q.aggregate}(value)`; + const agg = + q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`; const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined; const limit = Math.min(q.limit ?? 1000, this.maxResultRows); // Parameterized query — values travel as HTTP params, never spliced. - const conditions = ['tenant_id = {tenant:String}', 'metric = {metric:String}']; + const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"]; const params: Record = { param_tenant: q.tenant_id, param_metric: q.metric }; if (q.dataset_id) { - conditions.push('dataset_id = {dataset:String}'); + conditions.push("dataset_id = {dataset:String}"); params.param_dataset = q.dataset_id; } if (q.from_time) { - conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})'); + conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})"); params.param_fromTime = q.from_time; } if (q.to_time) { - conditions.push('event_time < parseDateTime64BestEffort({toTime:String})'); + conditions.push("event_time < parseDateTime64BestEffort({toTime:String})"); params.param_toTime = q.to_time; } const sql = - `SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ''}${agg} AS value ` + + `SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` + `FROM ${this.database}.opt_metric_events ` + - `WHERE ${conditions.join(' AND ')} ` + - (groupExpr ? `GROUP BY grp ORDER BY value DESC ` : '') + + `WHERE ${conditions.join(" AND ")} ` + + (groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") + `LIMIT ${limit} FORMAT JSON`; const json = (await this.exec(sql, undefined, { - readonly: '1', + readonly: "1", max_execution_time: String(this.maxExecutionTimeSec), max_result_rows: String(this.maxResultRows), ...params, @@ -185,7 +182,7 @@ export class ClickHouseAnalytics { async health(): Promise { try { - await this.exec('SELECT 1 FORMAT JSON'); + await this.exec("SELECT 1 FORMAT JSON"); return true; } catch { return false; @@ -193,24 +190,28 @@ export class ClickHouseAnalytics { } // ----------------------------------------------------------------------- - private async exec(sql: string, body?: string, settings?: Record): Promise { + private async exec( + sql: string, + body?: string, + settings?: Record + ): Promise { const url = new URL(this.baseUrl); - url.searchParams.set('query', sql); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v); const headers: Record = {}; - if (this.authHeader) headers['Authorization'] = this.authHeader; + if (this.authHeader) headers["Authorization"] = this.authHeader; let res: Response; try { res = await fetch(url.toString(), { - method: 'POST', + method: "POST", headers, - body: body ?? '', + body: body ?? "", signal: AbortSignal.timeout(this.timeoutMs), }); } catch (cause) { - throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause }); } const text = await res.text(); if (!res.ok) { @@ -218,8 +219,8 @@ export class ClickHouseAnalytics { const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status); throw new SiaxError( siaxErrorCodeForUpstreamStatus(res.status), - 'clickhouse', - `query failed (CH code ${code})`, + "clickhouse", + `query failed (CH code ${code})` ); } if (!text) return undefined; diff --git a/src/analytics/domain/contract.ts b/src/analytics/domain/contract.ts index 7453c31..0a3ba10 100644 --- a/src/analytics/domain/contract.ts +++ b/src/analytics/domain/contract.ts @@ -18,10 +18,10 @@ * shipped under a minor bump is exactly the failure this gate exists to * catch, so it is reported as a violation rather than tolerated. */ -import crypto from 'node:crypto'; -import { SiaxError } from '../../platform/errors'; -import type { AnalyticsFieldType, FieldRole, PiiClassification } from './entities'; -import type { AnalyticsTier } from './tiers'; +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities"; +import type { AnalyticsTier } from "./tiers"; export interface ContractField { name: string; @@ -38,7 +38,7 @@ export interface ContractQualityRule { rule_id: string; kind: string; field_name?: string; - severity: 'warn' | 'block'; + severity: "warn" | "block"; } /** BI-P004's named contents, one field each. */ @@ -77,18 +77,14 @@ const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/; export function parseSemVer(v: string): SemVer { const m = SEMVER_RE.exec(v); if (!m) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `contract version "${v}" is not major.minor.patch`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`); } return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) }; } // ── change classification ────────────────────────────────────────────────── -export type ChangeSeverity = 'compatible' | 'breaking'; +export type ChangeSeverity = "compatible" | "breaking"; export interface ContractChange { severity: ChangeSeverity; @@ -104,7 +100,7 @@ export interface ContractChange { * Everything absent from this table is breaking. */ const SAFE_TYPE_WIDENING: Partial> = { - date: ['timestamp'], + date: ["timestamp"], }; function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean { @@ -136,9 +132,9 @@ function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] { if (previous.dataset_id !== next.dataset_id) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'contract compatibility is only defined between versions of the SAME dataset', + "INVALID_REQUEST", + "0pt.analytics", + "contract compatibility is only defined between versions of the SAME dataset", ); } const changes: ContractChange[] = []; @@ -148,49 +144,44 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): for (const [name, pf] of prevFields) { const nf = nextFields.get(name); if (!nf) { - changes.push({ - severity: 'breaking', - code: 'field_removed', - field: name, - detail: `field "${name}" removed`, - }); + changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` }); continue; } if (!typeChangeIsCompatible(pf.type, nf.type)) { changes.push({ - severity: 'breaking', - code: 'field_type_changed', + severity: "breaking", + code: "field_type_changed", field: name, detail: `field "${name}" type ${pf.type} -> ${nf.type}`, }); } else if (pf.type !== nf.type) { changes.push({ - severity: 'compatible', - code: 'field_type_widened', + severity: "compatible", + code: "field_type_widened", field: name, detail: `field "${name}" widened ${pf.type} -> ${nf.type}`, }); } if (!pf.nullable && nf.nullable) { changes.push({ - severity: 'breaking', - code: 'field_became_nullable', + severity: "breaking", + code: "field_became_nullable", field: name, detail: `field "${name}" became nullable`, }); } if (pf.nullable && !nf.nullable) { changes.push({ - severity: 'compatible', - code: 'field_became_required', + severity: "compatible", + code: "field_became_required", field: name, detail: `field "${name}" is no longer nullable`, }); } if (pf.role !== nf.role) { changes.push({ - severity: 'breaking', - code: 'field_role_changed', + severity: "breaking", + code: "field_role_changed", field: name, detail: `field "${name}" role ${pf.role} -> ${nf.role}`, }); @@ -201,8 +192,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): if (prevFields.has(name)) continue; if (nf.nullable) { changes.push({ - severity: 'compatible', - code: 'nullable_field_added', + severity: "compatible", + code: "nullable_field_added", field: name, detail: `nullable field "${name}" added`, }); @@ -210,8 +201,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): // A required new field breaks producers, and breaks readers that // reconstruct rows (they now have no value to supply). changes.push({ - severity: 'breaking', - code: 'required_field_added', + severity: "breaking", + code: "required_field_added", field: name, detail: `required (non-nullable) field "${name}" added`, }); @@ -223,75 +214,51 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v); if (!idsEqual) { changes.push({ - severity: 'breaking', - code: 'primary_identifier_changed', - detail: `primary identifier [${previous.primary_identifiers.join(',')}] -> [${next.primary_identifiers.join(',')}]`, + severity: "breaking", + code: "primary_identifier_changed", + detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`, }); } for (const d of previous.dimensions) { if (!next.dimensions.includes(d)) { - changes.push({ - severity: 'breaking', - code: 'dimension_removed', - field: d, - detail: `dimension "${d}" removed`, - }); + changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` }); } } for (const d of next.dimensions) { if (!previous.dimensions.includes(d)) { - changes.push({ - severity: 'compatible', - code: 'dimension_added', - field: d, - detail: `dimension "${d}" added`, - }); + changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` }); } } for (const m of previous.measures) { if (!next.measures.includes(m)) { - changes.push({ - severity: 'breaking', - code: 'measure_removed', - field: m, - detail: `measure "${m}" removed`, - }); + changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` }); } } for (const m of next.measures) { if (!previous.measures.includes(m)) { - changes.push({ - severity: 'compatible', - code: 'measure_added', - field: m, - detail: `measure "${m}" added`, - }); + changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` }); } } if (next.freshness_slo_seconds > previous.freshness_slo_seconds) { changes.push({ - severity: 'breaking', - code: 'freshness_slo_loosened', + severity: "breaking", + code: "freshness_slo_loosened", detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`, }); } if (next.retention_days < previous.retention_days) { changes.push({ - severity: 'breaking', - code: 'retention_shortened', + severity: "breaking", + code: "retention_shortened", detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`, }); } for (const p of previous.allowed_purposes) { if (!next.allowed_purposes.includes(p)) { - changes.push({ - severity: 'breaking', - code: 'purpose_removed', - detail: `allowed purpose "${p}" removed`, - }); + changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` }); } } @@ -303,8 +270,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): }; if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) { changes.push({ - severity: 'breaking', - code: 'pii_classification_raised', + severity: "breaking", + code: "pii_classification_raised", detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`, }); } @@ -312,16 +279,16 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r])); for (const [id, pr] of prevRules) { const nr = next.quality_rules.find((r) => r.rule_id === id); - if (!nr && pr.severity === 'block') { + if (!nr && pr.severity === "block") { changes.push({ - severity: 'breaking', - code: 'blocking_quality_rule_removed', + severity: "breaking", + code: "blocking_quality_rule_removed", detail: `blocking quality rule "${id}" removed`, }); - } else if (nr && pr.severity === 'block' && nr.severity === 'warn') { + } else if (nr && pr.severity === "block" && nr.severity === "warn") { changes.push({ - severity: 'breaking', - code: 'blocking_quality_rule_downgraded', + severity: "breaking", + code: "blocking_quality_rule_downgraded", detail: `quality rule "${id}" downgraded block -> warn`, }); } @@ -329,8 +296,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract): if (previous.tier !== next.tier) { changes.push({ - severity: 'breaking', - code: 'tier_changed', + severity: "breaking", + code: "tier_changed", detail: `tier ${previous.tier} -> ${next.tier}`, }); } @@ -367,7 +334,7 @@ export function checkContractCompatibility( const pv = parseSemVer(previous.version); const nv = parseSemVer(next.version); const changes = diffContracts(previous, next); - const breaking = changes.filter((c) => c.severity === 'breaking'); + const breaking = changes.filter((c) => c.severity === "breaking"); const forward = nv.major > pv.major || @@ -397,7 +364,7 @@ export function checkContractCompatibility( reason: `${breaking.length} breaking change(s) require a MAJOR version bump ` + `(${previous.version} -> ${next.version} is not one): ` + - breaking.map((b) => b.code).join(', '), + breaking.map((b) => b.code).join(", "), }; } @@ -412,17 +379,10 @@ export function checkContractCompatibility( } /** Throwing form for call sites that treat incompatibility as a failure. */ -export function assertContractCompatible( - previous: DatasetContract, - next: DatasetContract, -): CompatibilityVerdict { +export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict { const verdict = checkContractCompatibility(previous, next); if (!verdict.compatible) { - throw new SiaxError( - 'CONFLICT', - '0pt.analytics', - verdict.reason ?? 'dataset contract compatibility gate failed', - ); + throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed"); } return verdict; } @@ -435,28 +395,24 @@ export function assertContractCompatible( export function validateContract(contract: DatasetContract): void { parseSemVer(contract.version); if (!contract.dataset_id.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'contract needs a dataset_id'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id"); } if (contract.fields.length === 0) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - `contract ${contract.dataset_id} declares no fields`, - ); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`); } const names = new Set(); for (const f of contract.fields) { if (names.has(f.name)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `duplicate field "${f.name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`); } names.add(f.name); } - for (const label of ['primary_identifiers', 'dimensions', 'measures'] as const) { + for (const label of ["primary_identifiers", "dimensions", "measures"] as const) { for (const n of contract[label]) { if (!names.has(n)) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `${label} references unknown field "${n}" in ${contract.dataset_id}`, ); } @@ -464,23 +420,23 @@ export function validateContract(contract: DatasetContract): void { } if (contract.primary_identifiers.length === 0) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `contract ${contract.dataset_id} declares no primary identifier`, ); } if (contract.allowed_purposes.length === 0) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`, ); } if (contract.freshness_slo_seconds <= 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'freshness_slo_seconds must be positive'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive"); } if (contract.retention_days < 0) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'retention_days must not be negative'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative"); } } @@ -501,7 +457,7 @@ export function contractHash(contract: DatasetContract): string { nullable: f.nullable, role: f.role, primary_identifier: f.primary_identifier ?? false, - classification: f.classification ?? 'none', + classification: f.classification ?? "none", })) .sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)), primary_identifiers: contract.primary_identifiers, @@ -514,5 +470,5 @@ export function contractHash(contract: DatasetContract): string { quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)), allowed_purposes: [...contract.allowed_purposes].sort(), }; - return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex'); + return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex"); } diff --git a/src/analytics/domain/entities.ts b/src/analytics/domain/entities.ts index b96faba..29d0017 100644 --- a/src/analytics/domain/entities.ts +++ b/src/analytics/domain/entities.ts @@ -35,7 +35,7 @@ * defines its own copies of these tables and never receives engine * credentials. */ -import type { AnalyticsTier } from './tiers'; +import type { AnalyticsTier } from "./tiers"; /** Every tenant-owned analytics record. */ export interface TenantScoped { @@ -46,28 +46,28 @@ export interface TenantScoped { // ── field/typing vocabulary ──────────────────────────────────────────────── export const ANALYTICS_FIELD_TYPES = [ - 'string', - 'int64', - 'float64', - 'bool', - 'date', - 'timestamp', - 'decimal', - 'json', + "string", + "int64", + "float64", + "bool", + "date", + "timestamp", + "decimal", + "json", ] as const; export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number]; /** A field's analytical role. Measures aggregate; dimensions group. */ -export const FIELD_ROLES = ['identifier', 'dimension', 'measure', 'attribute'] as const; +export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const; export type FieldRole = (typeof FIELD_ROLES)[number]; /** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */ -export const PII_CLASSIFICATIONS = ['none', 'pseudonymous', 'personal', 'sensitive'] as const; +export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const; export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number]; // ── 1. AnalyticsDataset ──────────────────────────────────────────────────── -export const DATASET_STATUSES = ['draft', 'active', 'deprecated', 'retired'] as const; +export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const; export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number]; /** @@ -137,7 +137,7 @@ export interface DatasetField extends TenantScoped { // ── 4. DatasetGrant ──────────────────────────────────────────────────────── -export const GRANT_LEVELS = ['read', 'query', 'export', 'admin'] as const; +export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const; export type GrantLevel = (typeof GRANT_LEVELS)[number]; /** @@ -150,7 +150,7 @@ export interface DatasetGrant extends TenantScoped { dataset_id: string; /** Subject: a CL0UD principal, role, or BI application binding id. */ subject_id: string; - subject_type: 'user' | 'role' | 'service' | 'bi_application'; + subject_type: "user" | "role" | "service" | "bi_application"; level: GrantLevel; allowed_purposes: string[]; expires_at?: string; @@ -161,14 +161,14 @@ export interface DatasetGrant extends TenantScoped { // ── 5. MetricDefinition ──────────────────────────────────────────────────── export const METRIC_AGGREGATIONS = [ - 'sum', - 'avg', - 'min', - 'max', - 'count', - 'count_distinct', - 'ratio', - 'derived', + "sum", + "avg", + "min", + "max", + "count", + "count_distinct", + "ratio", + "derived", ] as const; export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number]; @@ -244,7 +244,7 @@ export interface SemanticModel extends TenantScoped { // ── 9. AnalyticsQuery ────────────────────────────────────────────────────── -export const QUERY_SURFACES = ['metric', 'sql', 'projection'] as const; +export const QUERY_SURFACES = ["metric", "sql", "projection"] as const; export type QuerySurface = (typeof QUERY_SURFACES)[number]; /** @@ -268,12 +268,12 @@ export interface AnalyticsQuery extends TenantScoped { // ── 10. QueryExecution ───────────────────────────────────────────────────── export const EXECUTION_STATES = [ - 'queued', - 'running', - 'succeeded', - 'failed', - 'cancelled', - 'rejected', + "queued", + "running", + "succeeded", + "failed", + "cancelled", + "rejected", ] as const; export type ExecutionState = (typeof EXECUTION_STATES)[number]; @@ -299,7 +299,7 @@ export interface QueryExecution extends TenantScoped { // ── 11. QueryResult ──────────────────────────────────────────────────────── -export const RESULT_ENCODINGS = ['json', 'arrow_ipc'] as const; +export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const; export type ResultEncoding = (typeof RESULT_ENCODINGS)[number]; export interface QueryResult extends TenantScoped { @@ -345,7 +345,7 @@ export interface DashboardVersion extends TenantScoped { // ── 14. ReportDefinition ─────────────────────────────────────────────────── -export const REPORT_FORMATS = ['pdf', 'csv', 'xlsx', 'parquet', 'arrow_ipc'] as const; +export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const; export type ReportFormat = (typeof REPORT_FORMATS)[number]; export interface ReportDefinition extends TenantScoped { @@ -378,18 +378,18 @@ export interface ReportRun extends TenantScoped { // ── 16. DataQualityRule ──────────────────────────────────────────────────── export const QUALITY_RULE_KINDS = [ - 'not_null', - 'unique', - 'accepted_values', - 'range', - 'freshness', - 'row_count', - 'referential', - 'custom', + "not_null", + "unique", + "accepted_values", + "range", + "freshness", + "row_count", + "referential", + "custom", ] as const; export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number]; -export const QUALITY_SEVERITIES = ['warn', 'block'] as const; +export const QUALITY_SEVERITIES = ["warn", "block"] as const; export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number]; export interface DataQualityRule extends TenantScoped { @@ -423,7 +423,7 @@ export interface DataQualityRun extends TenantScoped { // ── 18. LineageReference ─────────────────────────────────────────────────── -export const LINEAGE_DIRECTIONS = ['input', 'output'] as const; +export const LINEAGE_DIRECTIONS = ["input", "output"] as const; export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number]; /** @@ -438,7 +438,7 @@ export interface LineageReference extends TenantScoped { job_name: string; direction: LineageDirection; /** The analytics entity this edge touches. */ - entity_type: 'dataset' | 'metric' | 'dashboard' | 'report'; + entity_type: "dataset" | "metric" | "dashboard" | "report"; entity_id: string; /** Namespaced dataset name on the other side of the edge. */ counterpart_ref: string; @@ -447,7 +447,7 @@ export interface LineageReference extends TenantScoped { // ── 19. BIApplicationBinding ─────────────────────────────────────────────── -export const BI_APPLICATION_KINDS = ['superset', 'evidence', 'des1gn', 'notebook', 'custom'] as const; +export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const; export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number]; /** @@ -482,55 +482,55 @@ export interface BIApplicationBinding extends TenantScoped { * or omission is a test failure rather than drift. */ export const ANALYTICS_DOMAIN_ENTITIES = [ - 'AnalyticsDataset', - 'DatasetVersion', - 'DatasetField', - 'DatasetGrant', - 'MetricDefinition', - 'MetricVersion', - 'DimensionDefinition', - 'SemanticModel', - 'AnalyticsQuery', - 'QueryExecution', - 'QueryResult', - 'DashboardDefinition', - 'DashboardVersion', - 'ReportDefinition', - 'ReportRun', - 'DataQualityRule', - 'DataQualityRun', - 'LineageReference', - 'BIApplicationBinding', + "AnalyticsDataset", + "DatasetVersion", + "DatasetField", + "DatasetGrant", + "MetricDefinition", + "MetricVersion", + "DimensionDefinition", + "SemanticModel", + "AnalyticsQuery", + "QueryExecution", + "QueryResult", + "DashboardDefinition", + "DashboardVersion", + "ReportDefinition", + "ReportRun", + "DataQualityRule", + "DataQualityRun", + "LineageReference", + "BIApplicationBinding", ] as const; export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number]; /** Entity -> physical table in sql/008_analytics_domain_model.sql. */ export const ANALYTICS_ENTITY_TABLES: Record = { - AnalyticsDataset: 'opt_analytics_dataset', - DatasetVersion: 'opt_analytics_dataset_version', - DatasetField: 'opt_analytics_dataset_field', - DatasetGrant: 'opt_analytics_dataset_grant', - MetricDefinition: 'opt_analytics_metric_definition', - MetricVersion: 'opt_analytics_metric_version', - DimensionDefinition: 'opt_analytics_dimension_definition', - SemanticModel: 'opt_analytics_semantic_model', - AnalyticsQuery: 'opt_analytics_query', - QueryExecution: 'opt_analytics_query_execution', - QueryResult: 'opt_analytics_query_result', - DashboardDefinition: 'opt_analytics_dashboard_definition', - DashboardVersion: 'opt_analytics_dashboard_version', - ReportDefinition: 'opt_analytics_report_definition', - ReportRun: 'opt_analytics_report_run', - DataQualityRule: 'opt_analytics_data_quality_rule', - DataQualityRun: 'opt_analytics_data_quality_run', - LineageReference: 'opt_analytics_lineage_reference', - BIApplicationBinding: 'opt_analytics_bi_application_binding', + AnalyticsDataset: "opt_analytics_dataset", + DatasetVersion: "opt_analytics_dataset_version", + DatasetField: "opt_analytics_dataset_field", + DatasetGrant: "opt_analytics_dataset_grant", + MetricDefinition: "opt_analytics_metric_definition", + MetricVersion: "opt_analytics_metric_version", + DimensionDefinition: "opt_analytics_dimension_definition", + SemanticModel: "opt_analytics_semantic_model", + AnalyticsQuery: "opt_analytics_query", + QueryExecution: "opt_analytics_query_execution", + QueryResult: "opt_analytics_query_result", + DashboardDefinition: "opt_analytics_dashboard_definition", + DashboardVersion: "opt_analytics_dashboard_version", + ReportDefinition: "opt_analytics_report_definition", + ReportRun: "opt_analytics_report_run", + DataQualityRule: "opt_analytics_data_quality_rule", + DataQualityRun: "opt_analytics_data_quality_run", + LineageReference: "opt_analytics_lineage_reference", + BIApplicationBinding: "opt_analytics_bi_application_binding", }; /** The four entities BI-P002 requires to be immutable once written. */ export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [ - 'DatasetVersion', - 'MetricVersion', - 'SemanticModel', - 'DashboardVersion', + "DatasetVersion", + "MetricVersion", + "SemanticModel", + "DashboardVersion", ]; diff --git a/src/analytics/domain/tiers.ts b/src/analytics/domain/tiers.ts index bc0b4a2..d67e590 100644 --- a/src/analytics/domain/tiers.ts +++ b/src/analytics/domain/tiers.ts @@ -21,24 +21,24 @@ * ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product * repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers. */ -import { SiaxError } from '../../platform/errors'; +import { SiaxError } from "../../platform/errors"; -export const ANALYTICS_TIERS = ['RAW', 'NORMALIZED', 'CURATED', 'SERVING'] as const; +export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const; export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number]; /** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */ export const TIER_PHYSICAL_HOME: Record = { - RAW: 'st0re:iceberg|parquet', - NORMALIZED: 'st0re:iceberg', - CURATED: 'st0re:iceberg', - SERVING: 'clickhouse:projection', + RAW: "st0re:iceberg|parquet", + NORMALIZED: "st0re:iceberg", + CURATED: "st0re:iceberg", + SERVING: "clickhouse:projection", }; /** Tiers a BI presentation surface may read without an explicit exception. */ -export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ['SERVING', 'CURATED']; +export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"]; export function isAnalyticsTier(v: unknown): v is AnalyticsTier { - return typeof v === 'string' && (ANALYTICS_TIERS as readonly string[]).includes(v); + return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v); } /** @@ -64,8 +64,8 @@ export function assertReadableByBiSurface( if (BI_READABLE_TIERS.includes(dataset.tier)) return; if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return; throw new SiaxError( - 'AUTHORIZATION_DENIED', - '0pt.analytics', + "AUTHORIZATION_DENIED", + "0pt.analytics", `BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` + `without a recorded raw_access_exception (BI-P003)`, ); @@ -75,8 +75,8 @@ export function assertReadableByBiSurface( export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void { if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`, ); } diff --git a/src/analytics/engine/clickhouse-engine.ts b/src/analytics/engine/clickhouse-engine.ts index e2e27d0..7753f24 100644 --- a/src/analytics/engine/clickhouse-engine.ts +++ b/src/analytics/engine/clickhouse-engine.ts @@ -26,17 +26,17 @@ * LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts * ClickHouse pipelines, never quarantined market data. */ -import crypto from 'node:crypto'; -import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../../platform/errors'; -import { ClickHouseAnalytics } from '../clickhouse-analytics'; +import crypto from "node:crypto"; +import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors"; +import { ClickHouseAnalytics } from "../clickhouse-analytics"; import { compileMetricQuery, compileProjectionQuery, dimensionCatalogFrom, FORCED_TENANT_PREDICATE, type DimensionCatalog, -} from './query-compiler'; -import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from './schema'; +} from "./query-compiler"; +import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema"; import { applyQuota, clickhouseSettings, @@ -44,7 +44,7 @@ import { DEFAULT_BUDGETS, planningSettings, type ResourceBudget, -} from './resource-governance'; +} from "./resource-governance"; import type { AnalystQueryRequest, AnalyticsIdentity, @@ -56,7 +56,7 @@ import type { QueryEstimate, QueryExplanation, SiaxAnalyticsEngine, -} from './siax-analytics-engine'; +} from "./siax-analytics-engine"; export interface ClickHouseEngineOptions { baseUrl: string; @@ -66,13 +66,13 @@ export interface ClickHouseEngineOptions { timeoutMs?: number; /** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */ dimensionCatalogs?: Record>; - schema?: Omit; + schema?: Omit; } const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined }; export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { - readonly provider = 'clickhouse'; + readonly provider = "clickhouse"; /** PRESERVED pre-existing implementation, composed not replaced. */ private readonly legacy: ClickHouseAnalytics; @@ -81,15 +81,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { private readonly database: string; private readonly timeoutMs: number; private readonly dimensionCatalogs: Record>; - private readonly schemaOpts: Omit; + private readonly schemaOpts: Omit; private readonly gate: ConcurrencyGate; /** query_id -> AbortController, so cancelQuery can stop the client side too. */ private readonly inFlight = new Map(); constructor(opts: ClickHouseEngineOptions) { - this.baseUrl = opts.baseUrl.replace(/\/+$/, ''); - this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined; - this.database = opts.database ?? 'opt_analytics'; + this.baseUrl = opts.baseUrl.replace(/\/+$/, ""); + this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined; + this.database = opts.database ?? "opt_analytics"; this.timeoutMs = opts.timeoutMs ?? 30_000; this.dimensionCatalogs = opts.dimensionCatalogs ?? {}; this.schemaOpts = opts.schema ?? {}; @@ -112,7 +112,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { } private budgetFor(identity: AnalyticsIdentity): ResourceBudget { - const tier = identity.budget_tier ?? 'INTERACTIVE'; + const tier = identity.budget_tier ?? "INTERACTIVE"; return applyQuota(DEFAULT_BUDGETS[tier], identity.quota); } @@ -125,13 +125,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { private assertIdentity(identity: AnalyticsIdentity): void { if (!identity?.tenant_id || !identity.tenant_id.trim()) { throw new SiaxError( - 'UNAUTHENTICATED', - '0pt.analytics', - 'analytics requires a verified CL0UD identity carrying tenant_id', + "UNAUTHENTICATED", + "0pt.analytics", + "analytics requires a verified CL0UD identity carrying tenant_id", ); } if (!identity.principal_id || !identity.principal_id.trim()) { - throw new SiaxError('UNAUTHENTICATED', '0pt.analytics', 'analytics requires a verified principal_id'); + throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id"); } } @@ -150,16 +150,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { async health(): Promise { try { const legacyOk = await this.legacy.health(); - if (!legacyOk) - return { healthy: false, provider: this.provider, detail: 'legacy health probe returned false' }; - const v = await this.raw('SELECT version() AS v FORMAT JSON', {}, {}); + if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" }; + const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {}); const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v; return { healthy: true, provider: this.provider, version }; } catch (err) { return { healthy: false, provider: this.provider, - detail: err instanceof SiaxError ? err.code : 'unreachable', + detail: err instanceof SiaxError ? err.code : "unreachable", }; } } @@ -192,13 +191,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { budget.maxResultRows, ); const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as - ClickHouseJsonResponse | undefined; + | ClickHouseJsonResponse + | undefined; const groupNames = request.group_by ?? []; const rows = (json?.data ?? []).map((r) => { const groups: Record = {}; groupNames.forEach((name, i) => { const v = (r as Record)[`grp${i}`]; - groups[name] = v === undefined || v === null ? '' : String(v); + groups[name] = v === undefined || v === null ? "" : String(v); }); return { ...(groupNames.length ? { groups } : {}), @@ -230,15 +230,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { */ async executeLegacyMetric( identity: AnalyticsIdentity, - request: { - metric: string; - aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count'; - dataset_id?: string; - group_by?: string; - from_time?: string; - to_time?: string; - limit?: number; - }, + request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number }, ): Promise { this.assertIdentity(identity); const started = Date.now(); @@ -272,9 +264,9 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // BI-P033 (analytics authorization) exist would create precisely the // "raw SQL attempt" hole BI-P011's adversarial list enumerates. throw new SiaxError( - 'CAPABILITY_UNAVAILABLE', - '0pt.analytics', - 'analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land', + "CAPABILITY_UNAVAILABLE", + "0pt.analytics", + "analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land", ); } @@ -304,7 +296,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read // AFTER primary-key pruning — which is why tenant_id leads every ORDER BY // in schema.ts: the estimate for one tenant does not include everyone. - const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, ''); + const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, ""); let rowsToRead = 0; let partsRead = 0; try { @@ -321,7 +313,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // An estimate that cannot be produced must not read as "cheap". throw err instanceof SiaxError ? err - : new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'could not estimate query'); + : new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query"); } // Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes. // 128 B/row is a deliberately PESSIMISTIC constant for these tables @@ -355,7 +347,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise { this.assertIdentity(identity); if (!queryId.startsWith(this.queryIdPrefix(identity))) { - throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this tenant'); + throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant"); } this.inFlight.get(queryId)?.abort(); this.inFlight.delete(queryId); @@ -365,7 +357,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { await this.raw( `KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`, { param_killId: queryId }, - { max_execution_time: '10' }, + { max_execution_time: "10" }, ); return true; } catch { @@ -396,14 +388,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { this.catalogFor(identity), budget.maxResultRows, ); - const sql = compiled.sql.replace(/ FORMAT JSON$/, ''); + const sql = compiled.sql.replace(/ FORMAT JSON$/, ""); const res = (await this.raw( `EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`, compiled.params, planningSettings(budget), )) as string | undefined; return { - plan: typeof res === 'string' ? res : JSON.stringify(res ?? ''), + plan: typeof res === "string" ? res : JSON.stringify(res ?? ""), forced_tenant_predicate: FORCED_TENANT_PREDICATE, table: compiled.table, }; @@ -422,22 +414,22 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { const body = request.rows .map((r) => { if (!r.entity_key || !r.entity_key.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every projection row needs an entity_key'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key"); } return JSON.stringify({ // tenant/project come from the VERIFIED identity, never the row — // a row-supplied tenant_id is the INF0 Wave-2 bug class. tenant_id: identity.tenant_id, - project_id: identity.project_id ?? '', + project_id: identity.project_id ?? "", dataset_id: request.dataset_id, entity_key: r.entity_key, - as_of: (r.as_of ?? now.toISOString()).replace('T', ' ').replace(/Z$/, ''), + as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""), version: r.version ?? version, attributes: r.attributes ?? {}, measures: r.measures ?? {}, }); }) - .join('\n'); + .join("\n"); await this.raw( `INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`, {}, @@ -463,7 +455,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { await this.raw( `OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`, { param_tenant: identity.tenant_id }, - { max_execution_time: '120' }, + { max_execution_time: "120" }, ); const compiled = compileProjectionQuery( { tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 }, @@ -474,7 +466,8 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { `SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` + `WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`; const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as - { data?: { value: number | string }[] } | undefined; + | { data?: { value: number | string }[] } + | undefined; return { dataset_id, row_count: Number(res?.data?.[0]?.value ?? 0), @@ -485,16 +478,16 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // ── transport ──────────────────────────────────────────────────────────── private queryIdPrefix(identity: AnalyticsIdentity): string { - return `opt_${crypto.createHash('sha256').update(identity.tenant_id).digest('hex').slice(0, 12)}_`; + return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`; } private newQueryId(identity: AnalyticsIdentity): string { - return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, '')}`; + return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`; } /** DDL path: no readonly, no result parsing beyond errors. */ private async ddl(sql: string): Promise { - await this.raw(sql, {}, { max_execution_time: '60' }); + await this.raw(sql, {}, { max_execution_time: "60" }); } private async raw( @@ -505,13 +498,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { body?: string, ): Promise { const url = new URL(this.baseUrl); - url.searchParams.set('query', sql); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); - if (queryId) url.searchParams.set('query_id', queryId); + if (queryId) url.searchParams.set("query_id", queryId); const headers: Record = {}; - if (this.authHeader) headers['Authorization'] = this.authHeader; + if (this.authHeader) headers["Authorization"] = this.authHeader; const controller = new AbortController(); if (queryId) this.inFlight.set(queryId, controller); @@ -520,13 +513,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { let res: Response; try { res = await fetch(url.toString(), { - method: 'POST', + method: "POST", headers, - body: body ?? '', + body: body ?? "", signal: controller.signal, }); } catch (cause) { - throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause }); } finally { clearTimeout(timeout); } @@ -537,7 +530,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status); throw new SiaxError( siaxErrorCodeForUpstreamStatus(res.status), - 'clickhouse', + "clickhouse", `query failed (CH code ${code})`, ); } @@ -555,10 +548,10 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine { // have already been flushed, the JSON body can carry an `exception` key // under a 200 — a truncated answer indistinguishable from a whole one. // Treat any body carrying an exception as a failure, whatever the status. - if (parsed && typeof parsed === 'object' && 'exception' in parsed) { - const ex = String((parsed as { exception?: unknown }).exception ?? ''); - const code = /Code: (\d+)/.exec(ex)?.[1] ?? 'unknown'; - throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `query failed mid-stream (CH code ${code})`); + if (parsed && typeof parsed === "object" && "exception" in parsed) { + const ex = String((parsed as { exception?: unknown }).exception ?? ""); + const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown"; + throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`); } return parsed; } diff --git a/src/analytics/engine/query-compiler.ts b/src/analytics/engine/query-compiler.ts index 3a11c54..ffb6aa2 100644 --- a/src/analytics/engine/query-compiler.ts +++ b/src/analytics/engine/query-compiler.ts @@ -27,20 +27,20 @@ * the guarantee rather than replacing it, and * tests/analytics/tenant-predicate-regression.test.ts asserts BOTH. */ -import { SiaxError } from '../../platform/errors'; -import type { AnalyticsTable } from './schema'; +import { SiaxError } from "../../platform/errors"; +import type { AnalyticsTable } from "./schema"; /** A physical identifier that has been VALIDATED, not merely received. */ const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; export function assertSafeIdentifier(value: string, what: string): string { if (!IDENT_RE.test(value)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid ${what} identifier`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`); } return value; } -export const METRIC_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const; +export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const; export type MetricAggregate = (typeof METRIC_AGGREGATES)[number]; /** @@ -97,15 +97,15 @@ export interface CompiledQuery { * forcing, the test fails on a missing substring rather than on a subtle * behavioural difference that a mocked engine would happily hide. */ -export const FORCED_TENANT_PREDICATE = 'tenant_id = {tenant:String}'; +export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}"; const AGG_SQL: Record = { - sum: 'sum(value)', - avg: 'avg(value)', - min: 'min(value)', - max: 'max(value)', - count: 'count()', - count_distinct: 'uniqExact(event_id)', + sum: "sum(value)", + avg: "avg(value)", + min: "min(value)", + max: "max(value)", + count: "count()", + count_distinct: "uniqExact(event_id)", }; const MAX_GROUP_BY = 8; @@ -126,13 +126,13 @@ export function compileMetricQuery( ): CompiledQuery { if (!req.tenant_id || !req.tenant_id.trim()) { // Fail closed. A blank tenant must never compile to "no predicate". - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); } - assertSafeIdentifier(db, 'database'); - const table: AnalyticsTable = req.table ?? 'opt_analytics_event'; - assertSafeIdentifier(table, 'table'); + assertSafeIdentifier(db, "database"); + const table: AnalyticsTable = req.table ?? "opt_analytics_event"; + assertSafeIdentifier(table, "table"); if (!METRIC_AGGREGATES.includes(req.aggregate)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'unsupported aggregate'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate"); } const params: Record = { @@ -141,22 +141,22 @@ export function compileMetricQuery( }; // ── the forced predicate. First condition, always, no branch. ───────── - const conditions: string[] = [FORCED_TENANT_PREDICATE, 'metric = {metric:String}']; + const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"]; if (req.project_id) { - conditions.push('project_id = {project:String}'); + conditions.push("project_id = {project:String}"); params.param_project = req.project_id; } if (req.dataset_id) { - conditions.push('dataset_id = {dataset:String}'); + conditions.push("dataset_id = {dataset:String}"); params.param_dataset = req.dataset_id; } if (req.from_time) { - conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})'); + conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})"); params.param_fromTime = req.from_time; } if (req.to_time) { - conditions.push('event_time < parseDateTime64BestEffort({toTime:String})'); + conditions.push("event_time < parseDateTime64BestEffort({toTime:String})"); params.param_toTime = req.to_time; } @@ -165,14 +165,16 @@ export function compileMetricQuery( for (const [name, value] of Object.entries(req.filters ?? {})) { const physical = catalog.resolve(name); if (physical === undefined) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown filter dimension "${name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`); } // `physical` came from the catalog, but validate anyway: a corrupted or // hostile catalog row must not become an injection vector either. - assertSafeIdentifier(physical, 'dimension'); + assertSafeIdentifier(physical, "dimension"); const nameKey = `f${filterIdx}n`; const valKey = `f${filterIdx}v`; - conditions.push(`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`); + conditions.push( + `dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`, + ); params[`param_${nameKey}`] = physical; params[`param_${valKey}`] = value; filterIdx += 1; @@ -182,8 +184,8 @@ export function compileMetricQuery( const groupBy = req.group_by ?? []; if (groupBy.length > MAX_GROUP_BY) { throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', + "INVALID_REQUEST", + "0pt.analytics", `at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`, ); } @@ -192,9 +194,9 @@ export function compileMetricQuery( groupBy.forEach((name, i) => { const physical = catalog.resolve(name); if (physical === undefined) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown group_by dimension "${name}"`); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`); } - assertSafeIdentifier(physical, 'dimension'); + assertSafeIdentifier(physical, "dimension"); const key = `g${i}`; // The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text. selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`); @@ -205,10 +207,10 @@ export function compileMetricQuery( const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); const sql = - `SELECT ${selectParts.length ? selectParts.join(', ') + ', ' : ''}${AGG_SQL[req.aggregate]} AS value ` + + `SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` + `FROM ${db}.${table} ` + - `WHERE ${conditions.join(' AND ')} ` + - (groupParts.length ? `GROUP BY ${groupParts.join(', ')} ORDER BY value DESC ` : '') + + `WHERE ${conditions.join(" AND ")} ` + + (groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") + `LIMIT ${limit} FORMAT JSON`; return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table }; @@ -224,29 +226,29 @@ export function compileProjectionQuery( maxResultRows: number, ): CompiledQuery { if (!req.tenant_id || !req.tenant_id.trim()) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query"); } - assertSafeIdentifier(db, 'database'); + assertSafeIdentifier(db, "database"); const params: Record = { param_tenant: req.tenant_id, param_dataset: req.dataset_id, }; - const conditions = [FORCED_TENANT_PREDICATE, 'dataset_id = {dataset:String}']; + const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"]; if (req.project_id) { - conditions.push('project_id = {project:String}'); + conditions.push("project_id = {project:String}"); params.param_project = req.project_id; } if (req.entity_keys && req.entity_keys.length > 0) { - conditions.push('has({entityKeys:Array(String)}, entity_key)'); + conditions.push("has({entityKeys:Array(String)}, entity_key)"); params.param_entityKeys = JSON.stringify(req.entity_keys); } const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows)); const sql = `SELECT entity_key, as_of, version, attributes, measures ` + `FROM ${db}.opt_dataset_projection ` + - `WHERE ${conditions.join(' AND ')} ` + + `WHERE ${conditions.join(" AND ")} ` + `ORDER BY entity_key LIMIT ${limit} FORMAT JSON`; - return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: 'opt_dataset_projection' }; + return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" }; } /** @@ -273,29 +275,25 @@ export function restrictedTenantUserDdl(args: { tables: readonly string[]; settingsProfile?: string; }): string[] { - assertSafeIdentifier(args.db, 'database'); - assertSafeIdentifier(args.userName, 'user'); + assertSafeIdentifier(args.db, "database"); + assertSafeIdentifier(args.userName, "user"); // A tenant id is `t_` in this estate; enforce that shape strictly // rather than trusting the caller. if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) { - throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant id is not policy-safe'); + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe"); } - if (args.password.includes("'") || args.password.includes('\\')) { - throw new SiaxError( - 'INVALID_REQUEST', - '0pt.analytics', - 'password contains characters that cannot be embedded', - ); + if (args.password.includes("'") || args.password.includes("\\")) { + throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded"); } const profile = args.settingsProfile; - if (profile) assertSafeIdentifier(profile, 'settings profile'); + if (profile) assertSafeIdentifier(profile, "settings profile"); const stmts: string[] = [ `CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` + (profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`), ]; for (const table of args.tables) { - assertSafeIdentifier(table, 'table'); + assertSafeIdentifier(table, "table"); stmts.push( `CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` + `FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`, diff --git a/src/analytics/engine/resource-governance.ts b/src/analytics/engine/resource-governance.ts index fe7cb6c..4d2e184 100644 --- a/src/analytics/engine/resource-governance.ts +++ b/src/analytics/engine/resource-governance.ts @@ -30,7 +30,7 @@ * subsidises a query bomb. `INTERACTIVE` is the dashboard default and is * deliberately mean. */ -import { SiaxError } from '../../platform/errors'; +import { SiaxError } from "../../platform/errors"; export interface ResourceBudget { /** Seconds. ClickHouse `max_execution_time`. */ @@ -47,7 +47,7 @@ export interface ResourceBudget { maxConcurrentQueries: number; } -export const BUDGET_TIERS = ['INTERACTIVE', 'ANALYTICAL', 'BATCH'] as const; +export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const; export type BudgetTier = (typeof BUDGET_TIERS)[number]; /** @@ -91,7 +91,7 @@ export const DEFAULT_BUDGETS: Record = { export function applyQuota(base: ResourceBudget, quota: Partial | undefined): ResourceBudget { if (!quota) return { ...base }; const pick = (b: number, q: number | undefined): number => - typeof q === 'number' && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b; + typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b; return { maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec), maxResultRows: pick(base.maxResultRows, quota.maxResultRows), @@ -114,14 +114,14 @@ export function applyQuota(base: ResourceBudget, quota: Partial */ export function clickhouseSettings(budget: ResourceBudget): Record { return { - readonly: '1', + readonly: "1", max_execution_time: String(budget.maxExecutionTimeSec), - timeout_overflow_mode: 'throw', + timeout_overflow_mode: "throw", max_result_rows: String(budget.maxResultRows), - result_overflow_mode: 'throw', + result_overflow_mode: "throw", max_rows_to_read: String(budget.maxRowsToRead), max_bytes_to_read: String(budget.maxBytesToRead), - read_overflow_mode: 'throw', + read_overflow_mode: "throw", max_memory_usage: String(budget.maxMemoryUsageBytes), }; } @@ -139,21 +139,21 @@ export function clickhouseSettings(budget: ResourceBudget): Record { return { - readonly: '1', + readonly: "1", max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)), - timeout_overflow_mode: 'throw', + timeout_overflow_mode: "throw", max_memory_usage: String(budget.maxMemoryUsageBytes), }; } /** Every setting name BI-P012 requires, for the DoD assertion. */ export const REQUIRED_GOVERNANCE_SETTINGS = [ - 'max_execution_time', - 'max_result_rows', - 'max_rows_to_read', - 'max_bytes_to_read', - 'max_memory_usage', - 'max_concurrent_queries', + "max_execution_time", + "max_result_rows", + "max_rows_to_read", + "max_bytes_to_read", + "max_memory_usage", + "max_concurrent_queries", ] as const; /** @@ -192,8 +192,8 @@ export class ConcurrencyGate { const current = this.inFlight.get(tenantId) ?? 0; if (current >= limit) { throw new SiaxError( - 'RATE_LIMITED', - '0pt.analytics', + "RATE_LIMITED", + "0pt.analytics", `tenant already has ${current} analytics queries in flight (limit ${limit})`, ); } diff --git a/src/analytics/engine/schema.ts b/src/analytics/engine/schema.ts index 1825a8d..657d1fb 100644 --- a/src/analytics/engine/schema.ts +++ b/src/analytics/engine/schema.ts @@ -55,14 +55,14 @@ export interface AnalyticsSchemaOptions { /** Physical tables this module owns (opt_metric_events is NOT one of them). */ export const ANALYTICS_TABLES = [ - 'opt_analytics_event', - 'opt_metric_daily', - 'opt_dataset_projection', + "opt_analytics_event", + "opt_metric_daily", + "opt_dataset_projection", ] as const; export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number]; /** The materialized view wiring opt_analytics_event -> opt_metric_daily. */ -export const ANALYTICS_MATERIALIZED_VIEWS = ['opt_metric_daily_mv'] as const; +export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const; /** * DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so @@ -163,6 +163,7 @@ export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] { */ export function sortingKeySql(db: string, table: string): string { return ( - `SELECT sorting_key FROM system.tables ` + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` + `SELECT sorting_key FROM system.tables ` + + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` ); } diff --git a/src/analytics/engine/siax-analytics-engine.ts b/src/analytics/engine/siax-analytics-engine.ts index 168dd02..53ac17a 100644 --- a/src/analytics/engine/siax-analytics-engine.ts +++ b/src/analytics/engine/siax-analytics-engine.ts @@ -19,9 +19,9 @@ * (VehIQ) call an API in front of this interface; they never hold a provider * credential and never construct an engine. */ -import type { CompiledQuery, MetricAggregate } from './query-compiler'; -import type { AnalyticsTable } from './schema'; -import type { BudgetTier, ResourceBudget } from './resource-governance'; +import type { CompiledQuery, MetricAggregate } from "./query-compiler"; +import type { AnalyticsTable } from "./schema"; +import type { BudgetTier, ResourceBudget } from "./resource-governance"; /** * The VERIFIED caller. `tenant_id` here is the one the engine forces into @@ -158,24 +158,21 @@ export interface SiaxAnalyticsEngine { explain(identity: AnalyticsIdentity, request: MetricRequest): Promise; - ingestProjection( - identity: AnalyticsIdentity, - request: ProjectionIngestRequest, - ): Promise<{ ingested: number }>; + ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>; refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise; } /** The eight capability names, for the provider contract test. */ export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [ - 'health', - 'executeMetric', - 'executeQuery', - 'estimateQuery', - 'cancelQuery', - 'explain', - 'ingestProjection', - 'refreshProjection', + "health", + "executeMetric", + "executeQuery", + "estimateQuery", + "cancelQuery", + "explain", + "ingestProjection", + "refreshProjection", ] as const; export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number]; diff --git a/src/analytics/repositories/analytics-domain-repository.ts b/src/analytics/repositories/analytics-domain-repository.ts index b7a341e..8b01c37 100644 --- a/src/analytics/repositories/analytics-domain-repository.ts +++ b/src/analytics/repositories/analytics-domain-repository.ts @@ -19,9 +19,9 @@ * Uses postgres.js through `RepositorySql`, the same connection type the other * P0 repositories take (src/repositories/db.ts). */ -import crypto from 'node:crypto'; -import { SiaxError } from '../../platform/errors'; -import type { RepositorySql } from '../../repositories/db'; +import crypto from "node:crypto"; +import { SiaxError } from "../../platform/errors"; +import type { RepositorySql } from "../../repositories/db"; import type { AnalyticsDataset, DatasetVersion, @@ -29,8 +29,8 @@ import type { MetricDefinition, MetricVersion, PiiClassification, -} from '../domain/entities'; -import type { AnalyticsTier } from '../domain/tiers'; +} from "../domain/entities"; +import type { AnalyticsTier } from "../domain/tiers"; /** The verified tenant scope every call must present. */ export interface TenantScope { @@ -41,9 +41,9 @@ export interface TenantScope { function assertScope(scope: TenantScope): void { if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) { throw new SiaxError( - 'UNAUTHENTICATED', - '0pt.analytics', - 'analytics persistence requires a verified tenant_id + project_id', + "UNAUTHENTICATED", + "0pt.analytics", + "analytics persistence requires a verified tenant_id + project_id", ); } } @@ -90,7 +90,7 @@ function toDataset(r: DatasetRow): AnalyticsDataset { quality_rule_ids: r.quality_rule_ids ?? [], retention_days: Number(r.retention_days), classification: r.classification as PiiClassification, - status: r.status as AnalyticsDataset['status'], + status: r.status as AnalyticsDataset["status"], ...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}), ...(r.description ? { description: r.description } : {}), created_at: iso(r.created_at)!, @@ -104,10 +104,7 @@ export class AnalyticsDomainRepository { // ── AnalyticsDataset ───────────────────────────────────────────────────── - async createDataset( - scope: TenantScope, - input: Omit, - ): Promise { + async createDataset(scope: TenantScope, input: Omit): Promise { assertScope(scope); const rows = await this.sql` INSERT INTO opt_analytics_dataset @@ -136,10 +133,7 @@ export class AnalyticsDomainRepository { return rows[0] ? toDataset(rows[0]) : undefined; } - async listDatasets( - scope: TenantScope, - filter?: { tier?: AnalyticsTier; status?: string }, - ): Promise { + async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise { assertScope(scope); const rows = await this.sql` SELECT * FROM opt_analytics_dataset @@ -161,20 +155,14 @@ export class AnalyticsDomainRepository { */ async createDatasetVersion( scope: TenantScope, - input: { - dataset_id: string; - contract_hash: string; - schema_contract_version: string; - record_count?: number; - created_by?: string; - }, + input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string }, ): Promise { assertScope(scope); const owner = await this.getDataset(scope, input.dataset_id); if (!owner) { - throw new SiaxError('NOT_FOUND', '0pt.analytics', 'dataset not found in this tenant'); + throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant"); } - const id = `dsv_${crypto.randomUUID().replace(/-/g, '')}`; + const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`; try { const rows = await this.sql< { @@ -214,8 +202,8 @@ export class AnalyticsDomainRepository { ...(r.created_by ? { created_by: r.created_by } : {}), }; } catch (err) { - if (String(err).includes('duplicate key')) { - throw new SiaxError('CONFLICT', '0pt.analytics', 'a concurrent writer minted this dataset version'); + if (String(err).includes("duplicate key")) { + throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version"); } throw err; } @@ -224,18 +212,7 @@ export class AnalyticsDomainRepository { async listDatasetVersions(scope: TenantScope, datasetId: string): Promise { assertScope(scope); const rows = await this.sql< - { - dataset_version_id: string; - dataset_id: string; - tenant_id: string; - project_id: string; - version: number; - contract_hash: string; - schema_contract_version: string; - record_count: string | number | null; - created_at: string | Date; - created_by: string | null; - }[] + { dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[] >` SELECT * FROM opt_analytics_dataset_version WHERE dataset_id = ${datasetId} @@ -260,29 +237,11 @@ export class AnalyticsDomainRepository { async createMetric( scope: TenantScope, - input: Omit< - MetricDefinition, - 'tenant_id' | 'project_id' | 'created_at' | 'updated_at' | 'current_version' - >, + input: Omit, ): Promise { assertScope(scope); const rows = await this.sql< - { - metric_id: string; - tenant_id: string; - project_id: string; - name: string; - dataset_id: string; - aggregation: string; - measure_field: string; - denominator_field: string | null; - allowed_dimension_ids: string[]; - current_version: number; - owner_id: string; - description: string | null; - created_at: string | Date; - updated_at: string | Date | null; - }[] + { metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[] >` INSERT INTO opt_analytics_metric_definition (metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field, @@ -299,7 +258,7 @@ export class AnalyticsDomainRepository { project_id: r.project_id, name: r.name, dataset_id: r.dataset_id, - aggregation: r.aggregation as MetricDefinition['aggregation'], + aggregation: r.aggregation as MetricDefinition["aggregation"], measure_field: r.measure_field, ...(r.denominator_field ? { denominator_field: r.denominator_field } : {}), allowed_dimension_ids: r.allowed_dimension_ids ?? [], @@ -317,21 +276,10 @@ export class AnalyticsDomainRepository { input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string }, ): Promise { assertScope(scope); - const hash = crypto.createHash('sha256').update(input.definition_json).digest('hex'); - const id = `mv_${crypto.randomUUID().replace(/-/g, '')}`; + const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex"); + const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`; const rows = await this.sql< - { - metric_version_id: string; - metric_id: string; - tenant_id: string; - project_id: string; - version: number; - definition_json: string; - definition_hash: string; - change_reason: string | null; - created_at: string | Date; - created_by: string | null; - }[] + { metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[] >` INSERT INTO opt_analytics_metric_version (metric_version_id, metric_id, tenant_id, project_id, version, definition_json, @@ -361,22 +309,11 @@ export class AnalyticsDomainRepository { async createDimension( scope: TenantScope, - input: Omit, + input: Omit, ): Promise { assertScope(scope); const rows = await this.sql< - { - dimension_id: string; - tenant_id: string; - project_id: string; - name: string; - dataset_id: string; - field_name: string; - field_type: string; - classification: string; - description: string | null; - created_at: string | Date; - }[] + { dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[] >` INSERT INTO opt_analytics_dimension_definition (dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description) @@ -392,7 +329,7 @@ export class AnalyticsDomainRepository { name: r.name, dataset_id: r.dataset_id, field_name: r.field_name, - field_type: r.field_type as DimensionDefinition['field_type'], + field_type: r.field_type as DimensionDefinition["field_type"], classification: r.classification as PiiClassification, ...(r.description ? { description: r.description } : {}), created_at: iso(r.created_at)!, diff --git a/tests/analytics/duckdb-provider.test.ts b/tests/analytics/duckdb-provider.test.ts index 9039821..43fcac7 100644 --- a/tests/analytics/duckdb-provider.test.ts +++ b/tests/analytics/duckdb-provider.test.ts @@ -13,6 +13,7 @@ * /etc/passwd. Measured, recorded, and asserted here — because a comment * saying so would rot the moment someone "simplified" the bootstrap. */ +import { execFileSync } from 'node:child_process'; import fs from 'node:fs'; import path from 'node:path'; import { describe, expect, it } from 'vitest'; @@ -307,6 +308,19 @@ describe('BI-P014 — the committed fixtures', () => { }); it('does NOT ship the Iceberg table, because its manifests carry absolute paths', () => { - expect(fs.existsSync(path.join(FIXTURES, 'iceberg'))).toBe(false); + // Asserted against GIT, not the filesystem. The claim is about what is + // COMMITTED; the live suite REBUILDS that table locally, so a filesystem + // check fails whenever both suites run in one pass — it was testing the + // wrong thing rather than catching a real problem. + const tracked = execFileSync('git', ['ls-files', 'tests/fixtures/duckdb/'], { + encoding: 'utf8', + cwd: process.cwd(), + }) + .split('\n') + .filter(Boolean); + // The parquet/csv fixtures ARE tracked, so a version that shipped nothing + // would fail here too rather than passing vacuously. + expect(tracked.length).toBeGreaterThan(0); + expect(tracked.filter((f) => f.includes('/iceberg'))).toEqual([]); }); }); diff --git a/tests/integration/analytics-arrow-clickhouse.integration.test.ts b/tests/integration/analytics-arrow-clickhouse.integration.test.ts index b87e38f..648ed06 100644 --- a/tests/integration/analytics-arrow-clickhouse.integration.test.ts +++ b/tests/integration/analytics-arrow-clickhouse.integration.test.ts @@ -17,13 +17,13 @@ * ran against real pyarrow + DuckDB; that transcript is committed at * evidence/bi-p002-p012/bi-p005-arrow-roundtrip.json. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; import { arrowStreamSettings, clickhouseArrowStreamFormat, decodeArrowStream, encodeArrowStream, -} from '@/src/analytics/arrow/ipc'; +} from "@/src/analytics/arrow/ipc"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -32,29 +32,31 @@ const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_arrow_${Date.now().toString(36)}`; function authHeaders(): Record { - return CLICKHOUSE_AUTH ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` } : {}; + return CLICKHOUSE_AUTH + ? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}` } + : {}; } async function chBytes(sql: string, settings: Record = {}): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); - const res = await fetch(url.toString(), { method: 'POST', headers: authHeaders(), body: '' }); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: "" }); const buf = new Uint8Array(await res.arrayBuffer()); if (!res.ok) throw new Error(`CH ${res.status}: ${new TextDecoder().decode(buf).slice(0, 300)}`); return buf; } async function chText(sql: string, body?: BodyInit): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); - const res = await fetch(url.toString(), { method: 'POST', headers: authHeaders(), body: body ?? '' }); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); + const res = await fetch(url.toString(), { method: "POST", headers: authHeaders(), body: body ?? "" }); const text = await res.text(); if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 400)}`); return text; } -run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { +run("BI-P005 Arrow IPC against a LIVE ClickHouse", () => { beforeAll(async () => { await chText(`CREATE DATABASE IF NOT EXISTS ${DB}`); await chText( @@ -68,7 +70,7 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { if (CLICKHOUSE_URL) await chText(`DROP DATABASE IF EXISTS ${DB}`); }, 60_000); - it('THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count', async () => { + it("THEIR encoder, OUR reader: ClickHouse ArrowStream decodes with the expected schema and row count", async () => { const bytes = await chBytes( `SELECT toString(t) AS tenant_id, toInt64(n) AS n, toFloat64(v) AS v, b AS flag, toDateTime64(ts, 3, 'UTC') AS ts @@ -84,38 +86,35 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { ); const info = decodeArrowStream(bytes); expect(info.rowCount).toBe(3); - expect(info.fields.map((f) => f.name)).toEqual(['tenant_id', 'n', 'v', 'flag', 'ts']); + expect(info.fields.map((f) => f.name)).toEqual(["tenant_id", "n", "v", "flag", "ts"]); // Utf8=5 (NOT Binary=4 — that is what output_format_arrow_string_as_string buys), // Int=2, FloatingPoint=3, Bool=6, Timestamp=10. expect(info.fields.map((f) => f.typeType)).toEqual([5, 2, 3, 6, 10]); }); - it('WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents', async () => { + it("WITHOUT output_format_arrow_string_as_string, ClickHouse emits Binary — the semantic loss the setting prevents", async () => { const bytes = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, { - output_format_arrow_string_as_string: '0', + output_format_arrow_string_as_string: "0", }); // Arrow Type union: Binary = 4, Utf8 = 5. expect(decodeArrowStream(bytes).fields[0].typeType).toBe(4); - const asText = await chBytes( - `SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, - arrowStreamSettings(), - ); + const asText = await chBytes(`SELECT 'x' AS s FORMAT ${clickhouseArrowStreamFormat()}`, arrowStreamSettings()); expect(decodeArrowStream(asText).fields[0].typeType).toBe(5); }); - it('OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged', async () => { + it("OUR encoder, THEIR reader: ClickHouse INSERTs our Arrow and returns every value unchanged", async () => { const bytes = encodeArrowStream([ - { name: 'tenant_id', type: 'utf8', values: ['t_alpha', 't_beta', 't_gamma'] }, - { name: 'n', type: 'int64', values: [1, 2, 3] }, - { name: 'v', type: 'float64', values: [1.5, 2.5, 3.25] }, - { name: 'flag', type: 'bool', values: [true, false, true] }, + { name: "tenant_id", type: "utf8", values: ["t_alpha", "t_beta", "t_gamma"] }, + { name: "n", type: "int64", values: [1, 2, 3] }, + { name: "v", type: "float64", values: [1.5, 2.5, 3.25] }, + { name: "flag", type: "bool", values: [true, false, true] }, { - name: 'ts', - type: 'timestamp_ms', + name: "ts", + type: "timestamp_ms", values: [ - new Date('2026-01-01T00:00:00.000Z'), - new Date('2026-06-15T08:09:10.123Z'), - new Date('2026-09-06T12:34:56.789Z'), + new Date("2026-01-01T00:00:00.000Z"), + new Date("2026-06-15T08:09:10.123Z"), + new Date("2026-09-06T12:34:56.789Z"), ], }, ]); @@ -124,30 +123,22 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { const back = await chText( `SELECT tenant_id, n, v, flag, toString(ts) AS ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT JSON`, ); - const rows = ( - JSON.parse(back) as { - data: { - tenant_id: string; - n: string | number; - v: number | string; - flag: boolean | number; - ts: string; - }[]; - } - ).data; + const rows = (JSON.parse(back) as { + data: { tenant_id: string; n: string | number; v: number | string; flag: boolean | number; ts: string }[]; + }).data; expect(rows).toHaveLength(3); - expect(rows.map((r) => r.tenant_id)).toEqual(['t_alpha', 't_beta', 't_gamma']); + expect(rows.map((r) => r.tenant_id)).toEqual(["t_alpha", "t_beta", "t_gamma"]); expect(rows.map((r) => Number(r.n))).toEqual([1, 2, 3]); expect(rows.map((r) => Number(r.v))).toEqual([1.5, 2.5, 3.25]); expect(rows.map((r) => Boolean(Number(r.flag)))).toEqual([true, false, true]); expect(rows.map((r) => r.ts)).toEqual([ - '2026-01-01 00:00:00.000', - '2026-06-15 08:09:10.123', - '2026-09-06 12:34:56.789', + "2026-01-01 00:00:00.000", + "2026-06-15 08:09:10.123", + "2026-09-06 12:34:56.789", ]); }); - it('full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows', async () => { + it("full circle: ClickHouse -> Arrow -> our decoder -> ClickHouse yields identical rows", async () => { const exported = await chBytes( `SELECT tenant_id, n, v, flag, ts FROM ${DB}.arrow_roundtrip ORDER BY n FORMAT ArrowStream`, arrowStreamSettings(), @@ -167,18 +158,14 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { expect(Number((JSON.parse(diff) as { data: { c: number | string }[] }).data[0].c)).toBe(0); }); - it('a large result survives the round trip (batching/offsets are not only correct for three rows)', async () => { + it("a large result survives the round trip (batching/offsets are not only correct for three rows)", async () => { const n = 20_000; const bytes = encodeArrowStream([ - { name: 'tenant_id', type: 'utf8', values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, - { name: 'n', type: 'int64', values: Array.from({ length: n }, (_, i) => i) }, - { name: 'v', type: 'float64', values: Array.from({ length: n }, (_, i) => i / 3) }, - { name: 'flag', type: 'bool', values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, - { - name: 'ts', - type: 'timestamp_ms', - values: Array.from({ length: n }, () => new Date('2026-09-06T00:00:00Z')), - }, + { name: "tenant_id", type: "utf8", values: Array.from({ length: n }, (_, i) => `t_${i % 7}`) }, + { name: "n", type: "int64", values: Array.from({ length: n }, (_, i) => i) }, + { name: "v", type: "float64", values: Array.from({ length: n }, (_, i) => i / 3) }, + { name: "flag", type: "bool", values: Array.from({ length: n }, (_, i) => i % 2 === 0) }, + { name: "ts", type: "timestamp_ms", values: Array.from({ length: n }, () => new Date("2026-09-06T00:00:00Z")) }, ]); expect(decodeArrowStream(bytes).rowCount).toBe(n); @@ -187,11 +174,7 @@ run('BI-P005 Arrow IPC against a LIVE ClickHouse', () => { const agg = await chText( `SELECT count() AS c, sum(n) AS s, uniqExact(tenant_id) AS t, countIf(flag) AS f FROM ${DB}.arrow_big FORMAT JSON`, ); - const d = ( - JSON.parse(agg) as { - data: { c: string | number; s: string | number; t: string | number; f: string | number }[]; - } - ).data[0]; + const d = (JSON.parse(agg) as { data: { c: string | number; s: string | number; t: string | number; f: string | number }[] }).data[0]; expect(Number(d.c)).toBe(n); expect(Number(d.s)).toBe((n * (n - 1)) / 2); expect(Number(d.t)).toBe(7); diff --git a/tests/integration/analytics-clickhouse.integration.test.ts b/tests/integration/analytics-clickhouse.integration.test.ts index 64e9f00..8d072cb 100644 --- a/tests/integration/analytics-clickhouse.integration.test.ts +++ b/tests/integration/analytics-clickhouse.integration.test.ts @@ -7,19 +7,19 @@ * JSONEachRow ingest, the closed metric-query DSL, and — the §69-relevant * part — that the FORCED tenant predicate isolates tenants on real data. */ -import { describe, it, expect, beforeAll } from 'vitest'; -import { ClickHouseAnalytics } from '@/src/analytics/clickhouse-analytics'; +import { describe, it, expect, beforeAll } from "vitest"; +import { ClickHouseAnalytics } from "@/src/analytics/clickhouse-analytics"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_itest_${Date.now().toString(36)}`; -run('Analytics capability against a LIVE ClickHouse', () => { +run("Analytics capability against a LIVE ClickHouse", () => { // The skip-variant still evaluates this callback body, so the constructor // must tolerate the env-less case (the tests themselves never run then). const analytics = new ClickHouseAnalytics({ - baseUrl: CLICKHOUSE_URL ?? 'http://skipped.invalid', + baseUrl: CLICKHOUSE_URL ?? "http://skipped.invalid", basicAuth: process.env.CLICKHOUSE_AUTH, database: DB, }); @@ -31,90 +31,66 @@ run('Analytics capability against a LIVE ClickHouse', () => { await analytics.ensureSchema(30); }, 60_000); - it('ingests events and answers aggregate metric queries', async () => { + it("ingests events and answers aggregate metric queries", async () => { const res = await analytics.ingest([ - { tenant_id: 't_alpha', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 100 }, - { tenant_id: 't_alpha', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 50 }, - { tenant_id: 't_alpha', dataset_id: 'ds_vehicles', metric: 'rows_ingested', value: 7 }, - { tenant_id: 't_beta', dataset_id: 'ds_companies', metric: 'rows_ingested', value: 999 }, + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 100 }, + { tenant_id: "t_alpha", dataset_id: "ds_companies", metric: "rows_ingested", value: 50 }, + { tenant_id: "t_alpha", dataset_id: "ds_vehicles", metric: "rows_ingested", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_companies", metric: "rows_ingested", value: 999 }, ]); expect(res.ingested).toBe(4); const total = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'rows_ingested', - aggregate: 'sum', + tenant_id: "t_alpha", + metric: "rows_ingested", + aggregate: "sum", }); expect(total).toEqual([{ value: 157 }]); const perDataset = await analytics.queryMetric({ - tenant_id: 't_alpha', - dataset_id: 'ds_companies', - metric: 'rows_ingested', - aggregate: 'sum', + tenant_id: "t_alpha", + dataset_id: "ds_companies", + metric: "rows_ingested", + aggregate: "sum", }); expect(perDataset).toEqual([{ value: 150 }]); }, 30_000); it("REAL tenant isolation: t_beta's numbers never leak into t_alpha aggregates", async () => { - const alpha = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'rows_ingested', - aggregate: 'max', - }); + const alpha = await analytics.queryMetric({ tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "max" }); expect(alpha[0].value).toBe(100); // NOT 999 - const beta = await analytics.queryMetric({ - tenant_id: 't_beta', - metric: 'rows_ingested', - aggregate: 'sum', - }); + const beta = await analytics.queryMetric({ tenant_id: "t_beta", metric: "rows_ingested", aggregate: "sum" }); expect(beta[0].value).toBe(999); - const ghost = await analytics.queryMetric({ - tenant_id: 't_gamma', - metric: 'rows_ingested', - aggregate: 'count', - }); + const ghost = await analytics.queryMetric({ tenant_id: "t_gamma", metric: "rows_ingested", aggregate: "count" }); expect(ghost[0].value).toBe(0); }); - it('group-by dimension queries work through the closed DSL', async () => { + it("group-by dimension queries work through the closed DSL", async () => { await analytics.ingest([ - { - tenant_id: 't_alpha', - dataset_id: 'ds_x', - metric: 'dq_score', - value: 90, - dimensions: { severity: 'warning' }, - }, - { - tenant_id: 't_alpha', - dataset_id: 'ds_x', - metric: 'dq_score', - value: 40, - dimensions: { severity: 'critical' }, - }, + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 90, dimensions: { severity: "warning" } }, + { tenant_id: "t_alpha", dataset_id: "ds_x", metric: "dq_score", value: 40, dimensions: { severity: "critical" } }, ]); const rows = await analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'dq_score', - aggregate: 'avg', - group_by: 'severity', + tenant_id: "t_alpha", + metric: "dq_score", + aggregate: "avg", + group_by: "severity", }); expect(rows).toHaveLength(2); - expect(rows.find((r) => r.group === 'warning')?.value).toBe(90); + expect(rows.find((r) => r.group === "warning")?.value).toBe(90); }); - it('rejects tenant-less queries and invalid group_by identifiers (injection surface closed)', async () => { + it("rejects tenant-less queries and invalid group_by identifiers (injection surface closed)", async () => { await expect( - analytics.queryMetric({ tenant_id: ' ', metric: 'x', aggregate: 'sum' }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + analytics.queryMetric({ tenant_id: " ", metric: "x", aggregate: "sum" }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); await expect( analytics.queryMetric({ - tenant_id: 't_alpha', - metric: 'x', - aggregate: 'sum', + tenant_id: "t_alpha", + metric: "x", + aggregate: "sum", group_by: "sev'; DROP TABLE opt_metric_events; --", - }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + }) + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); }); }); diff --git a/tests/integration/analytics-engine-clickhouse.integration.test.ts b/tests/integration/analytics-engine-clickhouse.integration.test.ts index db88ff5..d798ca9 100644 --- a/tests/integration/analytics-engine-clickhouse.integration.test.ts +++ b/tests/integration/analytics-engine-clickhouse.integration.test.ts @@ -12,38 +12,38 @@ * Gated on CLICKHOUSE_URL; skips honestly when unset. CI's `integration` job * supplies it and hard-fails on a skipped suite. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; -import { ANALYTICS_MATERIALIZED_VIEWS, ANALYTICS_TABLES } from '@/src/analytics/engine/schema'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { ANALYTICS_MATERIALIZED_VIEWS, ANALYTICS_TABLES } from "@/src/analytics/engine/schema"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; const run = CLICKHOUSE_URL ? describe : describe.skip; const DB = `opt_engine_${Date.now().toString(36)}`; -const ID: AnalyticsIdentity = { tenant_id: 't_alpha', project_id: 'p_main', principal_id: 'usr_alpha' }; +const ID: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; let engine: ClickHouseAnalyticsEngine; async function raw(sql: string, body?: string): Promise { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); const headers: Record = {}; - if (CLICKHOUSE_AUTH) headers['Authorization'] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); const text = await res.text(); if (!res.ok) throw new Error(`CH ${res.status}: ${text.slice(0, 300)}`); return text; } -run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { +run("BI-P009/BI-P010 ClickHouse provider against a LIVE server", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_alpha: { severity: 'severity', make: 'vehicle_make' } }, + dimensionCatalogs: { t_alpha: { severity: "severity", make: "vehicle_make" } }, }); await engine.ensureSchema(); await engine.ensureSchema(); // idempotent, as the DDL claims @@ -58,67 +58,67 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { it("health() reports the live server's version", async () => { const h = await engine.health(); expect(h.healthy).toBe(true); - expect(h.provider).toBe('clickhouse'); + expect(h.provider).toBe("clickhouse"); expect(h.version).toMatch(/^\d+\.\d+/); }); // ── BI-P010: the schema is what it claims to be ───────────────────────── - it('BI-P010: four purpose-built tables exist, NOT one giant table', async () => { + it("BI-P010: four purpose-built tables exist, NOT one giant table", async () => { const listed = await raw( `SELECT name FROM system.tables WHERE database = '${DB}' AND engine != 'MaterializedView' ORDER BY name FORMAT TSV`, ); - const names = listed.trim().split('\n').filter(Boolean); + const names = listed.trim().split("\n").filter(Boolean); // The three this module owns, PLUS the preserved opt_metric_events. for (const t of ANALYTICS_TABLES) expect(names).toContain(t); - expect(names).toContain('opt_metric_events'); + expect(names).toContain("opt_metric_events"); expect(names.length).toBeGreaterThanOrEqual(4); }); - it('BI-P010: the tables use different engines and different sort keys — the split is real', async () => { + it("BI-P010: the tables use different engines and different sort keys — the split is real", async () => { const meta = await raw( `SELECT name, engine, sorting_key FROM system.tables WHERE database = '${DB}' ORDER BY name FORMAT JSON`, ); const rows = (JSON.parse(meta) as { data: { name: string; engine: string; sorting_key: string }[] }).data; const by = Object.fromEntries(rows.map((r) => [r.name, r])); - expect(by['opt_analytics_event'].engine).toBe('MergeTree'); - expect(by['opt_metric_daily'].engine).toBe('AggregatingMergeTree'); - expect(by['opt_dataset_projection'].engine).toBe('ReplacingMergeTree'); + expect(by["opt_analytics_event"].engine).toBe("MergeTree"); + expect(by["opt_metric_daily"].engine).toBe("AggregatingMergeTree"); + expect(by["opt_dataset_projection"].engine).toBe("ReplacingMergeTree"); // Different ORDER BY keys — the reason one table cannot serve both jobs. - expect(by['opt_analytics_event'].sorting_key).not.toBe(by['opt_dataset_projection'].sorting_key); + expect(by["opt_analytics_event"].sorting_key).not.toBe(by["opt_dataset_projection"].sorting_key); // Every one still leads with tenant_id. - for (const t of ANALYTICS_TABLES) expect(by[t].sorting_key.startsWith('tenant_id')).toBe(true); + for (const t of ANALYTICS_TABLES) expect(by[t].sorting_key.startsWith("tenant_id")).toBe(true); }); - it('BI-P010: TTL and LowCardinality are actually applied, not just written in a comment', async () => { + it("BI-P010: TTL and LowCardinality are actually applied, not just written in a comment", async () => { const ddl = await raw(`SHOW CREATE TABLE ${DB}.opt_analytics_event FORMAT TSVRaw`); - expect(ddl).toContain('TTL'); - expect(ddl).toContain('LowCardinality(String)'); - expect(ddl).toContain('PARTITION BY toYYYYMM(event_time)'); + expect(ddl).toContain("TTL"); + expect(ddl).toContain("LowCardinality(String)"); + expect(ddl).toContain("PARTITION BY toYYYYMM(event_time)"); const rollup = await raw(`SHOW CREATE TABLE ${DB}.opt_metric_daily FORMAT TSVRaw`); - expect(rollup).toContain('AggregateFunction(sum, Float64)'); + expect(rollup).toContain("AggregateFunction(sum, Float64)"); }); - it('BI-P010: the materialized view actually populates the rollup', async () => { + it("BI-P010: the materialized view actually populates the rollup", async () => { const views = await raw( `SELECT name FROM system.tables WHERE database='${DB}' AND engine='MaterializedView' FORMAT TSV`, ); for (const v of ANALYTICS_MATERIALIZED_VIEWS) expect(views).toContain(v); - await engine.ingestProjection(ID, { dataset_id: 'warmup', rows: [{ entity_key: 'w1' }] }); + await engine.ingestProjection(ID, { dataset_id: "warmup", rows: [{ entity_key: "w1" }] }); const rows = Array.from({ length: 20 }, (_, i) => JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_mv', - metric: 'mv_metric', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_mv", + metric: "mv_metric", event_id: `mv${i}`, - event_time: '2026-09-03 12:00:00.000', + event_time: "2026-09-03 12:00:00.000", value: 2, dim_names: [], dim_values: [], }), - ).join('\n'); + ).join("\n"); await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows); const rolled = await raw( @@ -132,75 +132,72 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { // ── capability 2: executeMetric (already covered by the isolation suite) ─ - it('executeMetric returns grouped rows keyed by the CALLER-FACING dimension name', async () => { - const rows = ['warning', 'critical', 'critical'].map((sev, i) => + it("executeMetric returns grouped rows keyed by the CALLER-FACING dimension name", async () => { + const rows = ["warning", "critical", "critical"].map((sev, i) => JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_g', - metric: 'grouped', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_g", + metric: "grouped", event_id: `g${i}`, - event_time: '2026-09-04 09:00:00.000', + event_time: "2026-09-04 09:00:00.000", value: 5, - dim_names: ['severity'], + dim_names: ["severity"], dim_values: [sev], }), ); - await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows.join('\n')); + await raw(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, rows.join("\n")); const result = await engine.executeMetric(ID, { - metric: 'grouped', - aggregate: 'sum', - group_by: ['severity'], + metric: "grouped", + aggregate: "sum", + group_by: ["severity"], }); - const critical = result.rows.find((r) => r.groups?.severity === 'critical'); + const critical = result.rows.find((r) => r.groups?.severity === "critical"); expect(critical?.value).toBe(10); expect(result.stats.result_rows).toBe(2); - expect(result.query_id.startsWith('opt_')).toBe(true); + expect(result.query_id.startsWith("opt_")).toBe(true); }); // ── capability 4: estimateQuery ───────────────────────────────────────── - it('estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget', async () => { - const est = await engine.estimateQuery(ID, { metric: 'grouped', aggregate: 'sum' }); + it("estimateQuery gets a real EXPLAIN ESTIMATE from the server and compares it to the budget", async () => { + const est = await engine.estimateQuery(ID, { metric: "grouped", aggregate: "sum" }); expect(est.estimated_rows_to_read).toBeGreaterThan(0); expect(est.within_budget).toBe(true); expect(est.budget.maxRowsToRead).toBeGreaterThan(0); }); - it('estimateQuery REFUSES a query whose estimate exceeds a tightened quota', async () => { + it("estimateQuery REFUSES a query whose estimate exceeds a tightened quota", async () => { const tiny: AnalyticsIdentity = { ...ID, quota: { maxRowsToRead: 1 } }; - const est = await engine.estimateQuery(tiny, { metric: 'grouped', aggregate: 'sum' }); + const est = await engine.estimateQuery(tiny, { metric: "grouped", aggregate: "sum" }); expect(est.within_budget).toBe(false); expect(est.reason).toMatch(/over budget/); }); // ── capability 6: explain ─────────────────────────────────────────────── - it('explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts', async () => { - const ex = await engine.explain(ID, { metric: 'grouped', aggregate: 'sum', group_by: ['severity'] }); - expect(ex.forced_tenant_predicate).toBe('tenant_id = {tenant:String}'); - expect(ex.table).toBe('opt_analytics_event'); + it("explain() proves the forced predicate reaches the PRIMARY KEY, pruning parts", async () => { + const ex = await engine.explain(ID, { metric: "grouped", aggregate: "sum", group_by: ["severity"] }); + expect(ex.forced_tenant_predicate).toBe("tenant_id = {tenant:String}"); + expect(ex.table).toBe("opt_analytics_event"); // This is the payoff of BI-P010's tenant-first ORDER BY: the server // reports tenant_id as a PrimaryKey key with a condition on it, i.e. the // isolation predicate is an index range scan, not a post-read filter. - expect(ex.plan).toContain('ReadFromMergeTree'); - expect(ex.plan).toContain('PrimaryKey'); + expect(ex.plan).toContain("ReadFromMergeTree"); + expect(ex.plan).toContain("PrimaryKey"); expect(ex.plan).toMatch(/Keys:[\s\S]*tenant_id/); expect(ex.plan).toMatch(/tenant_id in \['t_alpha', 't_alpha'\]/); // The plan describes only the CALLER's own tenant; no other tenant id // may appear in a plan handed back to them. - expect(ex.plan).not.toContain('t_beta'); + expect(ex.plan).not.toContain("t_beta"); }); // ── capability 5: cancelQuery ─────────────────────────────────────────── - it('cancelQuery issues a real KILL QUERY the server accepts', async () => { + it("cancelQuery issues a real KILL QUERY the server accepts", async () => { // Nothing is running under this id, so the honest outcome is "the KILL // statement was valid and matched nothing" -> true, with no exception. - const ok = await engine.cancelQuery( - ID, - `${(await engine.executeMetric(ID, { metric: 'grouped', aggregate: 'count' })).query_id}`, - ); + const ok = await engine.cancelQuery(ID, `${(await engine.executeMetric(ID, { metric: "grouped", aggregate: "count" })).query_id}`); expect(ok).toBe(true); }); @@ -208,10 +205,10 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { it("ingestProjection writes the SERVING tier with the identity's tenant, ignoring any row-level claim", async () => { const res = await engine.ingestProjection(ID, { - dataset_id: 'ds_vehicles', + dataset_id: "ds_vehicles", rows: [ - { entity_key: 'VIN001', attributes: { make: 'volvo' }, measures: { price: 189000 } }, - { entity_key: 'VIN002', attributes: { make: 'saab' }, measures: { price: 42000 } }, + { entity_key: "VIN001", attributes: { make: "volvo" }, measures: { price: 189000 } }, + { entity_key: "VIN002", attributes: { make: "saab" }, measures: { price: 42000 } }, ], }); expect(res.ingested).toBe(2); @@ -222,18 +219,18 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { expect(Number((JSON.parse(stored) as { data: { t: number | string }[] }).data[0].t)).toBe(1); }); - it('refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL)', async () => { + it("refreshProjection makes a re-ingest idempotent (ReplacingMergeTree + OPTIMIZE FINAL)", async () => { // Re-ingest the SAME entity keys at a higher version. await engine.ingestProjection(ID, { - dataset_id: 'ds_vehicles', + dataset_id: "ds_vehicles", rows: [ - { entity_key: 'VIN001', version: 9_999_999_999_999, measures: { price: 175000 } }, - { entity_key: 'VIN002', version: 9_999_999_999_999, measures: { price: 41000 } }, + { entity_key: "VIN001", version: 9_999_999_999_999, measures: { price: 175000 } }, + { entity_key: "VIN002", version: 9_999_999_999_999, measures: { price: 41000 } }, ], }); - const refreshed = await engine.refreshProjection(ID, 'ds_vehicles'); + const refreshed = await engine.refreshProjection(ID, "ds_vehicles"); expect(refreshed.row_count).toBe(2); // 2 entities, not 4 rows - expect(refreshed.dataset_id).toBe('ds_vehicles'); + expect(refreshed.dataset_id).toBe("ds_vehicles"); const winner = await raw( `SELECT measures['price'] AS p FROM ${DB}.opt_dataset_projection FINAL ` + @@ -244,12 +241,12 @@ run('BI-P009/BI-P010 ClickHouse provider against a LIVE server', () => { // ── the PRESERVED path still works through the engine ─────────────────── - it('the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine', async () => { + it("the PRESERVED ClickHouseAnalytics path still ingests and answers through the engine", async () => { await engine.legacyAnalytics.ingest([ - { tenant_id: 't_alpha', dataset_id: 'ds_legacy', metric: 'legacy_metric', value: 7 }, - { tenant_id: 't_beta', dataset_id: 'ds_legacy', metric: 'legacy_metric', value: 700 }, + { tenant_id: "t_alpha", dataset_id: "ds_legacy", metric: "legacy_metric", value: 7 }, + { tenant_id: "t_beta", dataset_id: "ds_legacy", metric: "legacy_metric", value: 700 }, ]); - const res = await engine.executeLegacyMetric(ID, { metric: 'legacy_metric', aggregate: 'sum' }); + const res = await engine.executeLegacyMetric(ID, { metric: "legacy_metric", aggregate: "sum" }); expect(res.rows[0].value).toBe(7); // NOT 707 }); }); diff --git a/tests/integration/analytics-resource-governance.integration.test.ts b/tests/integration/analytics-resource-governance.integration.test.ts index a4451da..a0b5094 100644 --- a/tests/integration/analytics-resource-governance.integration.test.ts +++ b/tests/integration/analytics-resource-governance.integration.test.ts @@ -14,10 +14,10 @@ * timed WHILE the bomb is in flight and compared against a declared SLO, * not merely observed to "still work". */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; -import { DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -37,7 +37,7 @@ const DB = `opt_gov_${Date.now().toString(36)}`; */ const UNRELATED_TENANT_SLO_MS = 5000; -const VICTIM: AnalyticsIdentity = { tenant_id: 't_victim', project_id: 'p_main', principal_id: 'usr_v' }; +const VICTIM: AnalyticsIdentity = { tenant_id: "t_victim", project_id: "p_main", principal_id: "usr_v" }; let engine: ClickHouseAnalyticsEngine; @@ -47,40 +47,40 @@ async function chRun( settings: Record = {}, body?: string, ): Promise<{ ok: boolean; code?: string; text: string; ms: number }> { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); const headers: Record = {}; - if (CLICKHOUSE_AUTH) headers['Authorization'] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}`; + if (CLICKHOUSE_AUTH) headers["Authorization"] = `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString("base64")}`; const t0 = Date.now(); - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); const text = await res.text(); return { ok: res.ok, code: /Code: (\d+)/.exec(text)?.[1], text, ms: Date.now() - t0 }; } -run('BI-P012 resource governance against a LIVE ClickHouse', () => { +run("BI-P012 resource governance against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_victim: { severity: 'severity' } }, + dimensionCatalogs: { t_victim: { severity: "severity" } }, }); await engine.ensureSchema(); // A small, real dataset for the unrelated tenant. const rows = Array.from({ length: 500 }, (_, i) => JSON.stringify({ - tenant_id: 't_victim', - project_id: 'p_main', - dataset_id: 'ds_v', - metric: 'victim_metric', + tenant_id: "t_victim", + project_id: "p_main", + dataset_id: "ds_v", + metric: "victim_metric", event_id: `v${i}`, - event_time: '2026-09-05 08:00:00.000', + event_time: "2026-09-05 08:00:00.000", value: 1, - dim_names: ['severity'], - dim_values: ['warning'], + dim_names: ["severity"], + dim_values: ["warning"], }), - ).join('\n'); + ).join("\n"); const ins = await chRun(`INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, {}, rows); expect(ins.ok, ins.text.slice(0, 200)).toBe(true); }, 120_000); @@ -91,51 +91,51 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // ── each named limit, fired individually ──────────────────────────────── - it('max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)', async () => { - const r = await chRun('SELECT count() FROM numbers(1000000000) FORMAT JSON', { - readonly: '1', - max_rows_to_read: '1000', - read_overflow_mode: 'throw', + it("max_rows_to_read terminates a read bomb (CH code 158 TOO_MANY_ROWS)", async () => { + const r = await chRun("SELECT count() FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_rows_to_read: "1000", + read_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('158'); + expect(r.code).toBe("158"); }); - it('max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)', async () => { - const r = await chRun('SELECT sum(number) FROM numbers(1000000000) FORMAT JSON', { - readonly: '1', - max_bytes_to_read: '1024', - read_overflow_mode: 'throw', + it("max_bytes_to_read terminates a scan bomb (CH code 307 TOO_MANY_BYTES)", async () => { + const r = await chRun("SELECT sum(number) FROM numbers(1000000000) FORMAT JSON", { + readonly: "1", + max_bytes_to_read: "1024", + read_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('307'); + expect(r.code).toBe("307"); }); - it('max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)', async () => { - const r = await chRun('SELECT count() FROM numbers(50000000000) FORMAT JSON', { - readonly: '1', - max_execution_time: '1', - timeout_overflow_mode: 'throw', + it("max_execution_time terminates a slow bomb (CH code 159 TIMEOUT_EXCEEDED)", async () => { + const r = await chRun("SELECT count() FROM numbers(50000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "1", + timeout_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('159'); + expect(r.code).toBe("159"); expect(r.ms).toBeLessThan(20_000); }); - it('max_result_rows terminates a result bomb and does NOT return a partial result', async () => { - const r = await chRun('SELECT number FROM numbers(100000) FORMAT JSON', { - readonly: '1', - max_result_rows: '10', - result_overflow_mode: 'throw', + it("max_result_rows terminates a result bomb and does NOT return a partial result", async () => { + const r = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "throw", }); expect(r.ok).toBe(false); - expect(r.code).toBe('396'); // TOO_MANY_ROWS_OR_BYTES + expect(r.code).toBe("396"); // TOO_MANY_ROWS_OR_BYTES // The dangerous alternative: result_overflow_mode=break returns 10 rows // with HTTP 200 and no indication of truncation. - const broken = await chRun('SELECT number FROM numbers(100000) FORMAT JSON', { - readonly: '1', - max_result_rows: '10', - result_overflow_mode: 'break', + const broken = await chRun("SELECT number FROM numbers(100000) FORMAT JSON", { + readonly: "1", + max_result_rows: "10", + result_overflow_mode: "break", }); // MEASURED on ClickHouse 24.8: break mode answers HTTP 200 with 65409 of // 100000 rows (it stops at a block boundary, not at the stated 10) and @@ -147,38 +147,37 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { expect(brokenRows).toBeLessThan(100_000); }); - it('max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)', async () => { - const r = await chRun('SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON', { - readonly: '1', - max_memory_usage: '1000000', - max_execution_time: '30', - }); + it("max_memory_usage terminates a memory bomb (CH code 241 MEMORY_LIMIT_EXCEEDED)", async () => { + const r = await chRun( + "SELECT uniqExact(toString(number)) FROM numbers(50000000) FORMAT JSON", + { readonly: "1", max_memory_usage: "1000000", max_execution_time: "30" }, + ); expect(r.ok).toBe(false); - expect(r.code).toBe('241'); + expect(r.code).toBe("241"); }); - it('readonly=1 means the query text cannot re-raise its own limits', async () => { + it("readonly=1 means the query text cannot re-raise its own limits", async () => { const r = await chRun( - 'SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON', - { readonly: '1', max_rows_to_read: '1000', read_overflow_mode: 'throw' }, + "SELECT count() FROM numbers(1000000000) SETTINGS max_rows_to_read = 10000000000 FORMAT JSON", + { readonly: "1", max_rows_to_read: "1000", read_overflow_mode: "throw" }, ); expect(r.ok).toBe(false); // 164 = READONLY (changing settings is forbidden); 158 = the limit still // applied. Either proves the caller did not win — but NOT a 200. - expect(['164', '158']).toContain(r.code); + expect(["164", "158"]).toContain(r.code); }); - it('readonly=1 blocks writes through the query surface', async () => { - const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: '1' }); + it("readonly=1 blocks writes through the query surface", async () => { + const r = await chRun(`DROP TABLE ${DB}.opt_analytics_event FORMAT JSON`, { readonly: "1" }); expect(r.ok).toBe(false); }); // ── the half that is usually skipped ──────────────────────────────────── - it('an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight', async () => { + it("an unrelated tenant stays within its declared SLO WHILE a query bomb is in flight", async () => { // Baseline first, so "fast enough" is measured against this server, not // against a number someone hoped for. - const baseline = await engine.executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }); + const baseline = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(baseline.rows[0].value).toBe(500); const baselineMs = baseline.stats.duration_ms; @@ -187,23 +186,23 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // numbers(20e9) in ~5s because it can skip the values entirely. Hashing // every row cannot be short-circuited, so this one really does burn CPU // until the limit kills it. - const bomb = chRun('SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON', { - readonly: '1', - max_execution_time: '8', - timeout_overflow_mode: 'throw', + const bomb = chRun("SELECT sum(cityHash64(number)) FROM numbers(20000000000) FORMAT JSON", { + readonly: "1", + max_execution_time: "8", + timeout_overflow_mode: "throw", }); // While it burns, run the unrelated tenant's interactive query repeatedly. const samples: number[] = []; for (let i = 0; i < 5; i++) { - const r = await engine.executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }); + const r = await engine.executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }); expect(r.rows[0].value).toBe(500); // still CORRECT, not just fast samples.push(r.stats.duration_ms); } const bombResult = await bomb; - expect(bombResult.ok, 'the bomb was NOT terminated').toBe(false); - expect(bombResult.code).toBe('159'); + expect(bombResult.ok, "the bomb was NOT terminated").toBe(false); + expect(bombResult.code).toBe("159"); const worst = Math.max(...samples); expect( @@ -214,24 +213,24 @@ run('BI-P012 resource governance against a LIVE ClickHouse', () => { // ── the local admission gate, on the real engine ──────────────────────── - it('the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only', async () => { + it("the per-tenant concurrency gate refuses the N+1st in-flight query for THAT tenant only", async () => { const limit = DEFAULT_BUDGETS.INTERACTIVE.maxConcurrentQueries; // Fire limit+4 concurrently; some must be refused with RATE_LIMITED // rather than all of them reaching the server. const attempts = Array.from({ length: limit + 4 }, () => engine - .executeMetric(VICTIM, { metric: 'victim_metric', aggregate: 'count' }) - .then(() => 'ok' as const) - .catch((e: { code?: string }) => e.code ?? 'error'), + .executeMetric(VICTIM, { metric: "victim_metric", aggregate: "count" }) + .then(() => "ok" as const) + .catch((e: { code?: string }) => e.code ?? "error"), ); const outcomes = await Promise.all(attempts); - expect(outcomes.filter((o) => o === 'RATE_LIMITED').length).toBeGreaterThan(0); - expect(outcomes.filter((o) => o === 'ok').length).toBeGreaterThan(0); + expect(outcomes.filter((o) => o === "RATE_LIMITED").length).toBeGreaterThan(0); + expect(outcomes.filter((o) => o === "ok").length).toBeGreaterThan(0); // A different tenant is unaffected by the noisy one's exhausted slots. const other = await engine.executeMetric( - { tenant_id: 't_quiet', project_id: 'p_main', principal_id: 'usr_q' }, - { metric: 'victim_metric', aggregate: 'count' }, + { tenant_id: "t_quiet", project_id: "p_main", principal_id: "usr_q" }, + { metric: "victim_metric", aggregate: "count" }, ); expect(other.rows[0].value).toBe(0); // its own (empty) data, and no error }, 60_000); diff --git a/tests/integration/analytics-tenant-isolation.integration.test.ts b/tests/integration/analytics-tenant-isolation.integration.test.ts index 4d1f60f..3a1f9c9 100644 --- a/tests/integration/analytics-tenant-isolation.integration.test.ts +++ b/tests/integration/analytics-tenant-isolation.integration.test.ts @@ -26,16 +26,16 @@ * `integration` job sets it from a service container and hard-fails if the * suites skip, so this is not a self-skipping green in practice. */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { ClickHouseAnalyticsEngine } from '@/src/analytics/engine/clickhouse-engine'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { ClickHouseAnalyticsEngine } from "@/src/analytics/engine/clickhouse-engine"; import { compileMetricQuery, dimensionCatalogFrom, restrictedTenantUserDdl, -} from '@/src/analytics/engine/query-compiler'; -import { ANALYTICS_TABLES, sortingKeySql } from '@/src/analytics/engine/schema'; -import { DEFAULT_BUDGETS } from '@/src/analytics/engine/resource-governance'; -import type { AnalyticsIdentity } from '@/src/analytics/engine/siax-analytics-engine'; +} from "@/src/analytics/engine/query-compiler"; +import { ANALYTICS_TABLES, sortingKeySql } from "@/src/analytics/engine/schema"; +import { DEFAULT_BUDGETS } from "@/src/analytics/engine/resource-governance"; +import type { AnalyticsIdentity } from "@/src/analytics/engine/siax-analytics-engine"; const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL; const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH; @@ -45,10 +45,10 @@ const DB = `opt_isolation_${Date.now().toString(36)}`; const RESTRICTED_USER = `opt_iso_${Date.now().toString(36)}`; const RESTRICTED_PW = `iso_pw_${Math.random().toString(36).slice(2, 12)}`; -const ALPHA: AnalyticsIdentity = { tenant_id: 't_alpha', project_id: 'p_main', principal_id: 'usr_alpha' }; -const BETA: AnalyticsIdentity = { tenant_id: 't_beta', project_id: 'p_main', principal_id: 'usr_beta' }; +const ALPHA: AnalyticsIdentity = { tenant_id: "t_alpha", project_id: "p_main", principal_id: "usr_alpha" }; +const BETA: AnalyticsIdentity = { tenant_id: "t_beta", project_id: "p_main", principal_id: "usr_beta" }; -const catalog = dimensionCatalogFrom({ severity: 'severity' }); +const catalog = dimensionCatalogFrom({ severity: "severity" }); const ALL_TABLES: readonly string[] = ANALYTICS_TABLES; /** Raw HTTP against the live server, as a named user. */ @@ -59,37 +59,34 @@ async function chQuery( settings: Record = {}, body?: string, ): Promise<{ ok: boolean; status: number; text: string }> { - const url = new URL(CLICKHOUSE_URL ?? 'http://skipped.invalid'); - url.searchParams.set('query', sql); + const url = new URL(CLICKHOUSE_URL ?? "http://skipped.invalid"); + url.searchParams.set("query", sql); for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v); for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); const headers: Record = {}; - if (auth) headers['Authorization'] = `Basic ${Buffer.from(auth).toString('base64')}`; - const res = await fetch(url.toString(), { method: 'POST', headers, body: body ?? '' }); + if (auth) headers["Authorization"] = `Basic ${Buffer.from(auth).toString("base64")}`; + const res = await fetch(url.toString(), { method: "POST", headers, body: body ?? "" }); return { ok: res.ok, status: res.status, text: await res.text() }; } function rowsOf(text: string): { grp?: string; value: number }[] { if (!text.trim()) return []; const parsed = JSON.parse(text) as { data?: { grp?: string; value: number | string }[] }; - return (parsed.data ?? []).map((r) => ({ - ...(r.grp !== undefined ? { grp: r.grp } : {}), - value: Number(r.value), - })); + return (parsed.data ?? []).map((r) => ({ ...(r.grp !== undefined ? { grp: r.grp } : {}), value: Number(r.value) })); } let engine: ClickHouseAnalyticsEngine; -run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { +run("BI-P011 tenant isolation against a LIVE ClickHouse", () => { beforeAll(async () => { engine = new ClickHouseAnalyticsEngine({ baseUrl: CLICKHOUSE_URL!, basicAuth: CLICKHOUSE_AUTH, database: DB, - dimensionCatalogs: { t_alpha: { severity: 'severity' }, t_beta: { severity: 'severity' } }, + dimensionCatalogs: { t_alpha: { severity: "severity" }, t_beta: { severity: "severity" } }, }); const health = await engine.health(); - expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ''}`).toBe(true); + expect(health.healthy, `ClickHouse not healthy: ${health.detail ?? ""}`).toBe(true); await engine.ensureSchema(); // Two tenants, deliberately different magnitudes so a leak is unmistakable @@ -98,28 +95,28 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { for (let i = 0; i < 50; i++) { rows.push( JSON.stringify({ - tenant_id: 't_alpha', - project_id: 'p_main', - dataset_id: 'ds_shared', - metric: 'rows_ingested', + tenant_id: "t_alpha", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", event_id: `a${i}`, - event_time: '2026-09-01 10:00:00.000', + event_time: "2026-09-01 10:00:00.000", value: 1, - dim_names: ['severity'], - dim_values: [i % 2 === 0 ? 'warning' : 'critical'], + dim_names: ["severity"], + dim_values: [i % 2 === 0 ? "warning" : "critical"], }), ); rows.push( JSON.stringify({ - tenant_id: 't_beta', - project_id: 'p_main', - dataset_id: 'ds_shared', - metric: 'rows_ingested', + tenant_id: "t_beta", + project_id: "p_main", + dataset_id: "ds_shared", + metric: "rows_ingested", event_id: `b${i}`, - event_time: '2026-09-01 10:00:00.000', + event_time: "2026-09-01 10:00:00.000", value: 1000, - dim_names: ['severity'], - dim_values: ['critical'], + dim_names: ["severity"], + dim_values: ["critical"], }), ); } @@ -128,7 +125,7 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { CLICKHOUSE_AUTH, {}, {}, - rows.join('\n'), + rows.join("\n"), ); expect(ins.ok, `seed insert failed: ${ins.text.slice(0, 200)}`).toBe(true); }, 120_000); @@ -137,10 +134,7 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { if (!CLICKHOUSE_URL) return; // Disposable by construction: drop the user, its policies, and the DB. for (const table of ANALYTICS_TABLES) { - await chQuery( - `DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, - CLICKHOUSE_AUTH, - ); + await chQuery(`DROP ROW POLICY IF EXISTS ${RESTRICTED_USER}_${table}_rp ON ${DB}.${table}`, CLICKHOUSE_AUTH); } await chQuery(`DROP USER IF EXISTS ${RESTRICTED_USER}`, CLICKHOUSE_AUTH); await chQuery(`DROP DATABASE IF EXISTS ${DB}`, CLICKHOUSE_AUTH); @@ -155,89 +149,89 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { const unsafeSql = `SELECT sum(value) AS value FROM ${DB}.opt_analytics_event ` + `WHERE metric = {metric:String} LIMIT 1 FORMAT JSON`; - const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: 'rows_ingested' }); + const res = await chQuery(unsafeSql, CLICKHOUSE_AUTH, { param_metric: "rows_ingested" }); expect(res.ok).toBe(true); // 50*1 (alpha) + 50*1000 (beta) = 50050 — both tenants, one result. expect(rowsOf(res.text)[0].value).toBe(50050); }); it("CLOSED, layer 2 PRESENT: the compiled query returns t_alpha's 50 and never t_beta's 50000", async () => { - const result = await engine.executeMetric(ALPHA, { metric: 'rows_ingested', aggregate: 'sum' }); + const result = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "sum" }); expect(result.rows[0].value).toBe(50); - const beta = await engine.executeMetric(BETA, { metric: 'rows_ingested', aggregate: 'sum' }); + const beta = await engine.executeMetric(BETA, { metric: "rows_ingested", aggregate: "sum" }); expect(beta.rows[0].value).toBe(50_000); }); it("tenant A querying tenant B's dataset id gets ZERO rows, not B's rows", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - dataset_id: 'ds_shared', + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_shared", }); expect(result.rows[0].value).toBe(50); // only alpha's 50, though beta shares the dataset id }); - it('a forged dataset id yields an empty result, never an error that leaks existence', async () => { + it("a forged dataset id yields an empty result, never an error that leaks existence", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - dataset_id: 'ds_that_does_not_exist_or_belongs_to_someone_else', + metric: "rows_ingested", + aggregate: "count", + dataset_id: "ds_that_does_not_exist_or_belongs_to_someone_else", }); expect(result.rows[0].value).toBe(0); }); it("a tenant with no data at all sees zero, not everyone's aggregate", async () => { - const ghost: AnalyticsIdentity = { tenant_id: 't_gamma', project_id: 'p_main', principal_id: 'usr_g' }; - const result = await engine.executeMetric(ghost, { metric: 'rows_ingested', aggregate: 'count' }); + const ghost: AnalyticsIdentity = { tenant_id: "t_gamma", project_id: "p_main", principal_id: "usr_g" }; + const result = await engine.executeMetric(ghost, { metric: "rows_ingested", aggregate: "count" }); expect(result.rows[0].value).toBe(0); }); it("group-by results carry only this tenant's dimension values", async () => { const result = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', - group_by: ['severity'], + metric: "rows_ingested", + aggregate: "count", + group_by: ["severity"], }); const total = result.rows.reduce((a, r) => a + r.value, 0); expect(total).toBe(50); // t_beta's rows are all "critical" with value 1000; if any leaked, the // critical bucket would exceed alpha's 25. - const critical = result.rows.find((r) => r.groups?.severity === 'critical'); + const critical = result.rows.find((r) => r.groups?.severity === "critical"); expect(critical?.value).toBe(25); }); - it('SQL/dimension/filter injection through the live path changes nothing', async () => { + it("SQL/dimension/filter injection through the live path changes nothing", async () => { await expect( engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'sum', + metric: "rows_ingested", + aggregate: "sum", group_by: ["severity'; SELECT tenant_id FROM system.tables; --"], }), - ).rejects.toMatchObject({ code: 'INVALID_REQUEST' }); + ).rejects.toMatchObject({ code: "INVALID_REQUEST" }); // A hostile VALUE reaches the server as a bound parameter and simply // matches nothing. const injected = await engine.executeMetric(ALPHA, { - metric: 'rows_ingested', - aggregate: 'count', + metric: "rows_ingested", + aggregate: "count", filters: { severity: "warning' OR 1=1 --" }, }); expect(injected.rows[0].value).toBe(0); // And the table is still there afterwards. - const still = await engine.executeMetric(ALPHA, { metric: 'rows_ingested', aggregate: 'count' }); + const still = await engine.executeMetric(ALPHA, { metric: "rows_ingested", aggregate: "count" }); expect(still.rows[0].value).toBe(50); }); - it('raw SQL attempt: the analyst-SQL surface is refused, not silently executed', async () => { + it("raw SQL attempt: the analyst-SQL surface is refused, not silently executed", async () => { await expect( engine.executeQuery(ALPHA, { sql: `SELECT * FROM ${DB}.opt_analytics_event`, dataset_ids: [] }), - ).rejects.toMatchObject({ code: 'CAPABILITY_UNAVAILABLE' }); + ).rejects.toMatchObject({ code: "CAPABILITY_UNAVAILABLE" }); }); // ── layer 3: restricted users + row policies ───────────────────────────── - it('HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant', async () => { + it("HOLE, layer 3 ABSENT: an unrestricted user with no predicate sees every tenant", async () => { const res = await chQuery( `SELECT uniqExact(tenant_id) AS value FROM ${DB}.opt_analytics_event FORMAT JSON`, CLICKHOUSE_AUTH, @@ -246,10 +240,10 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { expect(rowsOf(res.text)[0].value).toBe(2); // t_alpha AND t_beta }); - it('CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant', async () => { + it("CLOSED, layer 3 PRESENT: the restricted user runs the SAME predicate-free statement and sees ONE tenant", async () => { for (const stmt of restrictedTenantUserDdl({ db: DB, - tenantId: 't_alpha', + tenantId: "t_alpha", userName: RESTRICTED_USER, password: RESTRICTED_PW, tables: ALL_TABLES, @@ -274,24 +268,14 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { expect(rowsOf(sum.text)[0].value).toBe(50); // alpha's total, not 50050 }); - it('the restricted user cannot write, and cannot reach a table it was not granted', async () => { + it("the restricted user cannot write, and cannot reach a table it was not granted", async () => { const restrictedAuth = `${RESTRICTED_USER}:${RESTRICTED_PW}`; const write = await chQuery( `INSERT INTO ${DB}.opt_analytics_event FORMAT JSONEachRow`, restrictedAuth, {}, {}, - JSON.stringify({ - tenant_id: 't_beta', - project_id: 'p', - dataset_id: 'd', - metric: 'm', - event_id: 'x', - event_time: '2026-09-01 00:00:00.000', - value: 1, - dim_names: [], - dim_values: [], - }), + JSON.stringify({ tenant_id: "t_beta", project_id: "p", dataset_id: "d", metric: "m", event_id: "x", event_time: "2026-09-01 00:00:00.000", value: 1, dim_names: [], dim_values: [] }), ); expect(write.ok).toBe(false); @@ -305,18 +289,18 @@ run('BI-P011 tenant isolation against a LIVE ClickHouse', () => { for (const table of ALL_TABLES) { const res = await chQuery(sortingKeySql(DB, table), CLICKHOUSE_AUTH); expect(res.ok).toBe(true); - expect(res.text.trim().startsWith('tenant_id'), `${table} sorting key: ${res.text.trim()}`).toBe(true); + expect(res.text.trim().startsWith("tenant_id"), `${table} sorting key: ${res.text.trim()}`).toBe(true); } }); - it('the tenant value never appears in the statement text the server receives', async () => { + it("the tenant value never appears in the statement text the server receives", async () => { const compiled = compileMetricQuery( - { tenant_id: 't_alpha', metric: 'rows_ingested', aggregate: 'sum' }, + { tenant_id: "t_alpha", metric: "rows_ingested", aggregate: "sum" }, DB, catalog, DEFAULT_BUDGETS.INTERACTIVE.maxResultRows, ); - expect(compiled.sql).not.toContain('t_alpha'); + expect(compiled.sql).not.toContain("t_alpha"); const res = await chQuery(compiled.sql, CLICKHOUSE_AUTH, compiled.params); expect(res.ok).toBe(true); expect(rowsOf(res.text)[0].value).toBe(50); diff --git a/tests/integration/search-opensearch.integration.test.ts b/tests/integration/search-opensearch.integration.test.ts index 6ab96a4..d8f20e6 100644 --- a/tests/integration/search-opensearch.integration.test.ts +++ b/tests/integration/search-opensearch.integration.test.ts @@ -15,10 +15,10 @@ * - full projection-disaster recovery: delete EVERYTHING, rebuild from * the canonical source, identical results (§68-style, scoped to 0pt) */ -import { describe, it, expect, beforeAll, afterAll } from 'vitest'; -import { HttpOpenSearchAdapter } from '@/src/search/opensearch-adapter'; -import { SearchService, type DocumentSource } from '@/src/search/search-service'; -import type { SearchDocument } from '@/src/search/search-types'; +import { describe, it, expect, beforeAll, afterAll } from "vitest"; +import { HttpOpenSearchAdapter } from "@/src/search/opensearch-adapter"; +import { SearchService, type DocumentSource } from "@/src/search/search-service"; +import type { SearchDocument } from "@/src/search/search-types"; const OPENSEARCH_URL = process.env.OPENSEARCH_URL; const run = OPENSEARCH_URL ? describe : describe.skip; @@ -27,52 +27,52 @@ const INDEX = `itest-${Date.now().toString(36)}`; const DOCS: SearchDocument[] = [ { - doc_id: 'c1', - tenant_id: 't_alpha', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal', 'customer_deliverable'], - source_refs: ['0pt:source:src_bolagsverket'], - payload: { name: 'Volvo Personvagnar AB', org_nr: '5561074967', employees: 20000 }, + doc_id: "c1", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal", "customer_deliverable"], + source_refs: ["0pt:source:src_bolagsverket"], + payload: { name: "Volvo Personvagnar AB", org_nr: "5561074967", employees: 20000 }, }, { - doc_id: 'c2', - tenant_id: 't_alpha', - purpose_tags: ['compliance'], - rights_tags: ['internal'], - payload: { name: 'Volvo Lastvagnar AB', org_nr: '5560135508', employees: 15000 }, + doc_id: "c2", + tenant_id: "t_alpha", + purpose_tags: ["compliance"], + rights_tags: ["internal"], + payload: { name: "Volvo Lastvagnar AB", org_nr: "5560135508", employees: 15000 }, }, { - doc_id: 'c3', - tenant_id: 't_beta', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal'], - payload: { name: 'Volvo Cars Retail', org_nr: '5569999999', employees: 300 }, + doc_id: "c3", + tenant_id: "t_beta", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Volvo Cars Retail", org_nr: "5569999999", employees: 300 }, }, ]; function source(docs: SearchDocument[]): DocumentSource { return { - name: 'itest:canonical-array', + name: "itest:canonical-array", async *iterate() { yield* docs; }, }; } -run('Search capability against a LIVE OpenSearch', () => { +run("Search capability against a LIVE OpenSearch", () => { // The skip-variant still evaluates this callback body, so the constructor // must tolerate the env-less case (the tests themselves never run then). - const adapter = new HttpOpenSearchAdapter({ baseUrl: OPENSEARCH_URL ?? 'http://skipped.invalid' }); + const adapter = new HttpOpenSearchAdapter({ baseUrl: OPENSEARCH_URL ?? "http://skipped.invalid" }); const search = new SearchService(adapter); beforeAll(async () => { const health = await adapter.health(); - expect(['green', 'yellow']).toContain(health.status); + expect(["green", "yellow"]).toContain(health.status); search.registerIndexDefinition({ index_name: INDEX, version: 1, - fields: { name: 'text', org_nr: 'keyword', employees: 'long' }, - default_search_fields: ['name'], + fields: { name: "text", org_nr: "keyword", employees: "long" }, + default_search_fields: ["name"], }); }, 30_000); @@ -82,7 +82,7 @@ run('Search capability against a LIVE OpenSearch', () => { } }); - it('rebuild g1 from canonical source; alias serves it', async () => { + it("rebuild g1 from canonical source; alias serves it", async () => { const result = await search.rebuild(INDEX, source(DOCS)); expect(result.new_generation).toBe(1); expect(result.docs_indexed).toBe(3); @@ -91,65 +91,57 @@ run('Search capability against a LIVE OpenSearch', () => { expect(state.active_generation).toBe(1); }, 60_000); - it('REAL tenant isolation: t_alpha never sees t_beta documents', async () => { + it("REAL tenant isolation: t_alpha never sees t_beta documents", async () => { const alpha = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'volvo', + tenant_id: "t_alpha", + purpose: "company_enrichment", + text: "volvo", }); - expect(alpha.hits.map((h) => h.doc_id)).toEqual(['c1']); // c2 wrong purpose, c3 wrong tenant + expect(alpha.hits.map((h) => h.doc_id)).toEqual(["c1"]); // c2 wrong purpose, c3 wrong tenant const beta = await search.query(INDEX, { - tenant_id: 't_beta', - purpose: 'company_enrichment', - text: 'volvo', + tenant_id: "t_beta", + purpose: "company_enrichment", + text: "volvo", }); - expect(beta.hits.map((h) => h.doc_id)).toEqual(['c3']); + expect(beta.hits.map((h) => h.doc_id)).toEqual(["c3"]); // Cross-tenant attack shape: a filter cannot break out of the tenant term. const attack = await search.query(INDEX, { - tenant_id: 't_beta', - purpose: 'company_enrichment', - filters: { org_nr: '5561074967' }, // t_alpha's org_nr + tenant_id: "t_beta", + purpose: "company_enrichment", + filters: { org_nr: "5561074967" }, // t_alpha's org_nr }); expect(attack.total).toBe(0); }); - it('purpose + rights filtering hold on the real engine', async () => { - const compliance = await search.query(INDEX, { tenant_id: 't_alpha', purpose: 'compliance' }); - expect(compliance.hits.map((h) => h.doc_id)).toEqual(['c2']); + it("purpose + rights filtering hold on the real engine", async () => { + const compliance = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "compliance" }); + expect(compliance.hits.map((h) => h.doc_id)).toEqual(["c2"]); const deliverable = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - rights_tag: 'customer_deliverable', + tenant_id: "t_alpha", + purpose: "company_enrichment", + rights_tag: "customer_deliverable", }); - expect(deliverable.hits.map((h) => h.doc_id)).toEqual(['c1']); + expect(deliverable.hits.map((h) => h.doc_id)).toEqual(["c1"]); }); - it('generation rebuild + switch + rollback (§42)', async () => { + it("generation rebuild + switch + rollback (§42)", async () => { const updated = DOCS.map((d) => - d.doc_id === 'c1' ? { ...d, payload: { ...d.payload, name: 'Volvo Personvagnar AB (uppdaterad)' } } : d, + d.doc_id === "c1" ? { ...d, payload: { ...d.payload, name: "Volvo Personvagnar AB (uppdaterad)" } } : d ); const r2 = await search.rebuild(INDEX, source(updated)); expect(r2.new_generation).toBe(2); - let res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'uppdaterad', - }); + let res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); expect(res.total).toBe(1); // Rollback to g1 — the old generation still exists and serves the alias. await search.switchGeneration(INDEX, 1); - res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'uppdaterad', - }); + res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "uppdaterad" }); expect(res.total).toBe(0); await search.switchGeneration(INDEX, 2); }, 60_000); - it('projection disaster: delete ALL generations, rebuild from canonical, results identical (§68)', async () => { + it("projection disaster: delete ALL generations, rebuild from canonical, results identical (§68)", async () => { for (const idx of await adapter.listIndices(`opt-${INDEX}--g*`)) { await adapter.deleteIndex(idx); } @@ -157,32 +149,24 @@ run('Search capability against a LIVE OpenSearch', () => { const rebuilt = await search.rebuild(INDEX, source(DOCS)); expect(rebuilt.docs_indexed).toBe(3); - const res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'volvo', - }); - expect(res.hits.map((h) => h.doc_id)).toEqual(['c1']); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "volvo" }); + expect(res.hits.map((h) => h.doc_id)).toEqual(["c1"]); }, 60_000); - it('incremental indexDocuments into the active generation', async () => { + it("incremental indexDocuments into the active generation", async () => { const out = await search.indexDocuments(INDEX, [ { - doc_id: 'c4', - tenant_id: 't_alpha', - purpose_tags: ['company_enrichment'], - rights_tags: ['internal'], - payload: { name: 'Polestar Performance AB', org_nr: '5568100000', employees: 2500 }, + doc_id: "c4", + tenant_id: "t_alpha", + purpose_tags: ["company_enrichment"], + rights_tags: ["internal"], + payload: { name: "Polestar Performance AB", org_nr: "5568100000", employees: 2500 }, }, ]); expect(out.indexed).toBe(1); const state = await search.generationState(INDEX); await adapter.refresh(search.physicalIndexFor(INDEX, state.active_generation!)); - const res = await search.query(INDEX, { - tenant_id: 't_alpha', - purpose: 'company_enrichment', - text: 'polestar', - }); + const res = await search.query(INDEX, { tenant_id: "t_alpha", purpose: "company_enrichment", text: "polestar" }); expect(res.total).toBe(1); }); });