fix(analytics): revert 18 files reformatted by mistake, and actually apply the Iceberg fix a previous message claimed
CI (SIAX Cloud) / integration (push) Successful in 40s
CI (SIAX Cloud) / quality (push) Successful in 55s
CI (SIAX Cloud) / security (push) Successful in 21s
Security Scan (SIAX Cloud) / security (push) Successful in 22s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
CI (SIAX Cloud) / quality (pull_request) Successful in 54s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 21s
CI (SIAX Cloud) / security (pull_request) Successful in 25s

Two corrections, both of my own errors.

1. SCOPE CREEP, NOW REVERTED. Running `prettier --write` on whole directories
   reformatted 18 files this branch had no business touching — the entire
   merged BI-P002..P012 core (arrow/, domain/, engine/, repositories/, five
   analytics integration suites) and, absurdly,
   search-opensearch.integration.test.ts, which has nothing to do with this
   work. The changes were purely stylistic quote churn, but they added ~1,400
   lines of noise to the diff, made the PR harder to review honestly, and
   invited conflicts with the four other lanes working in this estate. All 18
   are now byte-identical to origin/main again (verified with `git diff
   origin/main --name-only`, which reports nothing for them). Formatting is now
   applied only to the files this branch actually owns.

2. A COMMIT MESSAGE THAT DESCRIBED A FIX THAT WAS NOT IN THE CODE. Commit
   3ee0230 said the "does NOT ship the Iceberg table" assertion had been moved
   from a filesystem check to `git ls-files`. It had not. The patch anchor used
   double quotes, prettier had already converted that file to single quotes, so
   the replacement silently matched nothing and the script printed its success
   line anyway. The suite then passed — but only because I had just deleted the
   fixture directory, so the OLD filesystem assertion happened to be true.

   That is precisely the failure mode this branch keeps writing tests against:
   a green run that means something different from what it appears to mean. The
   fix is now genuinely applied and verified the way it should have been the
   first time — by running the suite WITH the Iceberg fixture present on disk,
   which is the case that previously failed. It passes, and it now also asserts
   that the parquet/csv fixtures ARE tracked, so a version shipping no fixture
   at all would fail rather than pass vacuously.

957 tests pass with ClickHouse, Postgres and DuckDB live. typecheck, eslint and
prettier clean on the files this branch owns.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
2026-09-07 03:16:30 +02:00
parent ba5c796b36
commit c864c8c417
23 changed files with 800 additions and 988 deletions
+2 -2
View File
@@ -148,14 +148,14 @@ export class FlatBufferBuilder {
}
startObject(numFields: number): void {
if (this.inObject) throw new Error('nested startObject');
if (this.inObject) throw new Error("nested startObject");
this.vtable = new Array<number>(numFields).fill(0);
this.objectStart = this.used;
this.inObject = true;
}
endObject(): Offset {
if (!this.inObject) throw new Error('endObject outside an object');
if (!this.inObject) throw new Error("endObject outside an object");
this.prep(SIZEOF_INT);
this.writeInt32(0); // soffset placeholder, patched below
const objectOffset = this.used;
+55 -63
View File
@@ -29,8 +29,8 @@
* padded so the body starts 8-byte aligned, then the body buffers.
* Terminated by 0xFFFFFFFF 0x00000000.
*/
import { FlatBufferBuilder, type Offset } from './flatbuffer-builder';
import { SiaxError } from '../../platform/errors';
import { FlatBufferBuilder, type Offset } from "./flatbuffer-builder";
import { SiaxError } from "../../platform/errors";
// ── FlatBuffer enum values from org.apache.arrow.flatbuf ────────────────────
@@ -51,13 +51,13 @@ const PRECISION_DOUBLE = 2;
// every 64-bit constant is built with BigInt(...) instead. Behaviour is
// identical — this is a compiler-target accommodation, not a semantic choice.
const BIG_ZERO = BigInt(0);
const INT64_MAX = BigInt('9223372036854775807');
const INT64_MIN = BigInt('-9223372036854775808');
const INT64_MAX = BigInt("9223372036854775807");
const INT64_MIN = BigInt("-9223372036854775808");
const TIME_UNIT_MILLISECOND = 1;
// ── public column model ────────────────────────────────────────────────────
export type ArrowColumnType = 'int64' | 'float64' | 'utf8' | 'bool' | 'timestamp_ms';
export type ArrowColumnType = "int64" | "float64" | "utf8" | "bool" | "timestamp_ms";
export interface ArrowColumn {
name: string;
@@ -78,39 +78,39 @@ export const ARROW_TYPE_SUPPORT: {
note: string;
}[] = [
{
clickhouse: 'Int64 / UInt32 / Int32',
arrow: 'int64',
duckdb: 'BIGINT',
python: 'int',
note: 'exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped',
clickhouse: "Int64 / UInt32 / Int32",
arrow: "int64",
duckdb: "BIGINT",
python: "int",
note: "exact; UInt64 above 2^63 is NOT supported and is rejected rather than silently wrapped",
},
{
clickhouse: 'Float64',
arrow: 'float64',
duckdb: 'DOUBLE',
python: 'float',
note: 'exact IEEE-754 round trip',
clickhouse: "Float64",
arrow: "float64",
duckdb: "DOUBLE",
python: "float",
note: "exact IEEE-754 round trip",
},
{
clickhouse: 'String / LowCardinality(String)',
arrow: 'utf8',
duckdb: 'VARCHAR',
python: 'str',
note: 'LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not',
clickhouse: "String / LowCardinality(String)",
arrow: "utf8",
duckdb: "VARCHAR",
python: "str",
note: "LowCardinality is decoded to plain utf8 — dictionary encoding is lost, values are not",
},
{
clickhouse: 'Bool / UInt8(0|1)',
arrow: 'bool',
duckdb: 'BOOLEAN',
python: 'bool',
note: 'bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64',
clickhouse: "Bool / UInt8(0|1)",
arrow: "bool",
duckdb: "BOOLEAN",
python: "bool",
note: "bit-packed one bit per value; ClickHouse UInt8 other than 0/1 is NOT a bool and must be sent as int64",
},
{
clickhouse: "DateTime64(3,'UTC')",
arrow: 'timestamp_ms',
duckdb: 'TIMESTAMP',
python: 'datetime',
note: 'millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set',
arrow: "timestamp_ms",
duckdb: "TIMESTAMP",
python: "datetime",
note: "millisecond precision; DateTime64(6/9) loses sub-ms and is therefore NOT in the supported set",
},
];
@@ -118,7 +118,7 @@ export const ARROW_TYPE_SUPPORT: {
/** ClickHouse output format producing an Arrow IPC *stream*. */
export function clickhouseArrowStreamFormat(): string {
return 'ArrowStream';
return "ArrowStream";
}
/**
@@ -129,8 +129,8 @@ export function clickhouseArrowStreamFormat(): string {
*/
export function arrowStreamSettings(): Record<string, string> {
return {
output_format_arrow_string_as_string: '1',
output_format_arrow_low_cardinality_as_dictionary: '0',
output_format_arrow_string_as_string: "1",
output_format_arrow_low_cardinality_as_dictionary: "0",
};
}
@@ -165,24 +165,20 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount:
const n = col.values.length;
switch (col.type) {
case 'int64': {
case "int64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const b = typeof v === 'bigint' ? v : BigInt(Math.trunc(Number(v)));
const b = typeof v === "bigint" ? v : BigInt(Math.trunc(Number(v)));
if (b > INT64_MAX || b < INT64_MIN) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`value out of int64 range in column "${col.name}"`,
);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `value out of int64 range in column "${col.name}"`);
}
dv.setBigInt64(i * 8, b, true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case 'float64': {
case "float64": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
@@ -191,17 +187,17 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount:
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case 'timestamp_ms': {
case "timestamp_ms": {
const data = new Uint8Array(alignTo8(n * 8));
const dv = new DataView(data.buffer);
col.values.forEach((v, i) => {
if (v === null || v === undefined) return;
const ms = v instanceof Date ? v.getTime() : typeof v === 'bigint' ? Number(v) : Number(v);
const ms = v instanceof Date ? v.getTime() : typeof v === "bigint" ? Number(v) : Number(v);
dv.setBigInt64(i * 8, BigInt(Math.trunc(ms)), true);
});
return { buffers: [validity, { bytes: data, length: n * 8 }], nullCount };
}
case 'bool': {
case "bool": {
const byteLen = Math.ceil(n / 8);
const data = new Uint8Array(alignTo8(byteLen));
col.values.forEach((v, i) => {
@@ -210,7 +206,7 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount:
});
return { buffers: [validity, { bytes: data, length: byteLen }], nullCount };
}
case 'utf8': {
case "utf8": {
const encoded = col.values.map((v) =>
v === null || v === undefined ? new Uint8Array(0) : new TextEncoder().encode(String(v)),
);
@@ -232,36 +228,36 @@ function encodeColumn(col: ArrowColumn): { buffers: EncodedBuffer[]; nullCount:
}
default: {
const never: never = col.type;
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType: number; offset: Offset } {
switch (type) {
case 'int64': {
case "int64": {
b.startObject(2);
b.addInt32(0, 64, 0); // bitWidth
b.addInt8(1, 1, 0); // is_signed
return { typeType: TYPE_INT, offset: b.endObject() };
}
case 'float64': {
case "float64": {
b.startObject(1);
b.addInt16(0, PRECISION_DOUBLE, 0); // precision = DOUBLE
return { typeType: TYPE_FLOATING_POINT, offset: b.endObject() };
}
case 'utf8': {
case "utf8": {
b.startObject(0);
return { typeType: TYPE_UTF8, offset: b.endObject() };
}
case 'bool': {
case "bool": {
b.startObject(0);
return { typeType: TYPE_BOOL, offset: b.endObject() };
}
case 'timestamp_ms': {
case "timestamp_ms": {
// Build the timezone string BEFORE startObject — FlatBuffers forbids
// creating a nested object while a table is open.
const tz = b.createString('UTC');
const tz = b.createString("UTC");
b.startObject(2);
b.addInt16(0, TIME_UNIT_MILLISECOND, 0);
b.addOffset(1, tz, 0);
@@ -269,7 +265,7 @@ function buildTypeTable(b: FlatBufferBuilder, type: ArrowColumnType): { typeType
}
default: {
const never: never = type;
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unsupported arrow type ${String(never)}`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unsupported arrow type ${String(never)}`);
}
}
}
@@ -365,14 +361,14 @@ function framedMessage(metadata: Uint8Array, body: Uint8Array): Uint8Array {
*/
export function encodeArrowStream(columns: readonly ArrowColumn[]): Uint8Array {
if (columns.length === 0) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'an Arrow stream needs at least one column');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "an Arrow stream needs at least one column");
}
const rowCount = columns[0].values.length;
for (const c of columns) {
if (c.values.length !== rowCount) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`column "${c.name}" has ${c.values.length} values, expected ${rowCount}`,
);
}
@@ -500,11 +496,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo {
while (pos + 8 <= bytes.byteLength) {
const continuation = dv.getUint32(pos, true);
if (continuation !== 0xffffffff) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
'not an Arrow IPC stream (missing continuation marker)',
);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "not an Arrow IPC stream (missing continuation marker)");
}
const metaLen = dv.getInt32(pos + 4, true);
if (metaLen === 0) break; // end of stream
@@ -521,7 +513,7 @@ export function decodeArrowStream(bytes: Uint8Array): ArrowStreamInfo {
fields = [];
for (let i = 0; i < n; i++) {
const f = schema.vectorElementTable(1, i);
fields.push({ name: f.readString(0) ?? '', typeType: f.readInt8(2, 0) });
fields.push({ name: f.readString(0) ?? "", typeType: f.readInt8(2, 0) });
}
}
} else if (headerType === MESSAGE_HEADER_RECORD_BATCH) {
@@ -546,8 +538,8 @@ export function assertArrowWithinResultBudget(bytes: Uint8Array, maxResultRows:
const info = decodeArrowStream(bytes);
if (info.rowCount > maxResultRows) {
throw new SiaxError(
'QUOTA_EXCEEDED',
'0pt.analytics',
"QUOTA_EXCEEDED",
"0pt.analytics",
`Arrow result has ${info.rowCount} rows, over the ${maxResultRows}-row budget`,
);
}
+36 -35
View File
@@ -25,7 +25,7 @@
*
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
*/
import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../platform/errors';
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
export interface ClickHouseAnalyticsOptions {
baseUrl: string;
@@ -52,7 +52,7 @@ export interface MetricQuery {
tenant_id: string;
dataset_id?: string;
metric: string;
aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count';
aggregate: "sum" | "avg" | "min" | "max" | "count";
/** Group by one dimension key (validated identifier). */
group_by?: string;
from_time?: string;
@@ -77,9 +77,9 @@ export class ClickHouseAnalytics {
private readonly maxResultRows: number;
constructor(opts: ClickHouseAnalyticsOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, '');
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined;
this.database = opts.database ?? 'opt_analytics';
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
this.database = opts.database ?? "opt_analytics";
this.timeoutMs = opts.timeoutMs ?? 30_000;
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
this.maxResultRows = opts.maxResultRows ?? 10_000;
@@ -100,7 +100,7 @@ export class ClickHouseAnalytics {
ENGINE = MergeTree()
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, metric, dataset_id, event_time)
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`,
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
);
}
@@ -109,14 +109,10 @@ export class ClickHouseAnalytics {
if (events.length === 0) return { ingested: 0 };
for (const e of events) {
if (!e.tenant_id?.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every analytics event needs tenant_id');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
}
if (!Number.isFinite(e.value)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`metric value for "${e.metric}" must be finite`,
);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
}
}
const rows = events
@@ -126,11 +122,11 @@ export class ClickHouseAnalytics {
dataset_id: e.dataset_id,
metric: e.metric,
value: e.value,
event_time: (e.event_time ?? new Date().toISOString()).replace('T', ' ').replace(/Z$/, ''),
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
dimensions: e.dimensions ?? {},
}),
})
)
.join('\n');
.join("\n");
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
return { ingested: events.length };
}
@@ -138,40 +134,41 @@ export class ClickHouseAnalytics {
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
if (!q.tenant_id?.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every metric query');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
}
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid group_by identifier`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
}
const agg = q.aggregate === 'count' ? 'count()' : `${q.aggregate}(value)`;
const agg =
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
// Parameterized query — values travel as HTTP params, never spliced.
const conditions = ['tenant_id = {tenant:String}', 'metric = {metric:String}'];
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
if (q.dataset_id) {
conditions.push('dataset_id = {dataset:String}');
conditions.push("dataset_id = {dataset:String}");
params.param_dataset = q.dataset_id;
}
if (q.from_time) {
conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})');
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
params.param_fromTime = q.from_time;
}
if (q.to_time) {
conditions.push('event_time < parseDateTime64BestEffort({toTime:String})');
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
params.param_toTime = q.to_time;
}
const sql =
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ''}${agg} AS value ` +
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
`FROM ${this.database}.opt_metric_events ` +
`WHERE ${conditions.join(' AND ')} ` +
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : '') +
`WHERE ${conditions.join(" AND ")} ` +
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
`LIMIT ${limit} FORMAT JSON`;
const json = (await this.exec(sql, undefined, {
readonly: '1',
readonly: "1",
max_execution_time: String(this.maxExecutionTimeSec),
max_result_rows: String(this.maxResultRows),
...params,
@@ -185,7 +182,7 @@ export class ClickHouseAnalytics {
async health(): Promise<boolean> {
try {
await this.exec('SELECT 1 FORMAT JSON');
await this.exec("SELECT 1 FORMAT JSON");
return true;
} catch {
return false;
@@ -193,24 +190,28 @@ export class ClickHouseAnalytics {
}
// -----------------------------------------------------------------------
private async exec(sql: string, body?: string, settings?: Record<string, string>): Promise<unknown> {
private async exec(
sql: string,
body?: string,
settings?: Record<string, string>
): Promise<unknown> {
const url = new URL(this.baseUrl);
url.searchParams.set('query', sql);
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
const headers: Record<string, string> = {};
if (this.authHeader) headers['Authorization'] = this.authHeader;
if (this.authHeader) headers["Authorization"] = this.authHeader;
let res: Response;
try {
res = await fetch(url.toString(), {
method: 'POST',
method: "POST",
headers,
body: body ?? '',
body: body ?? "",
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
}
const text = await res.text();
if (!res.ok) {
@@ -218,8 +219,8 @@ export class ClickHouseAnalytics {
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
'clickhouse',
`query failed (CH code ${code})`,
"clickhouse",
`query failed (CH code ${code})`
);
}
if (!text) return undefined;
+66 -110
View File
@@ -18,10 +18,10 @@
* shipped under a minor bump is exactly the failure this gate exists to
* catch, so it is reported as a violation rather than tolerated.
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { AnalyticsFieldType, FieldRole, PiiClassification } from './entities';
import type { AnalyticsTier } from './tiers';
import crypto from "node:crypto";
import { SiaxError } from "../../platform/errors";
import type { AnalyticsFieldType, FieldRole, PiiClassification } from "./entities";
import type { AnalyticsTier } from "./tiers";
export interface ContractField {
name: string;
@@ -38,7 +38,7 @@ export interface ContractQualityRule {
rule_id: string;
kind: string;
field_name?: string;
severity: 'warn' | 'block';
severity: "warn" | "block";
}
/** BI-P004's named contents, one field each. */
@@ -77,18 +77,14 @@ const SEMVER_RE = /^(\d+)\.(\d+)\.(\d+)$/;
export function parseSemVer(v: string): SemVer {
const m = SEMVER_RE.exec(v);
if (!m) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`contract version "${v}" is not major.minor.patch`,
);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract version "${v}" is not major.minor.patch`);
}
return { major: Number(m[1]), minor: Number(m[2]), patch: Number(m[3]) };
}
// ── change classification ──────────────────────────────────────────────────
export type ChangeSeverity = 'compatible' | 'breaking';
export type ChangeSeverity = "compatible" | "breaking";
export interface ContractChange {
severity: ChangeSeverity;
@@ -104,7 +100,7 @@ export interface ContractChange {
* Everything absent from this table is breaking.
*/
const SAFE_TYPE_WIDENING: Partial<Record<AnalyticsFieldType, readonly AnalyticsFieldType[]>> = {
date: ['timestamp'],
date: ["timestamp"],
};
function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType): boolean {
@@ -136,9 +132,9 @@ function typeChangeIsCompatible(from: AnalyticsFieldType, to: AnalyticsFieldType
export function diffContracts(previous: DatasetContract, next: DatasetContract): ContractChange[] {
if (previous.dataset_id !== next.dataset_id) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
'contract compatibility is only defined between versions of the SAME dataset',
"INVALID_REQUEST",
"0pt.analytics",
"contract compatibility is only defined between versions of the SAME dataset",
);
}
const changes: ContractChange[] = [];
@@ -148,49 +144,44 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
for (const [name, pf] of prevFields) {
const nf = nextFields.get(name);
if (!nf) {
changes.push({
severity: 'breaking',
code: 'field_removed',
field: name,
detail: `field "${name}" removed`,
});
changes.push({ severity: "breaking", code: "field_removed", field: name, detail: `field "${name}" removed` });
continue;
}
if (!typeChangeIsCompatible(pf.type, nf.type)) {
changes.push({
severity: 'breaking',
code: 'field_type_changed',
severity: "breaking",
code: "field_type_changed",
field: name,
detail: `field "${name}" type ${pf.type} -> ${nf.type}`,
});
} else if (pf.type !== nf.type) {
changes.push({
severity: 'compatible',
code: 'field_type_widened',
severity: "compatible",
code: "field_type_widened",
field: name,
detail: `field "${name}" widened ${pf.type} -> ${nf.type}`,
});
}
if (!pf.nullable && nf.nullable) {
changes.push({
severity: 'breaking',
code: 'field_became_nullable',
severity: "breaking",
code: "field_became_nullable",
field: name,
detail: `field "${name}" became nullable`,
});
}
if (pf.nullable && !nf.nullable) {
changes.push({
severity: 'compatible',
code: 'field_became_required',
severity: "compatible",
code: "field_became_required",
field: name,
detail: `field "${name}" is no longer nullable`,
});
}
if (pf.role !== nf.role) {
changes.push({
severity: 'breaking',
code: 'field_role_changed',
severity: "breaking",
code: "field_role_changed",
field: name,
detail: `field "${name}" role ${pf.role} -> ${nf.role}`,
});
@@ -201,8 +192,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
if (prevFields.has(name)) continue;
if (nf.nullable) {
changes.push({
severity: 'compatible',
code: 'nullable_field_added',
severity: "compatible",
code: "nullable_field_added",
field: name,
detail: `nullable field "${name}" added`,
});
@@ -210,8 +201,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
// A required new field breaks producers, and breaks readers that
// reconstruct rows (they now have no value to supply).
changes.push({
severity: 'breaking',
code: 'required_field_added',
severity: "breaking",
code: "required_field_added",
field: name,
detail: `required (non-nullable) field "${name}" added`,
});
@@ -223,75 +214,51 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
previous.primary_identifiers.every((v, i) => next.primary_identifiers[i] === v);
if (!idsEqual) {
changes.push({
severity: 'breaking',
code: 'primary_identifier_changed',
detail: `primary identifier [${previous.primary_identifiers.join(',')}] -> [${next.primary_identifiers.join(',')}]`,
severity: "breaking",
code: "primary_identifier_changed",
detail: `primary identifier [${previous.primary_identifiers.join(",")}] -> [${next.primary_identifiers.join(",")}]`,
});
}
for (const d of previous.dimensions) {
if (!next.dimensions.includes(d)) {
changes.push({
severity: 'breaking',
code: 'dimension_removed',
field: d,
detail: `dimension "${d}" removed`,
});
changes.push({ severity: "breaking", code: "dimension_removed", field: d, detail: `dimension "${d}" removed` });
}
}
for (const d of next.dimensions) {
if (!previous.dimensions.includes(d)) {
changes.push({
severity: 'compatible',
code: 'dimension_added',
field: d,
detail: `dimension "${d}" added`,
});
changes.push({ severity: "compatible", code: "dimension_added", field: d, detail: `dimension "${d}" added` });
}
}
for (const m of previous.measures) {
if (!next.measures.includes(m)) {
changes.push({
severity: 'breaking',
code: 'measure_removed',
field: m,
detail: `measure "${m}" removed`,
});
changes.push({ severity: "breaking", code: "measure_removed", field: m, detail: `measure "${m}" removed` });
}
}
for (const m of next.measures) {
if (!previous.measures.includes(m)) {
changes.push({
severity: 'compatible',
code: 'measure_added',
field: m,
detail: `measure "${m}" added`,
});
changes.push({ severity: "compatible", code: "measure_added", field: m, detail: `measure "${m}" added` });
}
}
if (next.freshness_slo_seconds > previous.freshness_slo_seconds) {
changes.push({
severity: 'breaking',
code: 'freshness_slo_loosened',
severity: "breaking",
code: "freshness_slo_loosened",
detail: `freshness SLO ${previous.freshness_slo_seconds}s -> ${next.freshness_slo_seconds}s`,
});
}
if (next.retention_days < previous.retention_days) {
changes.push({
severity: 'breaking',
code: 'retention_shortened',
severity: "breaking",
code: "retention_shortened",
detail: `retention ${previous.retention_days}d -> ${next.retention_days}d`,
});
}
for (const p of previous.allowed_purposes) {
if (!next.allowed_purposes.includes(p)) {
changes.push({
severity: 'breaking',
code: 'purpose_removed',
detail: `allowed purpose "${p}" removed`,
});
changes.push({ severity: "breaking", code: "purpose_removed", detail: `allowed purpose "${p}" removed` });
}
}
@@ -303,8 +270,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
};
if (PII_RANK[next.pii_classification] > PII_RANK[previous.pii_classification]) {
changes.push({
severity: 'breaking',
code: 'pii_classification_raised',
severity: "breaking",
code: "pii_classification_raised",
detail: `PII classification ${previous.pii_classification} -> ${next.pii_classification}`,
});
}
@@ -312,16 +279,16 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
const prevRules = new Map(previous.quality_rules.map((r) => [r.rule_id, r]));
for (const [id, pr] of prevRules) {
const nr = next.quality_rules.find((r) => r.rule_id === id);
if (!nr && pr.severity === 'block') {
if (!nr && pr.severity === "block") {
changes.push({
severity: 'breaking',
code: 'blocking_quality_rule_removed',
severity: "breaking",
code: "blocking_quality_rule_removed",
detail: `blocking quality rule "${id}" removed`,
});
} else if (nr && pr.severity === 'block' && nr.severity === 'warn') {
} else if (nr && pr.severity === "block" && nr.severity === "warn") {
changes.push({
severity: 'breaking',
code: 'blocking_quality_rule_downgraded',
severity: "breaking",
code: "blocking_quality_rule_downgraded",
detail: `quality rule "${id}" downgraded block -> warn`,
});
}
@@ -329,8 +296,8 @@ export function diffContracts(previous: DatasetContract, next: DatasetContract):
if (previous.tier !== next.tier) {
changes.push({
severity: 'breaking',
code: 'tier_changed',
severity: "breaking",
code: "tier_changed",
detail: `tier ${previous.tier} -> ${next.tier}`,
});
}
@@ -367,7 +334,7 @@ export function checkContractCompatibility(
const pv = parseSemVer(previous.version);
const nv = parseSemVer(next.version);
const changes = diffContracts(previous, next);
const breaking = changes.filter((c) => c.severity === 'breaking');
const breaking = changes.filter((c) => c.severity === "breaking");
const forward =
nv.major > pv.major ||
@@ -397,7 +364,7 @@ export function checkContractCompatibility(
reason:
`${breaking.length} breaking change(s) require a MAJOR version bump ` +
`(${previous.version} -> ${next.version} is not one): ` +
breaking.map((b) => b.code).join(', '),
breaking.map((b) => b.code).join(", "),
};
}
@@ -412,17 +379,10 @@ export function checkContractCompatibility(
}
/** Throwing form for call sites that treat incompatibility as a failure. */
export function assertContractCompatible(
previous: DatasetContract,
next: DatasetContract,
): CompatibilityVerdict {
export function assertContractCompatible(previous: DatasetContract, next: DatasetContract): CompatibilityVerdict {
const verdict = checkContractCompatibility(previous, next);
if (!verdict.compatible) {
throw new SiaxError(
'CONFLICT',
'0pt.analytics',
verdict.reason ?? 'dataset contract compatibility gate failed',
);
throw new SiaxError("CONFLICT", "0pt.analytics", verdict.reason ?? "dataset contract compatibility gate failed");
}
return verdict;
}
@@ -435,28 +395,24 @@ export function assertContractCompatible(
export function validateContract(contract: DatasetContract): void {
parseSemVer(contract.version);
if (!contract.dataset_id.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'contract needs a dataset_id');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "contract needs a dataset_id");
}
if (contract.fields.length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
`contract ${contract.dataset_id} declares no fields`,
);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `contract ${contract.dataset_id} declares no fields`);
}
const names = new Set<string>();
for (const f of contract.fields) {
if (names.has(f.name)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `duplicate field "${f.name}"`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `duplicate field "${f.name}"`);
}
names.add(f.name);
}
for (const label of ['primary_identifiers', 'dimensions', 'measures'] as const) {
for (const label of ["primary_identifiers", "dimensions", "measures"] as const) {
for (const n of contract[label]) {
if (!names.has(n)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`${label} references unknown field "${n}" in ${contract.dataset_id}`,
);
}
@@ -464,23 +420,23 @@ export function validateContract(contract: DatasetContract): void {
}
if (contract.primary_identifiers.length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`contract ${contract.dataset_id} declares no primary identifier`,
);
}
if (contract.allowed_purposes.length === 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`contract ${contract.dataset_id} declares no allowed_purposes (purpose limitation is not optional)`,
);
}
if (contract.freshness_slo_seconds <= 0) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'freshness_slo_seconds must be positive');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "freshness_slo_seconds must be positive");
}
if (contract.retention_days < 0) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'retention_days must not be negative');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "retention_days must not be negative");
}
}
@@ -501,7 +457,7 @@ export function contractHash(contract: DatasetContract): string {
nullable: f.nullable,
role: f.role,
primary_identifier: f.primary_identifier ?? false,
classification: f.classification ?? 'none',
classification: f.classification ?? "none",
}))
.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0)),
primary_identifiers: contract.primary_identifiers,
@@ -514,5 +470,5 @@ export function contractHash(contract: DatasetContract): string {
quality_rules: [...contract.quality_rules].sort((a, b) => (a.rule_id < b.rule_id ? -1 : 1)),
allowed_purposes: [...contract.allowed_purposes].sort(),
};
return crypto.createHash('sha256').update(JSON.stringify(canonical)).digest('hex');
return crypto.createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
}
+85 -85
View File
@@ -35,7 +35,7 @@
* defines its own copies of these tables and never receives engine
* credentials.
*/
import type { AnalyticsTier } from './tiers';
import type { AnalyticsTier } from "./tiers";
/** Every tenant-owned analytics record. */
export interface TenantScoped {
@@ -46,28 +46,28 @@ export interface TenantScoped {
// ── field/typing vocabulary ────────────────────────────────────────────────
export const ANALYTICS_FIELD_TYPES = [
'string',
'int64',
'float64',
'bool',
'date',
'timestamp',
'decimal',
'json',
"string",
"int64",
"float64",
"bool",
"date",
"timestamp",
"decimal",
"json",
] as const;
export type AnalyticsFieldType = (typeof ANALYTICS_FIELD_TYPES)[number];
/** A field's analytical role. Measures aggregate; dimensions group. */
export const FIELD_ROLES = ['identifier', 'dimension', 'measure', 'attribute'] as const;
export const FIELD_ROLES = ["identifier", "dimension", "measure", "attribute"] as const;
export type FieldRole = (typeof FIELD_ROLES)[number];
/** PII classification, aligned with the SIAX risk vocabulary (L0 = none). */
export const PII_CLASSIFICATIONS = ['none', 'pseudonymous', 'personal', 'sensitive'] as const;
export const PII_CLASSIFICATIONS = ["none", "pseudonymous", "personal", "sensitive"] as const;
export type PiiClassification = (typeof PII_CLASSIFICATIONS)[number];
// ── 1. AnalyticsDataset ────────────────────────────────────────────────────
export const DATASET_STATUSES = ['draft', 'active', 'deprecated', 'retired'] as const;
export const DATASET_STATUSES = ["draft", "active", "deprecated", "retired"] as const;
export type AnalyticsDatasetStatus = (typeof DATASET_STATUSES)[number];
/**
@@ -137,7 +137,7 @@ export interface DatasetField extends TenantScoped {
// ── 4. DatasetGrant ────────────────────────────────────────────────────────
export const GRANT_LEVELS = ['read', 'query', 'export', 'admin'] as const;
export const GRANT_LEVELS = ["read", "query", "export", "admin"] as const;
export type GrantLevel = (typeof GRANT_LEVELS)[number];
/**
@@ -150,7 +150,7 @@ export interface DatasetGrant extends TenantScoped {
dataset_id: string;
/** Subject: a CL0UD principal, role, or BI application binding id. */
subject_id: string;
subject_type: 'user' | 'role' | 'service' | 'bi_application';
subject_type: "user" | "role" | "service" | "bi_application";
level: GrantLevel;
allowed_purposes: string[];
expires_at?: string;
@@ -161,14 +161,14 @@ export interface DatasetGrant extends TenantScoped {
// ── 5. MetricDefinition ────────────────────────────────────────────────────
export const METRIC_AGGREGATIONS = [
'sum',
'avg',
'min',
'max',
'count',
'count_distinct',
'ratio',
'derived',
"sum",
"avg",
"min",
"max",
"count",
"count_distinct",
"ratio",
"derived",
] as const;
export type MetricAggregation = (typeof METRIC_AGGREGATIONS)[number];
@@ -244,7 +244,7 @@ export interface SemanticModel extends TenantScoped {
// ── 9. AnalyticsQuery ──────────────────────────────────────────────────────
export const QUERY_SURFACES = ['metric', 'sql', 'projection'] as const;
export const QUERY_SURFACES = ["metric", "sql", "projection"] as const;
export type QuerySurface = (typeof QUERY_SURFACES)[number];
/**
@@ -268,12 +268,12 @@ export interface AnalyticsQuery extends TenantScoped {
// ── 10. QueryExecution ─────────────────────────────────────────────────────
export const EXECUTION_STATES = [
'queued',
'running',
'succeeded',
'failed',
'cancelled',
'rejected',
"queued",
"running",
"succeeded",
"failed",
"cancelled",
"rejected",
] as const;
export type ExecutionState = (typeof EXECUTION_STATES)[number];
@@ -299,7 +299,7 @@ export interface QueryExecution extends TenantScoped {
// ── 11. QueryResult ────────────────────────────────────────────────────────
export const RESULT_ENCODINGS = ['json', 'arrow_ipc'] as const;
export const RESULT_ENCODINGS = ["json", "arrow_ipc"] as const;
export type ResultEncoding = (typeof RESULT_ENCODINGS)[number];
export interface QueryResult extends TenantScoped {
@@ -345,7 +345,7 @@ export interface DashboardVersion extends TenantScoped {
// ── 14. ReportDefinition ───────────────────────────────────────────────────
export const REPORT_FORMATS = ['pdf', 'csv', 'xlsx', 'parquet', 'arrow_ipc'] as const;
export const REPORT_FORMATS = ["pdf", "csv", "xlsx", "parquet", "arrow_ipc"] as const;
export type ReportFormat = (typeof REPORT_FORMATS)[number];
export interface ReportDefinition extends TenantScoped {
@@ -378,18 +378,18 @@ export interface ReportRun extends TenantScoped {
// ── 16. DataQualityRule ────────────────────────────────────────────────────
export const QUALITY_RULE_KINDS = [
'not_null',
'unique',
'accepted_values',
'range',
'freshness',
'row_count',
'referential',
'custom',
"not_null",
"unique",
"accepted_values",
"range",
"freshness",
"row_count",
"referential",
"custom",
] as const;
export type QualityRuleKind = (typeof QUALITY_RULE_KINDS)[number];
export const QUALITY_SEVERITIES = ['warn', 'block'] as const;
export const QUALITY_SEVERITIES = ["warn", "block"] as const;
export type QualitySeverity = (typeof QUALITY_SEVERITIES)[number];
export interface DataQualityRule extends TenantScoped {
@@ -423,7 +423,7 @@ export interface DataQualityRun extends TenantScoped {
// ── 18. LineageReference ───────────────────────────────────────────────────
export const LINEAGE_DIRECTIONS = ['input', 'output'] as const;
export const LINEAGE_DIRECTIONS = ["input", "output"] as const;
export type LineageDirection = (typeof LINEAGE_DIRECTIONS)[number];
/**
@@ -438,7 +438,7 @@ export interface LineageReference extends TenantScoped {
job_name: string;
direction: LineageDirection;
/** The analytics entity this edge touches. */
entity_type: 'dataset' | 'metric' | 'dashboard' | 'report';
entity_type: "dataset" | "metric" | "dashboard" | "report";
entity_id: string;
/** Namespaced dataset name on the other side of the edge. */
counterpart_ref: string;
@@ -447,7 +447,7 @@ export interface LineageReference extends TenantScoped {
// ── 19. BIApplicationBinding ───────────────────────────────────────────────
export const BI_APPLICATION_KINDS = ['superset', 'evidence', 'des1gn', 'notebook', 'custom'] as const;
export const BI_APPLICATION_KINDS = ["superset", "evidence", "des1gn", "notebook", "custom"] as const;
export type BiApplicationKind = (typeof BI_APPLICATION_KINDS)[number];
/**
@@ -482,55 +482,55 @@ export interface BIApplicationBinding extends TenantScoped {
* or omission is a test failure rather than drift.
*/
export const ANALYTICS_DOMAIN_ENTITIES = [
'AnalyticsDataset',
'DatasetVersion',
'DatasetField',
'DatasetGrant',
'MetricDefinition',
'MetricVersion',
'DimensionDefinition',
'SemanticModel',
'AnalyticsQuery',
'QueryExecution',
'QueryResult',
'DashboardDefinition',
'DashboardVersion',
'ReportDefinition',
'ReportRun',
'DataQualityRule',
'DataQualityRun',
'LineageReference',
'BIApplicationBinding',
"AnalyticsDataset",
"DatasetVersion",
"DatasetField",
"DatasetGrant",
"MetricDefinition",
"MetricVersion",
"DimensionDefinition",
"SemanticModel",
"AnalyticsQuery",
"QueryExecution",
"QueryResult",
"DashboardDefinition",
"DashboardVersion",
"ReportDefinition",
"ReportRun",
"DataQualityRule",
"DataQualityRun",
"LineageReference",
"BIApplicationBinding",
] as const;
export type AnalyticsDomainEntity = (typeof ANALYTICS_DOMAIN_ENTITIES)[number];
/** Entity -> physical table in sql/008_analytics_domain_model.sql. */
export const ANALYTICS_ENTITY_TABLES: Record<AnalyticsDomainEntity, string> = {
AnalyticsDataset: 'opt_analytics_dataset',
DatasetVersion: 'opt_analytics_dataset_version',
DatasetField: 'opt_analytics_dataset_field',
DatasetGrant: 'opt_analytics_dataset_grant',
MetricDefinition: 'opt_analytics_metric_definition',
MetricVersion: 'opt_analytics_metric_version',
DimensionDefinition: 'opt_analytics_dimension_definition',
SemanticModel: 'opt_analytics_semantic_model',
AnalyticsQuery: 'opt_analytics_query',
QueryExecution: 'opt_analytics_query_execution',
QueryResult: 'opt_analytics_query_result',
DashboardDefinition: 'opt_analytics_dashboard_definition',
DashboardVersion: 'opt_analytics_dashboard_version',
ReportDefinition: 'opt_analytics_report_definition',
ReportRun: 'opt_analytics_report_run',
DataQualityRule: 'opt_analytics_data_quality_rule',
DataQualityRun: 'opt_analytics_data_quality_run',
LineageReference: 'opt_analytics_lineage_reference',
BIApplicationBinding: 'opt_analytics_bi_application_binding',
AnalyticsDataset: "opt_analytics_dataset",
DatasetVersion: "opt_analytics_dataset_version",
DatasetField: "opt_analytics_dataset_field",
DatasetGrant: "opt_analytics_dataset_grant",
MetricDefinition: "opt_analytics_metric_definition",
MetricVersion: "opt_analytics_metric_version",
DimensionDefinition: "opt_analytics_dimension_definition",
SemanticModel: "opt_analytics_semantic_model",
AnalyticsQuery: "opt_analytics_query",
QueryExecution: "opt_analytics_query_execution",
QueryResult: "opt_analytics_query_result",
DashboardDefinition: "opt_analytics_dashboard_definition",
DashboardVersion: "opt_analytics_dashboard_version",
ReportDefinition: "opt_analytics_report_definition",
ReportRun: "opt_analytics_report_run",
DataQualityRule: "opt_analytics_data_quality_rule",
DataQualityRun: "opt_analytics_data_quality_run",
LineageReference: "opt_analytics_lineage_reference",
BIApplicationBinding: "opt_analytics_bi_application_binding",
};
/** The four entities BI-P002 requires to be immutable once written. */
export const IMMUTABLE_VERSION_ENTITIES: readonly AnalyticsDomainEntity[] = [
'DatasetVersion',
'MetricVersion',
'SemanticModel',
'DashboardVersion',
"DatasetVersion",
"MetricVersion",
"SemanticModel",
"DashboardVersion",
];
+12 -12
View File
@@ -21,24 +21,24 @@
* ARCHITECTURE BOUNDARY (BI program §0): 0pt owns tier semantics. Product
* repos (VehIQ et al) own what the rows MEAN; they never re-declare tiers.
*/
import { SiaxError } from '../../platform/errors';
import { SiaxError } from "../../platform/errors";
export const ANALYTICS_TIERS = ['RAW', 'NORMALIZED', 'CURATED', 'SERVING'] as const;
export const ANALYTICS_TIERS = ["RAW", "NORMALIZED", "CURATED", "SERVING"] as const;
export type AnalyticsTier = (typeof ANALYTICS_TIERS)[number];
/** Physical home a tier is expected to live in (BI-P003 "suggested" mapping). */
export const TIER_PHYSICAL_HOME: Record<AnalyticsTier, string> = {
RAW: 'st0re:iceberg|parquet',
NORMALIZED: 'st0re:iceberg',
CURATED: 'st0re:iceberg',
SERVING: 'clickhouse:projection',
RAW: "st0re:iceberg|parquet",
NORMALIZED: "st0re:iceberg",
CURATED: "st0re:iceberg",
SERVING: "clickhouse:projection",
};
/** Tiers a BI presentation surface may read without an explicit exception. */
export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ['SERVING', 'CURATED'];
export const BI_READABLE_TIERS: readonly AnalyticsTier[] = ["SERVING", "CURATED"];
export function isAnalyticsTier(v: unknown): v is AnalyticsTier {
return typeof v === 'string' && (ANALYTICS_TIERS as readonly string[]).includes(v);
return typeof v === "string" && (ANALYTICS_TIERS as readonly string[]).includes(v);
}
/**
@@ -64,8 +64,8 @@ export function assertReadableByBiSurface(
if (BI_READABLE_TIERS.includes(dataset.tier)) return;
if (dataset.raw_access_exception && dataset.raw_access_exception.trim().length > 0) return;
throw new SiaxError(
'AUTHORIZATION_DENIED',
'0pt.analytics',
"AUTHORIZATION_DENIED",
"0pt.analytics",
`BI surface "${surface}" may not read ${dataset.tier} dataset ${dataset.dataset_id} ` +
`without a recorded raw_access_exception (BI-P003)`,
);
@@ -75,8 +75,8 @@ export function assertReadableByBiSurface(
export function assertTierLineageSane(datasetTier: AnalyticsTier, sourceTier: AnalyticsTier): void {
if (TIER_RANK[sourceTier] > TIER_RANK[datasetTier]) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`a ${datasetTier} dataset cannot be built from a ${sourceTier} dataset (BI-P003 tier order)`,
);
}
+55 -62
View File
@@ -26,17 +26,17 @@
* LEGAL BOUNDARY (inherited): generic 0pt analytics only. Never the b0ts
* ClickHouse pipelines, never quarantined market data.
*/
import crypto from 'node:crypto';
import { SiaxError, siaxErrorCodeForUpstreamStatus } from '../../platform/errors';
import { ClickHouseAnalytics } from '../clickhouse-analytics';
import crypto from "node:crypto";
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../../platform/errors";
import { ClickHouseAnalytics } from "../clickhouse-analytics";
import {
compileMetricQuery,
compileProjectionQuery,
dimensionCatalogFrom,
FORCED_TENANT_PREDICATE,
type DimensionCatalog,
} from './query-compiler';
import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from './schema';
} from "./query-compiler";
import { analyticsSchemaDdl, type AnalyticsSchemaOptions } from "./schema";
import {
applyQuota,
clickhouseSettings,
@@ -44,7 +44,7 @@ import {
DEFAULT_BUDGETS,
planningSettings,
type ResourceBudget,
} from './resource-governance';
} from "./resource-governance";
import type {
AnalystQueryRequest,
AnalyticsIdentity,
@@ -56,7 +56,7 @@ import type {
QueryEstimate,
QueryExplanation,
SiaxAnalyticsEngine,
} from './siax-analytics-engine';
} from "./siax-analytics-engine";
export interface ClickHouseEngineOptions {
baseUrl: string;
@@ -66,13 +66,13 @@ export interface ClickHouseEngineOptions {
timeoutMs?: number;
/** Dimension allow-list per tenant. Absent tenant = no dimensions allowed. */
dimensionCatalogs?: Record<string, Record<string, string>>;
schema?: Omit<AnalyticsSchemaOptions, 'database'>;
schema?: Omit<AnalyticsSchemaOptions, "database">;
}
const EMPTY_CATALOG: DimensionCatalog = { resolve: () => undefined };
export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
readonly provider = 'clickhouse';
readonly provider = "clickhouse";
/** PRESERVED pre-existing implementation, composed not replaced. */
private readonly legacy: ClickHouseAnalytics;
@@ -81,15 +81,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
private readonly database: string;
private readonly timeoutMs: number;
private readonly dimensionCatalogs: Record<string, Record<string, string>>;
private readonly schemaOpts: Omit<AnalyticsSchemaOptions, 'database'>;
private readonly schemaOpts: Omit<AnalyticsSchemaOptions, "database">;
private readonly gate: ConcurrencyGate;
/** query_id -> AbortController, so cancelQuery can stop the client side too. */
private readonly inFlight = new Map<string, AbortController>();
constructor(opts: ClickHouseEngineOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, '');
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString('base64')}` : undefined;
this.database = opts.database ?? 'opt_analytics';
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
this.database = opts.database ?? "opt_analytics";
this.timeoutMs = opts.timeoutMs ?? 30_000;
this.dimensionCatalogs = opts.dimensionCatalogs ?? {};
this.schemaOpts = opts.schema ?? {};
@@ -112,7 +112,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
}
private budgetFor(identity: AnalyticsIdentity): ResourceBudget {
const tier = identity.budget_tier ?? 'INTERACTIVE';
const tier = identity.budget_tier ?? "INTERACTIVE";
return applyQuota(DEFAULT_BUDGETS[tier], identity.quota);
}
@@ -125,13 +125,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
private assertIdentity(identity: AnalyticsIdentity): void {
if (!identity?.tenant_id || !identity.tenant_id.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'analytics requires a verified CL0UD identity carrying tenant_id',
"UNAUTHENTICATED",
"0pt.analytics",
"analytics requires a verified CL0UD identity carrying tenant_id",
);
}
if (!identity.principal_id || !identity.principal_id.trim()) {
throw new SiaxError('UNAUTHENTICATED', '0pt.analytics', 'analytics requires a verified principal_id');
throw new SiaxError("UNAUTHENTICATED", "0pt.analytics", "analytics requires a verified principal_id");
}
}
@@ -150,16 +150,15 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
async health(): Promise<HealthReport> {
try {
const legacyOk = await this.legacy.health();
if (!legacyOk)
return { healthy: false, provider: this.provider, detail: 'legacy health probe returned false' };
const v = await this.raw('SELECT version() AS v FORMAT JSON', {}, {});
if (!legacyOk) return { healthy: false, provider: this.provider, detail: "legacy health probe returned false" };
const v = await this.raw("SELECT version() AS v FORMAT JSON", {}, {});
const version = (v as { data?: { v?: string }[] })?.data?.[0]?.v;
return { healthy: true, provider: this.provider, version };
} catch (err) {
return {
healthy: false,
provider: this.provider,
detail: err instanceof SiaxError ? err.code : 'unreachable',
detail: err instanceof SiaxError ? err.code : "unreachable",
};
}
}
@@ -192,13 +191,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
budget.maxResultRows,
);
const json = (await this.raw(compiled.sql, compiled.params, clickhouseSettings(budget), queryId)) as
ClickHouseJsonResponse | undefined;
| ClickHouseJsonResponse
| undefined;
const groupNames = request.group_by ?? [];
const rows = (json?.data ?? []).map((r) => {
const groups: Record<string, string> = {};
groupNames.forEach((name, i) => {
const v = (r as Record<string, unknown>)[`grp${i}`];
groups[name] = v === undefined || v === null ? '' : String(v);
groups[name] = v === undefined || v === null ? "" : String(v);
});
return {
...(groupNames.length ? { groups } : {}),
@@ -230,15 +230,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
*/
async executeLegacyMetric(
identity: AnalyticsIdentity,
request: {
metric: string;
aggregate: 'sum' | 'avg' | 'min' | 'max' | 'count';
dataset_id?: string;
group_by?: string;
from_time?: string;
to_time?: string;
limit?: number;
},
request: { metric: string; aggregate: "sum" | "avg" | "min" | "max" | "count"; dataset_id?: string; group_by?: string; from_time?: string; to_time?: string; limit?: number },
): Promise<EngineResult> {
this.assertIdentity(identity);
const started = Date.now();
@@ -272,9 +264,9 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
// BI-P033 (analytics authorization) exist would create precisely the
// "raw SQL attempt" hole BI-P011's adversarial list enumerates.
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics',
'analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land',
"CAPABILITY_UNAVAILABLE",
"0pt.analytics",
"analyst SQL is not enabled on the ClickHouse provider until BI-P017/BI-P033 land",
);
}
@@ -304,7 +296,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
// `EXPLAIN ESTIMATE` reports the parts/rows/marks the planner will read
// AFTER primary-key pruning — which is why tenant_id leads every ORDER BY
// in schema.ts: the estimate for one tenant does not include everyone.
const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, '');
const estimateSql = compiled.sql.replace(/ FORMAT JSON$/, "");
let rowsToRead = 0;
let partsRead = 0;
try {
@@ -321,7 +313,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
// An estimate that cannot be produced must not read as "cheap".
throw err instanceof SiaxError
? err
: new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'could not estimate query');
: new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "could not estimate query");
}
// Rough byte estimate: ClickHouse's ESTIMATE reports rows, not bytes.
// 128 B/row is a deliberately PESSIMISTIC constant for these tables
@@ -355,7 +347,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
async cancelQuery(identity: AnalyticsIdentity, queryId: string): Promise<boolean> {
this.assertIdentity(identity);
if (!queryId.startsWith(this.queryIdPrefix(identity))) {
throw new SiaxError('AUTHORIZATION_DENIED', '0pt.analytics', 'query does not belong to this tenant');
throw new SiaxError("AUTHORIZATION_DENIED", "0pt.analytics", "query does not belong to this tenant");
}
this.inFlight.get(queryId)?.abort();
this.inFlight.delete(queryId);
@@ -365,7 +357,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
await this.raw(
`KILL QUERY WHERE query_id = {killId:String} SYNC FORMAT JSON`,
{ param_killId: queryId },
{ max_execution_time: '10' },
{ max_execution_time: "10" },
);
return true;
} catch {
@@ -396,14 +388,14 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
this.catalogFor(identity),
budget.maxResultRows,
);
const sql = compiled.sql.replace(/ FORMAT JSON$/, '');
const sql = compiled.sql.replace(/ FORMAT JSON$/, "");
const res = (await this.raw(
`EXPLAIN PLAN indexes = 1 ${sql} FORMAT TSVRaw`,
compiled.params,
planningSettings(budget),
)) as string | undefined;
return {
plan: typeof res === 'string' ? res : JSON.stringify(res ?? ''),
plan: typeof res === "string" ? res : JSON.stringify(res ?? ""),
forced_tenant_predicate: FORCED_TENANT_PREDICATE,
table: compiled.table,
};
@@ -422,22 +414,22 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
const body = request.rows
.map((r) => {
if (!r.entity_key || !r.entity_key.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'every projection row needs an entity_key');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every projection row needs an entity_key");
}
return JSON.stringify({
// tenant/project come from the VERIFIED identity, never the row —
// a row-supplied tenant_id is the INF0 Wave-2 bug class.
tenant_id: identity.tenant_id,
project_id: identity.project_id ?? '',
project_id: identity.project_id ?? "",
dataset_id: request.dataset_id,
entity_key: r.entity_key,
as_of: (r.as_of ?? now.toISOString()).replace('T', ' ').replace(/Z$/, ''),
as_of: (r.as_of ?? now.toISOString()).replace("T", " ").replace(/Z$/, ""),
version: r.version ?? version,
attributes: r.attributes ?? {},
measures: r.measures ?? {},
});
})
.join('\n');
.join("\n");
await this.raw(
`INSERT INTO ${this.database}.opt_dataset_projection FORMAT JSONEachRow`,
{},
@@ -463,7 +455,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
await this.raw(
`OPTIMIZE TABLE ${this.database}.opt_dataset_projection PARTITION {tenant:String} FINAL`,
{ param_tenant: identity.tenant_id },
{ max_execution_time: '120' },
{ max_execution_time: "120" },
);
const compiled = compileProjectionQuery(
{ tenant_id: identity.tenant_id, project_id: identity.project_id, dataset_id, limit: 1 },
@@ -474,7 +466,8 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
`SELECT count() AS value FROM ${this.database}.opt_dataset_projection FINAL ` +
`WHERE ${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String} FORMAT JSON`;
const res = (await this.raw(countSql, compiled.params, clickhouseSettings(this.budgetFor(identity)))) as
{ data?: { value: number | string }[] } | undefined;
| { data?: { value: number | string }[] }
| undefined;
return {
dataset_id,
row_count: Number(res?.data?.[0]?.value ?? 0),
@@ -485,16 +478,16 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
// ── transport ────────────────────────────────────────────────────────────
private queryIdPrefix(identity: AnalyticsIdentity): string {
return `opt_${crypto.createHash('sha256').update(identity.tenant_id).digest('hex').slice(0, 12)}_`;
return `opt_${crypto.createHash("sha256").update(identity.tenant_id).digest("hex").slice(0, 12)}_`;
}
private newQueryId(identity: AnalyticsIdentity): string {
return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, '')}`;
return `${this.queryIdPrefix(identity)}${crypto.randomUUID().replace(/-/g, "")}`;
}
/** DDL path: no readonly, no result parsing beyond errors. */
private async ddl(sql: string): Promise<void> {
await this.raw(sql, {}, { max_execution_time: '60' });
await this.raw(sql, {}, { max_execution_time: "60" });
}
private async raw(
@@ -505,13 +498,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
body?: string,
): Promise<unknown> {
const url = new URL(this.baseUrl);
url.searchParams.set('query', sql);
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings)) url.searchParams.set(k, v);
for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v);
if (queryId) url.searchParams.set('query_id', queryId);
if (queryId) url.searchParams.set("query_id", queryId);
const headers: Record<string, string> = {};
if (this.authHeader) headers['Authorization'] = this.authHeader;
if (this.authHeader) headers["Authorization"] = this.authHeader;
const controller = new AbortController();
if (queryId) this.inFlight.set(queryId, controller);
@@ -520,13 +513,13 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
let res: Response;
try {
res = await fetch(url.toString(), {
method: 'POST',
method: "POST",
headers,
body: body ?? '',
body: body ?? "",
signal: controller.signal,
});
} catch (cause) {
throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause });
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
} finally {
clearTimeout(timeout);
}
@@ -537,7 +530,7 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
'clickhouse',
"clickhouse",
`query failed (CH code ${code})`,
);
}
@@ -555,10 +548,10 @@ export class ClickHouseAnalyticsEngine implements SiaxAnalyticsEngine {
// have already been flushed, the JSON body can carry an `exception` key
// under a 200 — a truncated answer indistinguishable from a whole one.
// Treat any body carrying an exception as a failure, whatever the status.
if (parsed && typeof parsed === 'object' && 'exception' in parsed) {
const ex = String((parsed as { exception?: unknown }).exception ?? '');
const code = /Code: (\d+)/.exec(ex)?.[1] ?? 'unknown';
throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `query failed mid-stream (CH code ${code})`);
if (parsed && typeof parsed === "object" && "exception" in parsed) {
const ex = String((parsed as { exception?: unknown }).exception ?? "");
const code = /Code: (\d+)/.exec(ex)?.[1] ?? "unknown";
throw new SiaxError("PERMANENT_FAILURE", "clickhouse", `query failed mid-stream (CH code ${code})`);
}
return parsed;
}
+47 -49
View File
@@ -27,20 +27,20 @@
* the guarantee rather than replacing it, and
* tests/analytics/tenant-predicate-regression.test.ts asserts BOTH.
*/
import { SiaxError } from '../../platform/errors';
import type { AnalyticsTable } from './schema';
import { SiaxError } from "../../platform/errors";
import type { AnalyticsTable } from "./schema";
/** A physical identifier that has been VALIDATED, not merely received. */
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
export function assertSafeIdentifier(value: string, what: string): string {
if (!IDENT_RE.test(value)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `invalid ${what} identifier`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid ${what} identifier`);
}
return value;
}
export const METRIC_AGGREGATES = ['sum', 'avg', 'min', 'max', 'count', 'count_distinct'] as const;
export const METRIC_AGGREGATES = ["sum", "avg", "min", "max", "count", "count_distinct"] as const;
export type MetricAggregate = (typeof METRIC_AGGREGATES)[number];
/**
@@ -97,15 +97,15 @@ export interface CompiledQuery {
* forcing, the test fails on a missing substring rather than on a subtle
* behavioural difference that a mocked engine would happily hide.
*/
export const FORCED_TENANT_PREDICATE = 'tenant_id = {tenant:String}';
export const FORCED_TENANT_PREDICATE = "tenant_id = {tenant:String}";
const AGG_SQL: Record<MetricAggregate, string> = {
sum: 'sum(value)',
avg: 'avg(value)',
min: 'min(value)',
max: 'max(value)',
count: 'count()',
count_distinct: 'uniqExact(event_id)',
sum: "sum(value)",
avg: "avg(value)",
min: "min(value)",
max: "max(value)",
count: "count()",
count_distinct: "uniqExact(event_id)",
};
const MAX_GROUP_BY = 8;
@@ -126,13 +126,13 @@ export function compileMetricQuery(
): CompiledQuery {
if (!req.tenant_id || !req.tenant_id.trim()) {
// Fail closed. A blank tenant must never compile to "no predicate".
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
}
assertSafeIdentifier(db, 'database');
const table: AnalyticsTable = req.table ?? 'opt_analytics_event';
assertSafeIdentifier(table, 'table');
assertSafeIdentifier(db, "database");
const table: AnalyticsTable = req.table ?? "opt_analytics_event";
assertSafeIdentifier(table, "table");
if (!METRIC_AGGREGATES.includes(req.aggregate)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'unsupported aggregate');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "unsupported aggregate");
}
const params: Record<string, string> = {
@@ -141,22 +141,22 @@ export function compileMetricQuery(
};
// ── the forced predicate. First condition, always, no branch. ─────────
const conditions: string[] = [FORCED_TENANT_PREDICATE, 'metric = {metric:String}'];
const conditions: string[] = [FORCED_TENANT_PREDICATE, "metric = {metric:String}"];
if (req.project_id) {
conditions.push('project_id = {project:String}');
conditions.push("project_id = {project:String}");
params.param_project = req.project_id;
}
if (req.dataset_id) {
conditions.push('dataset_id = {dataset:String}');
conditions.push("dataset_id = {dataset:String}");
params.param_dataset = req.dataset_id;
}
if (req.from_time) {
conditions.push('event_time >= parseDateTime64BestEffort({fromTime:String})');
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
params.param_fromTime = req.from_time;
}
if (req.to_time) {
conditions.push('event_time < parseDateTime64BestEffort({toTime:String})');
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
params.param_toTime = req.to_time;
}
@@ -165,14 +165,16 @@ export function compileMetricQuery(
for (const [name, value] of Object.entries(req.filters ?? {})) {
const physical = catalog.resolve(name);
if (physical === undefined) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown filter dimension "${name}"`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown filter dimension "${name}"`);
}
// `physical` came from the catalog, but validate anyway: a corrupted or
// hostile catalog row must not become an injection vector either.
assertSafeIdentifier(physical, 'dimension');
assertSafeIdentifier(physical, "dimension");
const nameKey = `f${filterIdx}n`;
const valKey = `f${filterIdx}v`;
conditions.push(`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`);
conditions.push(
`dim_values[indexOf(dim_names, {${nameKey}:String})] = {${valKey}:String}`,
);
params[`param_${nameKey}`] = physical;
params[`param_${valKey}`] = value;
filterIdx += 1;
@@ -182,8 +184,8 @@ export function compileMetricQuery(
const groupBy = req.group_by ?? [];
if (groupBy.length > MAX_GROUP_BY) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
"INVALID_REQUEST",
"0pt.analytics",
`at most ${MAX_GROUP_BY} group-by dimensions (got ${groupBy.length})`,
);
}
@@ -192,9 +194,9 @@ export function compileMetricQuery(
groupBy.forEach((name, i) => {
const physical = catalog.resolve(name);
if (physical === undefined) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', `unknown group_by dimension "${name}"`);
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `unknown group_by dimension "${name}"`);
}
assertSafeIdentifier(physical, 'dimension');
assertSafeIdentifier(physical, "dimension");
const key = `g${i}`;
// The ALIAS is compiler-generated (`grp0`, `grp1`), never caller text.
selectParts.push(`dim_values[indexOf(dim_names, {${key}:String})] AS grp${i}`);
@@ -205,10 +207,10 @@ export function compileMetricQuery(
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
const sql =
`SELECT ${selectParts.length ? selectParts.join(', ') + ', ' : ''}${AGG_SQL[req.aggregate]} AS value ` +
`SELECT ${selectParts.length ? selectParts.join(", ") + ", " : ""}${AGG_SQL[req.aggregate]} AS value ` +
`FROM ${db}.${table} ` +
`WHERE ${conditions.join(' AND ')} ` +
(groupParts.length ? `GROUP BY ${groupParts.join(', ')} ORDER BY value DESC ` : '') +
`WHERE ${conditions.join(" AND ")} ` +
(groupParts.length ? `GROUP BY ${groupParts.join(", ")} ORDER BY value DESC ` : "") +
`LIMIT ${limit} FORMAT JSON`;
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table };
@@ -224,29 +226,29 @@ export function compileProjectionQuery(
maxResultRows: number,
): CompiledQuery {
if (!req.tenant_id || !req.tenant_id.trim()) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant_id is required on every analytics query');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every analytics query");
}
assertSafeIdentifier(db, 'database');
assertSafeIdentifier(db, "database");
const params: Record<string, string> = {
param_tenant: req.tenant_id,
param_dataset: req.dataset_id,
};
const conditions = [FORCED_TENANT_PREDICATE, 'dataset_id = {dataset:String}'];
const conditions = [FORCED_TENANT_PREDICATE, "dataset_id = {dataset:String}"];
if (req.project_id) {
conditions.push('project_id = {project:String}');
conditions.push("project_id = {project:String}");
params.param_project = req.project_id;
}
if (req.entity_keys && req.entity_keys.length > 0) {
conditions.push('has({entityKeys:Array(String)}, entity_key)');
conditions.push("has({entityKeys:Array(String)}, entity_key)");
params.param_entityKeys = JSON.stringify(req.entity_keys);
}
const limit = Math.max(1, Math.min(Math.floor(req.limit ?? 1000), maxResultRows));
const sql =
`SELECT entity_key, as_of, version, attributes, measures ` +
`FROM ${db}.opt_dataset_projection ` +
`WHERE ${conditions.join(' AND ')} ` +
`WHERE ${conditions.join(" AND ")} ` +
`ORDER BY entity_key LIMIT ${limit} FORMAT JSON`;
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: 'opt_dataset_projection' };
return { sql, params, forcedTenantPredicate: FORCED_TENANT_PREDICATE, table: "opt_dataset_projection" };
}
/**
@@ -273,29 +275,25 @@ export function restrictedTenantUserDdl(args: {
tables: readonly string[];
settingsProfile?: string;
}): string[] {
assertSafeIdentifier(args.db, 'database');
assertSafeIdentifier(args.userName, 'user');
assertSafeIdentifier(args.db, "database");
assertSafeIdentifier(args.userName, "user");
// A tenant id is `t_<slug>` in this estate; enforce that shape strictly
// rather than trusting the caller.
if (!/^[A-Za-z0-9_-]{1,64}$/.test(args.tenantId)) {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics', 'tenant id is not policy-safe');
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant id is not policy-safe");
}
if (args.password.includes("'") || args.password.includes('\\')) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics',
'password contains characters that cannot be embedded',
);
if (args.password.includes("'") || args.password.includes("\\")) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "password contains characters that cannot be embedded");
}
const profile = args.settingsProfile;
if (profile) assertSafeIdentifier(profile, 'settings profile');
if (profile) assertSafeIdentifier(profile, "settings profile");
const stmts: string[] = [
`CREATE USER IF NOT EXISTS ${args.userName} IDENTIFIED WITH sha256_password BY '${args.password}'` +
(profile ? ` SETTINGS PROFILE ${profile}` : ` SETTINGS readonly = 1`),
];
for (const table of args.tables) {
assertSafeIdentifier(table, 'table');
assertSafeIdentifier(table, "table");
stmts.push(
`CREATE ROW POLICY IF NOT EXISTS ${args.userName}_${table}_rp ON ${args.db}.${table} ` +
`FOR SELECT USING tenant_id = '${args.tenantId}' TO ${args.userName}`,
+17 -17
View File
@@ -30,7 +30,7 @@
* subsidises a query bomb. `INTERACTIVE` is the dashboard default and is
* deliberately mean.
*/
import { SiaxError } from '../../platform/errors';
import { SiaxError } from "../../platform/errors";
export interface ResourceBudget {
/** Seconds. ClickHouse `max_execution_time`. */
@@ -47,7 +47,7 @@ export interface ResourceBudget {
maxConcurrentQueries: number;
}
export const BUDGET_TIERS = ['INTERACTIVE', 'ANALYTICAL', 'BATCH'] as const;
export const BUDGET_TIERS = ["INTERACTIVE", "ANALYTICAL", "BATCH"] as const;
export type BudgetTier = (typeof BUDGET_TIERS)[number];
/**
@@ -91,7 +91,7 @@ export const DEFAULT_BUDGETS: Record<BudgetTier, ResourceBudget> = {
export function applyQuota(base: ResourceBudget, quota: Partial<ResourceBudget> | undefined): ResourceBudget {
if (!quota) return { ...base };
const pick = (b: number, q: number | undefined): number =>
typeof q === 'number' && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b;
typeof q === "number" && Number.isFinite(q) && q > 0 ? Math.min(b, q) : b;
return {
maxExecutionTimeSec: pick(base.maxExecutionTimeSec, quota.maxExecutionTimeSec),
maxResultRows: pick(base.maxResultRows, quota.maxResultRows),
@@ -114,14 +114,14 @@ export function applyQuota(base: ResourceBudget, quota: Partial<ResourceBudget>
*/
export function clickhouseSettings(budget: ResourceBudget): Record<string, string> {
return {
readonly: '1',
readonly: "1",
max_execution_time: String(budget.maxExecutionTimeSec),
timeout_overflow_mode: 'throw',
timeout_overflow_mode: "throw",
max_result_rows: String(budget.maxResultRows),
result_overflow_mode: 'throw',
result_overflow_mode: "throw",
max_rows_to_read: String(budget.maxRowsToRead),
max_bytes_to_read: String(budget.maxBytesToRead),
read_overflow_mode: 'throw',
read_overflow_mode: "throw",
max_memory_usage: String(budget.maxMemoryUsageBytes),
};
}
@@ -139,21 +139,21 @@ export function clickhouseSettings(budget: ResourceBudget): Record<string, strin
*/
export function planningSettings(budget: ResourceBudget): Record<string, string> {
return {
readonly: '1',
readonly: "1",
max_execution_time: String(Math.min(budget.maxExecutionTimeSec, 15)),
timeout_overflow_mode: 'throw',
timeout_overflow_mode: "throw",
max_memory_usage: String(budget.maxMemoryUsageBytes),
};
}
/** Every setting name BI-P012 requires, for the DoD assertion. */
export const REQUIRED_GOVERNANCE_SETTINGS = [
'max_execution_time',
'max_result_rows',
'max_rows_to_read',
'max_bytes_to_read',
'max_memory_usage',
'max_concurrent_queries',
"max_execution_time",
"max_result_rows",
"max_rows_to_read",
"max_bytes_to_read",
"max_memory_usage",
"max_concurrent_queries",
] as const;
/**
@@ -192,8 +192,8 @@ export class ConcurrencyGate {
const current = this.inFlight.get(tenantId) ?? 0;
if (current >= limit) {
throw new SiaxError(
'RATE_LIMITED',
'0pt.analytics',
"RATE_LIMITED",
"0pt.analytics",
`tenant already has ${current} analytics queries in flight (limit ${limit})`,
);
}
+6 -5
View File
@@ -55,14 +55,14 @@ export interface AnalyticsSchemaOptions {
/** Physical tables this module owns (opt_metric_events is NOT one of them). */
export const ANALYTICS_TABLES = [
'opt_analytics_event',
'opt_metric_daily',
'opt_dataset_projection',
"opt_analytics_event",
"opt_metric_daily",
"opt_dataset_projection",
] as const;
export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number];
/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */
export const ANALYTICS_MATERIALIZED_VIEWS = ['opt_metric_daily_mv'] as const;
export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const;
/**
* DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so
@@ -163,6 +163,7 @@ export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] {
*/
export function sortingKeySql(db: string, table: string): string {
return (
`SELECT sorting_key FROM system.tables ` + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
`SELECT sorting_key FROM system.tables ` +
`WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
);
}
+12 -15
View File
@@ -19,9 +19,9 @@
* (VehIQ) call an API in front of this interface; they never hold a provider
* credential and never construct an engine.
*/
import type { CompiledQuery, MetricAggregate } from './query-compiler';
import type { AnalyticsTable } from './schema';
import type { BudgetTier, ResourceBudget } from './resource-governance';
import type { CompiledQuery, MetricAggregate } from "./query-compiler";
import type { AnalyticsTable } from "./schema";
import type { BudgetTier, ResourceBudget } from "./resource-governance";
/**
* The VERIFIED caller. `tenant_id` here is the one the engine forces into
@@ -158,24 +158,21 @@ export interface SiaxAnalyticsEngine {
explain(identity: AnalyticsIdentity, request: MetricRequest): Promise<QueryExplanation>;
ingestProjection(
identity: AnalyticsIdentity,
request: ProjectionIngestRequest,
): Promise<{ ingested: number }>;
ingestProjection(identity: AnalyticsIdentity, request: ProjectionIngestRequest): Promise<{ ingested: number }>;
refreshProjection(identity: AnalyticsIdentity, dataset_id: string): Promise<ProjectionRefreshResult>;
}
/** The eight capability names, for the provider contract test. */
export const SIAX_ANALYTICS_ENGINE_CAPABILITIES = [
'health',
'executeMetric',
'executeQuery',
'estimateQuery',
'cancelQuery',
'explain',
'ingestProjection',
'refreshProjection',
"health",
"executeMetric",
"executeQuery",
"estimateQuery",
"cancelQuery",
"explain",
"ingestProjection",
"refreshProjection",
] as const;
export type SiaxAnalyticsEngineCapability = (typeof SIAX_ANALYTICS_ENGINE_CAPABILITIES)[number];
@@ -19,9 +19,9 @@
* Uses postgres.js through `RepositorySql`, the same connection type the other
* P0 repositories take (src/repositories/db.ts).
*/
import crypto from 'node:crypto';
import { SiaxError } from '../../platform/errors';
import type { RepositorySql } from '../../repositories/db';
import crypto from "node:crypto";
import { SiaxError } from "../../platform/errors";
import type { RepositorySql } from "../../repositories/db";
import type {
AnalyticsDataset,
DatasetVersion,
@@ -29,8 +29,8 @@ import type {
MetricDefinition,
MetricVersion,
PiiClassification,
} from '../domain/entities';
import type { AnalyticsTier } from '../domain/tiers';
} from "../domain/entities";
import type { AnalyticsTier } from "../domain/tiers";
/** The verified tenant scope every call must present. */
export interface TenantScope {
@@ -41,9 +41,9 @@ export interface TenantScope {
function assertScope(scope: TenantScope): void {
if (!scope?.tenant_id?.trim() || !scope?.project_id?.trim()) {
throw new SiaxError(
'UNAUTHENTICATED',
'0pt.analytics',
'analytics persistence requires a verified tenant_id + project_id',
"UNAUTHENTICATED",
"0pt.analytics",
"analytics persistence requires a verified tenant_id + project_id",
);
}
}
@@ -90,7 +90,7 @@ function toDataset(r: DatasetRow): AnalyticsDataset {
quality_rule_ids: r.quality_rule_ids ?? [],
retention_days: Number(r.retention_days),
classification: r.classification as PiiClassification,
status: r.status as AnalyticsDataset['status'],
status: r.status as AnalyticsDataset["status"],
...(r.raw_access_exception ? { raw_access_exception: r.raw_access_exception } : {}),
...(r.description ? { description: r.description } : {}),
created_at: iso(r.created_at)!,
@@ -104,10 +104,7 @@ export class AnalyticsDomainRepository {
// ── AnalyticsDataset ─────────────────────────────────────────────────────
async createDataset(
scope: TenantScope,
input: Omit<AnalyticsDataset, 'tenant_id' | 'project_id' | 'created_at' | 'updated_at'>,
): Promise<AnalyticsDataset> {
async createDataset(scope: TenantScope, input: Omit<AnalyticsDataset, "tenant_id" | "project_id" | "created_at" | "updated_at">): Promise<AnalyticsDataset> {
assertScope(scope);
const rows = await this.sql<DatasetRow[]>`
INSERT INTO opt_analytics_dataset
@@ -136,10 +133,7 @@ export class AnalyticsDomainRepository {
return rows[0] ? toDataset(rows[0]) : undefined;
}
async listDatasets(
scope: TenantScope,
filter?: { tier?: AnalyticsTier; status?: string },
): Promise<AnalyticsDataset[]> {
async listDatasets(scope: TenantScope, filter?: { tier?: AnalyticsTier; status?: string }): Promise<AnalyticsDataset[]> {
assertScope(scope);
const rows = await this.sql<DatasetRow[]>`
SELECT * FROM opt_analytics_dataset
@@ -161,20 +155,14 @@ export class AnalyticsDomainRepository {
*/
async createDatasetVersion(
scope: TenantScope,
input: {
dataset_id: string;
contract_hash: string;
schema_contract_version: string;
record_count?: number;
created_by?: string;
},
input: { dataset_id: string; contract_hash: string; schema_contract_version: string; record_count?: number; created_by?: string },
): Promise<DatasetVersion> {
assertScope(scope);
const owner = await this.getDataset(scope, input.dataset_id);
if (!owner) {
throw new SiaxError('NOT_FOUND', '0pt.analytics', 'dataset not found in this tenant');
throw new SiaxError("NOT_FOUND", "0pt.analytics", "dataset not found in this tenant");
}
const id = `dsv_${crypto.randomUUID().replace(/-/g, '')}`;
const id = `dsv_${crypto.randomUUID().replace(/-/g, "")}`;
try {
const rows = await this.sql<
{
@@ -214,8 +202,8 @@ export class AnalyticsDomainRepository {
...(r.created_by ? { created_by: r.created_by } : {}),
};
} catch (err) {
if (String(err).includes('duplicate key')) {
throw new SiaxError('CONFLICT', '0pt.analytics', 'a concurrent writer minted this dataset version');
if (String(err).includes("duplicate key")) {
throw new SiaxError("CONFLICT", "0pt.analytics", "a concurrent writer minted this dataset version");
}
throw err;
}
@@ -224,18 +212,7 @@ export class AnalyticsDomainRepository {
async listDatasetVersions(scope: TenantScope, datasetId: string): Promise<DatasetVersion[]> {
assertScope(scope);
const rows = await this.sql<
{
dataset_version_id: string;
dataset_id: string;
tenant_id: string;
project_id: string;
version: number;
contract_hash: string;
schema_contract_version: string;
record_count: string | number | null;
created_at: string | Date;
created_by: string | null;
}[]
{ dataset_version_id: string; dataset_id: string; tenant_id: string; project_id: string; version: number; contract_hash: string; schema_contract_version: string; record_count: string | number | null; created_at: string | Date; created_by: string | null }[]
>`
SELECT * FROM opt_analytics_dataset_version
WHERE dataset_id = ${datasetId}
@@ -260,29 +237,11 @@ export class AnalyticsDomainRepository {
async createMetric(
scope: TenantScope,
input: Omit<
MetricDefinition,
'tenant_id' | 'project_id' | 'created_at' | 'updated_at' | 'current_version'
>,
input: Omit<MetricDefinition, "tenant_id" | "project_id" | "created_at" | "updated_at" | "current_version">,
): Promise<MetricDefinition> {
assertScope(scope);
const rows = await this.sql<
{
metric_id: string;
tenant_id: string;
project_id: string;
name: string;
dataset_id: string;
aggregation: string;
measure_field: string;
denominator_field: string | null;
allowed_dimension_ids: string[];
current_version: number;
owner_id: string;
description: string | null;
created_at: string | Date;
updated_at: string | Date | null;
}[]
{ metric_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; aggregation: string; measure_field: string; denominator_field: string | null; allowed_dimension_ids: string[]; current_version: number; owner_id: string; description: string | null; created_at: string | Date; updated_at: string | Date | null }[]
>`
INSERT INTO opt_analytics_metric_definition
(metric_id, tenant_id, project_id, name, dataset_id, aggregation, measure_field,
@@ -299,7 +258,7 @@ export class AnalyticsDomainRepository {
project_id: r.project_id,
name: r.name,
dataset_id: r.dataset_id,
aggregation: r.aggregation as MetricDefinition['aggregation'],
aggregation: r.aggregation as MetricDefinition["aggregation"],
measure_field: r.measure_field,
...(r.denominator_field ? { denominator_field: r.denominator_field } : {}),
allowed_dimension_ids: r.allowed_dimension_ids ?? [],
@@ -317,21 +276,10 @@ export class AnalyticsDomainRepository {
input: { metric_id: string; definition_json: string; change_reason?: string; created_by?: string },
): Promise<MetricVersion> {
assertScope(scope);
const hash = crypto.createHash('sha256').update(input.definition_json).digest('hex');
const id = `mv_${crypto.randomUUID().replace(/-/g, '')}`;
const hash = crypto.createHash("sha256").update(input.definition_json).digest("hex");
const id = `mv_${crypto.randomUUID().replace(/-/g, "")}`;
const rows = await this.sql<
{
metric_version_id: string;
metric_id: string;
tenant_id: string;
project_id: string;
version: number;
definition_json: string;
definition_hash: string;
change_reason: string | null;
created_at: string | Date;
created_by: string | null;
}[]
{ metric_version_id: string; metric_id: string; tenant_id: string; project_id: string; version: number; definition_json: string; definition_hash: string; change_reason: string | null; created_at: string | Date; created_by: string | null }[]
>`
INSERT INTO opt_analytics_metric_version
(metric_version_id, metric_id, tenant_id, project_id, version, definition_json,
@@ -361,22 +309,11 @@ export class AnalyticsDomainRepository {
async createDimension(
scope: TenantScope,
input: Omit<DimensionDefinition, 'tenant_id' | 'project_id' | 'created_at'>,
input: Omit<DimensionDefinition, "tenant_id" | "project_id" | "created_at">,
): Promise<DimensionDefinition> {
assertScope(scope);
const rows = await this.sql<
{
dimension_id: string;
tenant_id: string;
project_id: string;
name: string;
dataset_id: string;
field_name: string;
field_type: string;
classification: string;
description: string | null;
created_at: string | Date;
}[]
{ dimension_id: string; tenant_id: string; project_id: string; name: string; dataset_id: string; field_name: string; field_type: string; classification: string; description: string | null; created_at: string | Date }[]
>`
INSERT INTO opt_analytics_dimension_definition
(dimension_id, tenant_id, project_id, name, dataset_id, field_name, field_type, classification, description)
@@ -392,7 +329,7 @@ export class AnalyticsDomainRepository {
name: r.name,
dataset_id: r.dataset_id,
field_name: r.field_name,
field_type: r.field_type as DimensionDefinition['field_type'],
field_type: r.field_type as DimensionDefinition["field_type"],
classification: r.classification as PiiClassification,
...(r.description ? { description: r.description } : {}),
created_at: iso(r.created_at)!,