Files
0pt/src/analytics/clickhouse-analytics.ts
T
Simon AxelssonandClaude Fable 5 b46807da75 feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-31 16:49:32 +02:00

234 lines
8.2 KiB
TypeScript

/**
* src/analytics/clickhouse-analytics.ts
* ======================================
* ClickHouse-backed Analytics capability (Master-DoD §10 Analytics:
* ClickHouse, tenant isolation, dataset/event ingest, query limits, metric
* query capability, retention).
*
* LEGAL BOUNDARY (absolute): this module is a GENERIC analytics adapter for
* 0pt capability events/dataset metrics. It has NO connection to, and MUST
* NEVER be pointed at, the b0ts ClickHouse pipelines (Ratsit/person-PII —
* separate legal case) or any quarantined market='sverige'/'norge' market
* data (written DPIA prohibition). Nothing here reads services/scrapers/*.
*
* Tenant isolation model:
* - every table carries a `tenant_id` column;
* - EVERY query built by this service injects `tenant_id = {tenant}` as a
* parameterized predicate — callers cannot omit it;
* - callers NEVER pass raw SQL: the query surface is a closed metric-query
* DSL (metric + optional group-by + time range) compiled here, which is
* what makes the injection guarantee enforceable.
*
* Query limits (§10): every SELECT runs with readonly=1,
* max_execution_time, max_result_rows — enforced via ClickHouse settings on
* the HTTP interface.
*
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
*/
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
export interface ClickHouseAnalyticsOptions {
baseUrl: string;
/** "user:password" — from env/secret ref. */
basicAuth?: string;
database?: string;
timeoutMs?: number;
/** SELECT limits (§10 query limits). */
maxExecutionTimeSec?: number;
maxResultRows?: number;
}
export interface AnalyticsEvent {
tenant_id: string;
dataset_id: string;
metric: string;
value: number;
/** ISO timestamp; defaults to now. */
event_time?: string;
dimensions?: Record<string, string>;
}
export interface MetricQuery {
tenant_id: string;
dataset_id?: string;
metric: string;
aggregate: "sum" | "avg" | "min" | "max" | "count";
/** Group by one dimension key (validated identifier). */
group_by?: string;
from_time?: string;
to_time?: string;
/** Row limit; clamped to maxResultRows. */
limit?: number;
}
export interface MetricQueryRow {
group?: string;
value: number;
}
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
export class ClickHouseAnalytics {
private readonly baseUrl: string;
private readonly authHeader?: string;
private readonly database: string;
private readonly timeoutMs: number;
private readonly maxExecutionTimeSec: number;
private readonly maxResultRows: number;
constructor(opts: ClickHouseAnalyticsOptions) {
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
this.database = opts.database ?? "opt_analytics";
this.timeoutMs = opts.timeoutMs ?? 30_000;
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
this.maxResultRows = opts.maxResultRows ?? 10_000;
}
/** Create the events table (idempotent) with retention TTL (§10 retention). */
async ensureSchema(retentionDays = 90): Promise<void> {
await this.exec(`CREATE DATABASE IF NOT EXISTS ${this.database}`);
await this.exec(
`CREATE TABLE IF NOT EXISTS ${this.database}.opt_metric_events (
tenant_id LowCardinality(String),
dataset_id String,
metric LowCardinality(String),
value Float64,
event_time DateTime64(3, 'UTC'),
dimensions Map(String, String)
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, metric, dataset_id, event_time)
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
);
}
/** Ingest events (JSONEachRow). tenant_id is required per event. */
async ingest(events: AnalyticsEvent[]): Promise<{ ingested: number }> {
if (events.length === 0) return { ingested: 0 };
for (const e of events) {
if (!e.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
}
if (!Number.isFinite(e.value)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
}
}
const rows = events
.map((e) =>
JSON.stringify({
tenant_id: e.tenant_id,
dataset_id: e.dataset_id,
metric: e.metric,
value: e.value,
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
dimensions: e.dimensions ?? {},
})
)
.join("\n");
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
return { ingested: events.length };
}
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
if (!q.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
}
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
}
const agg =
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
// Parameterized query — values travel as HTTP params, never spliced.
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
if (q.dataset_id) {
conditions.push("dataset_id = {dataset:String}");
params.param_dataset = q.dataset_id;
}
if (q.from_time) {
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
params.param_fromTime = q.from_time;
}
if (q.to_time) {
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
params.param_toTime = q.to_time;
}
const sql =
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
`FROM ${this.database}.opt_metric_events ` +
`WHERE ${conditions.join(" AND ")} ` +
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
`LIMIT ${limit} FORMAT JSON`;
const json = (await this.exec(sql, undefined, {
readonly: "1",
max_execution_time: String(this.maxExecutionTimeSec),
max_result_rows: String(this.maxResultRows),
...params,
})) as { data?: { grp?: string; value: number | string }[] };
return (json?.data ?? []).map((r) => ({
...(r.grp !== undefined ? { group: r.grp } : {}),
value: Number(r.value),
}));
}
async health(): Promise<boolean> {
try {
await this.exec("SELECT 1 FORMAT JSON");
return true;
} catch {
return false;
}
}
// -----------------------------------------------------------------------
private async exec(
sql: string,
body?: string,
settings?: Record<string, string>
): Promise<unknown> {
const url = new URL(this.baseUrl);
url.searchParams.set("query", sql);
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
const headers: Record<string, string> = {};
if (this.authHeader) headers["Authorization"] = this.authHeader;
let res: Response;
try {
res = await fetch(url.toString(), {
method: "POST",
headers,
body: body ?? "",
signal: AbortSignal.timeout(this.timeoutMs),
});
} catch (cause) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
}
const text = await res.text();
if (!res.ok) {
// ClickHouse error text can include query fragments — keep only the code.
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
throw new SiaxError(
siaxErrorCodeForUpstreamStatus(res.status),
"clickhouse",
`query failed (CH code ${code})`
);
}
if (!text) return undefined;
try {
return JSON.parse(text);
} catch {
return text;
}
}
}