- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
234 lines
8.2 KiB
TypeScript
234 lines
8.2 KiB
TypeScript
/**
|
|
* src/analytics/clickhouse-analytics.ts
|
|
* ======================================
|
|
* ClickHouse-backed Analytics capability (Master-DoD §10 Analytics:
|
|
* ClickHouse, tenant isolation, dataset/event ingest, query limits, metric
|
|
* query capability, retention).
|
|
*
|
|
* LEGAL BOUNDARY (absolute): this module is a GENERIC analytics adapter for
|
|
* 0pt capability events/dataset metrics. It has NO connection to, and MUST
|
|
* NEVER be pointed at, the b0ts ClickHouse pipelines (Ratsit/person-PII —
|
|
* separate legal case) or any quarantined market='sverige'/'norge' market
|
|
* data (written DPIA prohibition). Nothing here reads services/scrapers/*.
|
|
*
|
|
* Tenant isolation model:
|
|
* - every table carries a `tenant_id` column;
|
|
* - EVERY query built by this service injects `tenant_id = {tenant}` as a
|
|
* parameterized predicate — callers cannot omit it;
|
|
* - callers NEVER pass raw SQL: the query surface is a closed metric-query
|
|
* DSL (metric + optional group-by + time range) compiled here, which is
|
|
* what makes the injection guarantee enforceable.
|
|
*
|
|
* Query limits (§10): every SELECT runs with readonly=1,
|
|
* max_execution_time, max_result_rows — enforced via ClickHouse settings on
|
|
* the HTTP interface.
|
|
*
|
|
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
|
|
*/
|
|
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
|
|
|
|
export interface ClickHouseAnalyticsOptions {
|
|
baseUrl: string;
|
|
/** "user:password" — from env/secret ref. */
|
|
basicAuth?: string;
|
|
database?: string;
|
|
timeoutMs?: number;
|
|
/** SELECT limits (§10 query limits). */
|
|
maxExecutionTimeSec?: number;
|
|
maxResultRows?: number;
|
|
}
|
|
|
|
export interface AnalyticsEvent {
|
|
tenant_id: string;
|
|
dataset_id: string;
|
|
metric: string;
|
|
value: number;
|
|
/** ISO timestamp; defaults to now. */
|
|
event_time?: string;
|
|
dimensions?: Record<string, string>;
|
|
}
|
|
|
|
export interface MetricQuery {
|
|
tenant_id: string;
|
|
dataset_id?: string;
|
|
metric: string;
|
|
aggregate: "sum" | "avg" | "min" | "max" | "count";
|
|
/** Group by one dimension key (validated identifier). */
|
|
group_by?: string;
|
|
from_time?: string;
|
|
to_time?: string;
|
|
/** Row limit; clamped to maxResultRows. */
|
|
limit?: number;
|
|
}
|
|
|
|
export interface MetricQueryRow {
|
|
group?: string;
|
|
value: number;
|
|
}
|
|
|
|
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
|
|
|
export class ClickHouseAnalytics {
|
|
private readonly baseUrl: string;
|
|
private readonly authHeader?: string;
|
|
private readonly database: string;
|
|
private readonly timeoutMs: number;
|
|
private readonly maxExecutionTimeSec: number;
|
|
private readonly maxResultRows: number;
|
|
|
|
constructor(opts: ClickHouseAnalyticsOptions) {
|
|
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
|
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
|
|
this.database = opts.database ?? "opt_analytics";
|
|
this.timeoutMs = opts.timeoutMs ?? 30_000;
|
|
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
|
|
this.maxResultRows = opts.maxResultRows ?? 10_000;
|
|
}
|
|
|
|
/** Create the events table (idempotent) with retention TTL (§10 retention). */
|
|
async ensureSchema(retentionDays = 90): Promise<void> {
|
|
await this.exec(`CREATE DATABASE IF NOT EXISTS ${this.database}`);
|
|
await this.exec(
|
|
`CREATE TABLE IF NOT EXISTS ${this.database}.opt_metric_events (
|
|
tenant_id LowCardinality(String),
|
|
dataset_id String,
|
|
metric LowCardinality(String),
|
|
value Float64,
|
|
event_time DateTime64(3, 'UTC'),
|
|
dimensions Map(String, String)
|
|
)
|
|
ENGINE = MergeTree()
|
|
PARTITION BY toYYYYMM(event_time)
|
|
ORDER BY (tenant_id, metric, dataset_id, event_time)
|
|
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
|
|
);
|
|
}
|
|
|
|
/** Ingest events (JSONEachRow). tenant_id is required per event. */
|
|
async ingest(events: AnalyticsEvent[]): Promise<{ ingested: number }> {
|
|
if (events.length === 0) return { ingested: 0 };
|
|
for (const e of events) {
|
|
if (!e.tenant_id?.trim()) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
|
|
}
|
|
if (!Number.isFinite(e.value)) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
|
|
}
|
|
}
|
|
const rows = events
|
|
.map((e) =>
|
|
JSON.stringify({
|
|
tenant_id: e.tenant_id,
|
|
dataset_id: e.dataset_id,
|
|
metric: e.metric,
|
|
value: e.value,
|
|
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
|
|
dimensions: e.dimensions ?? {},
|
|
})
|
|
)
|
|
.join("\n");
|
|
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
|
|
return { ingested: events.length };
|
|
}
|
|
|
|
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
|
|
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
|
|
if (!q.tenant_id?.trim()) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
|
|
}
|
|
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
|
|
}
|
|
const agg =
|
|
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
|
|
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
|
|
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
|
|
|
|
// Parameterized query — values travel as HTTP params, never spliced.
|
|
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
|
|
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
|
|
if (q.dataset_id) {
|
|
conditions.push("dataset_id = {dataset:String}");
|
|
params.param_dataset = q.dataset_id;
|
|
}
|
|
if (q.from_time) {
|
|
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
|
|
params.param_fromTime = q.from_time;
|
|
}
|
|
if (q.to_time) {
|
|
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
|
|
params.param_toTime = q.to_time;
|
|
}
|
|
|
|
const sql =
|
|
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
|
|
`FROM ${this.database}.opt_metric_events ` +
|
|
`WHERE ${conditions.join(" AND ")} ` +
|
|
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
|
|
`LIMIT ${limit} FORMAT JSON`;
|
|
|
|
const json = (await this.exec(sql, undefined, {
|
|
readonly: "1",
|
|
max_execution_time: String(this.maxExecutionTimeSec),
|
|
max_result_rows: String(this.maxResultRows),
|
|
...params,
|
|
})) as { data?: { grp?: string; value: number | string }[] };
|
|
|
|
return (json?.data ?? []).map((r) => ({
|
|
...(r.grp !== undefined ? { group: r.grp } : {}),
|
|
value: Number(r.value),
|
|
}));
|
|
}
|
|
|
|
async health(): Promise<boolean> {
|
|
try {
|
|
await this.exec("SELECT 1 FORMAT JSON");
|
|
return true;
|
|
} catch {
|
|
return false;
|
|
}
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
private async exec(
|
|
sql: string,
|
|
body?: string,
|
|
settings?: Record<string, string>
|
|
): Promise<unknown> {
|
|
const url = new URL(this.baseUrl);
|
|
url.searchParams.set("query", sql);
|
|
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
|
|
|
|
const headers: Record<string, string> = {};
|
|
if (this.authHeader) headers["Authorization"] = this.authHeader;
|
|
|
|
let res: Response;
|
|
try {
|
|
res = await fetch(url.toString(), {
|
|
method: "POST",
|
|
headers,
|
|
body: body ?? "",
|
|
signal: AbortSignal.timeout(this.timeoutMs),
|
|
});
|
|
} catch (cause) {
|
|
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
|
|
}
|
|
const text = await res.text();
|
|
if (!res.ok) {
|
|
// ClickHouse error text can include query fragments — keep only the code.
|
|
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
|
|
throw new SiaxError(
|
|
siaxErrorCodeForUpstreamStatus(res.status),
|
|
"clickhouse",
|
|
`query failed (CH code ${code})`
|
|
);
|
|
}
|
|
if (!text) return undefined;
|
|
try {
|
|
return JSON.parse(text);
|
|
} catch {
|
|
return text;
|
|
}
|
|
}
|
|
}
|