feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
da58176735
commit
b46807da75
@@ -0,0 +1,233 @@
|
||||
/**
|
||||
* src/analytics/clickhouse-analytics.ts
|
||||
* ======================================
|
||||
* ClickHouse-backed Analytics capability (Master-DoD §10 Analytics:
|
||||
* ClickHouse, tenant isolation, dataset/event ingest, query limits, metric
|
||||
* query capability, retention).
|
||||
*
|
||||
* LEGAL BOUNDARY (absolute): this module is a GENERIC analytics adapter for
|
||||
* 0pt capability events/dataset metrics. It has NO connection to, and MUST
|
||||
* NEVER be pointed at, the b0ts ClickHouse pipelines (Ratsit/person-PII —
|
||||
* separate legal case) or any quarantined market='sverige'/'norge' market
|
||||
* data (written DPIA prohibition). Nothing here reads services/scrapers/*.
|
||||
*
|
||||
* Tenant isolation model:
|
||||
* - every table carries a `tenant_id` column;
|
||||
* - EVERY query built by this service injects `tenant_id = {tenant}` as a
|
||||
* parameterized predicate — callers cannot omit it;
|
||||
* - callers NEVER pass raw SQL: the query surface is a closed metric-query
|
||||
* DSL (metric + optional group-by + time range) compiled here, which is
|
||||
* what makes the injection guarantee enforceable.
|
||||
*
|
||||
* Query limits (§10): every SELECT runs with readonly=1,
|
||||
* max_execution_time, max_result_rows — enforced via ClickHouse settings on
|
||||
* the HTTP interface.
|
||||
*
|
||||
* Retention: tables are created WITH TTL (event_time + INTERVAL n DAY).
|
||||
*/
|
||||
import { SiaxError, siaxErrorCodeForUpstreamStatus } from "../platform/errors";
|
||||
|
||||
export interface ClickHouseAnalyticsOptions {
|
||||
baseUrl: string;
|
||||
/** "user:password" — from env/secret ref. */
|
||||
basicAuth?: string;
|
||||
database?: string;
|
||||
timeoutMs?: number;
|
||||
/** SELECT limits (§10 query limits). */
|
||||
maxExecutionTimeSec?: number;
|
||||
maxResultRows?: number;
|
||||
}
|
||||
|
||||
export interface AnalyticsEvent {
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
metric: string;
|
||||
value: number;
|
||||
/** ISO timestamp; defaults to now. */
|
||||
event_time?: string;
|
||||
dimensions?: Record<string, string>;
|
||||
}
|
||||
|
||||
export interface MetricQuery {
|
||||
tenant_id: string;
|
||||
dataset_id?: string;
|
||||
metric: string;
|
||||
aggregate: "sum" | "avg" | "min" | "max" | "count";
|
||||
/** Group by one dimension key (validated identifier). */
|
||||
group_by?: string;
|
||||
from_time?: string;
|
||||
to_time?: string;
|
||||
/** Row limit; clamped to maxResultRows. */
|
||||
limit?: number;
|
||||
}
|
||||
|
||||
export interface MetricQueryRow {
|
||||
group?: string;
|
||||
value: number;
|
||||
}
|
||||
|
||||
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]*$/;
|
||||
|
||||
export class ClickHouseAnalytics {
|
||||
private readonly baseUrl: string;
|
||||
private readonly authHeader?: string;
|
||||
private readonly database: string;
|
||||
private readonly timeoutMs: number;
|
||||
private readonly maxExecutionTimeSec: number;
|
||||
private readonly maxResultRows: number;
|
||||
|
||||
constructor(opts: ClickHouseAnalyticsOptions) {
|
||||
this.baseUrl = opts.baseUrl.replace(/\/+$/, "");
|
||||
this.authHeader = opts.basicAuth ? `Basic ${Buffer.from(opts.basicAuth).toString("base64")}` : undefined;
|
||||
this.database = opts.database ?? "opt_analytics";
|
||||
this.timeoutMs = opts.timeoutMs ?? 30_000;
|
||||
this.maxExecutionTimeSec = opts.maxExecutionTimeSec ?? 10;
|
||||
this.maxResultRows = opts.maxResultRows ?? 10_000;
|
||||
}
|
||||
|
||||
/** Create the events table (idempotent) with retention TTL (§10 retention). */
|
||||
async ensureSchema(retentionDays = 90): Promise<void> {
|
||||
await this.exec(`CREATE DATABASE IF NOT EXISTS ${this.database}`);
|
||||
await this.exec(
|
||||
`CREATE TABLE IF NOT EXISTS ${this.database}.opt_metric_events (
|
||||
tenant_id LowCardinality(String),
|
||||
dataset_id String,
|
||||
metric LowCardinality(String),
|
||||
value Float64,
|
||||
event_time DateTime64(3, 'UTC'),
|
||||
dimensions Map(String, String)
|
||||
)
|
||||
ENGINE = MergeTree()
|
||||
PARTITION BY toYYYYMM(event_time)
|
||||
ORDER BY (tenant_id, metric, dataset_id, event_time)
|
||||
TTL toDateTime(event_time) + INTERVAL ${Math.floor(retentionDays)} DAY`
|
||||
);
|
||||
}
|
||||
|
||||
/** Ingest events (JSONEachRow). tenant_id is required per event. */
|
||||
async ingest(events: AnalyticsEvent[]): Promise<{ ingested: number }> {
|
||||
if (events.length === 0) return { ingested: 0 };
|
||||
for (const e of events) {
|
||||
if (!e.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "every analytics event needs tenant_id");
|
||||
}
|
||||
if (!Number.isFinite(e.value)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `metric value for "${e.metric}" must be finite`);
|
||||
}
|
||||
}
|
||||
const rows = events
|
||||
.map((e) =>
|
||||
JSON.stringify({
|
||||
tenant_id: e.tenant_id,
|
||||
dataset_id: e.dataset_id,
|
||||
metric: e.metric,
|
||||
value: e.value,
|
||||
event_time: (e.event_time ?? new Date().toISOString()).replace("T", " ").replace(/Z$/, ""),
|
||||
dimensions: e.dimensions ?? {},
|
||||
})
|
||||
)
|
||||
.join("\n");
|
||||
await this.exec(`INSERT INTO ${this.database}.opt_metric_events FORMAT JSONEachRow`, rows);
|
||||
return { ingested: events.length };
|
||||
}
|
||||
|
||||
/** Closed metric-query DSL — compiled with a FORCED tenant predicate. */
|
||||
async queryMetric(q: MetricQuery): Promise<MetricQueryRow[]> {
|
||||
if (!q.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", "tenant_id is required on every metric query");
|
||||
}
|
||||
if (q.group_by !== undefined && !IDENT_RE.test(q.group_by)) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.analytics", `invalid group_by identifier`);
|
||||
}
|
||||
const agg =
|
||||
q.aggregate === "count" ? "count()" : `${q.aggregate}(value)`;
|
||||
const groupExpr = q.group_by ? `dimensions['${q.group_by}']` : undefined;
|
||||
const limit = Math.min(q.limit ?? 1000, this.maxResultRows);
|
||||
|
||||
// Parameterized query — values travel as HTTP params, never spliced.
|
||||
const conditions = ["tenant_id = {tenant:String}", "metric = {metric:String}"];
|
||||
const params: Record<string, string> = { param_tenant: q.tenant_id, param_metric: q.metric };
|
||||
if (q.dataset_id) {
|
||||
conditions.push("dataset_id = {dataset:String}");
|
||||
params.param_dataset = q.dataset_id;
|
||||
}
|
||||
if (q.from_time) {
|
||||
conditions.push("event_time >= parseDateTime64BestEffort({fromTime:String})");
|
||||
params.param_fromTime = q.from_time;
|
||||
}
|
||||
if (q.to_time) {
|
||||
conditions.push("event_time < parseDateTime64BestEffort({toTime:String})");
|
||||
params.param_toTime = q.to_time;
|
||||
}
|
||||
|
||||
const sql =
|
||||
`SELECT ${groupExpr ? `${groupExpr} AS grp, ` : ""}${agg} AS value ` +
|
||||
`FROM ${this.database}.opt_metric_events ` +
|
||||
`WHERE ${conditions.join(" AND ")} ` +
|
||||
(groupExpr ? `GROUP BY grp ORDER BY value DESC ` : "") +
|
||||
`LIMIT ${limit} FORMAT JSON`;
|
||||
|
||||
const json = (await this.exec(sql, undefined, {
|
||||
readonly: "1",
|
||||
max_execution_time: String(this.maxExecutionTimeSec),
|
||||
max_result_rows: String(this.maxResultRows),
|
||||
...params,
|
||||
})) as { data?: { grp?: string; value: number | string }[] };
|
||||
|
||||
return (json?.data ?? []).map((r) => ({
|
||||
...(r.grp !== undefined ? { group: r.grp } : {}),
|
||||
value: Number(r.value),
|
||||
}));
|
||||
}
|
||||
|
||||
async health(): Promise<boolean> {
|
||||
try {
|
||||
await this.exec("SELECT 1 FORMAT JSON");
|
||||
return true;
|
||||
} catch {
|
||||
return false;
|
||||
}
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
private async exec(
|
||||
sql: string,
|
||||
body?: string,
|
||||
settings?: Record<string, string>
|
||||
): Promise<unknown> {
|
||||
const url = new URL(this.baseUrl);
|
||||
url.searchParams.set("query", sql);
|
||||
for (const [k, v] of Object.entries(settings ?? {})) url.searchParams.set(k, v);
|
||||
|
||||
const headers: Record<string, string> = {};
|
||||
if (this.authHeader) headers["Authorization"] = this.authHeader;
|
||||
|
||||
let res: Response;
|
||||
try {
|
||||
res = await fetch(url.toString(), {
|
||||
method: "POST",
|
||||
headers,
|
||||
body: body ?? "",
|
||||
signal: AbortSignal.timeout(this.timeoutMs),
|
||||
});
|
||||
} catch (cause) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "clickhouse", "clickhouse unreachable", { cause });
|
||||
}
|
||||
const text = await res.text();
|
||||
if (!res.ok) {
|
||||
// ClickHouse error text can include query fragments — keep only the code.
|
||||
const code = /Code: (\d+)/.exec(text)?.[1] ?? String(res.status);
|
||||
throw new SiaxError(
|
||||
siaxErrorCodeForUpstreamStatus(res.status),
|
||||
"clickhouse",
|
||||
`query failed (CH code ${code})`
|
||||
);
|
||||
}
|
||||
if (!text) return undefined;
|
||||
try {
|
||||
return JSON.parse(text);
|
||||
} catch {
|
||||
return text;
|
||||
}
|
||||
}
|
||||
}
|
||||
Reference in New Issue
Block a user