Files
0pt/src/analytics/engine/schema.ts
T

170 lines
7.4 KiB
TypeScript

/**
* src/analytics/engine/schema.ts
* ===============================
* BI-P010 — CLICKHOUSE PRODUCTION SCHEMA.
*
* BI-P010's explicit warning is "Avoid one giant unbounded table for every
* analytics use case." The existing `opt_metric_events` (owned by
* src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one
* table for one job: low-cardinality capability metrics. It stays, and it is
* NOT extended into the catch-all.
*
* Four tables, four jobs, four retention profiles:
*
* opt_metric_events (PRE-EXISTING, not defined here) — capability
* metrics, Map(String,String) dimensions, 90d.
* opt_analytics_event — high-volume per-tenant event stream. Wide
* dimension arrays, SHORT TTL (raw events are not
* the serving tier; they age into the rollups).
* opt_metric_daily — AggregatingMergeTree rollup fed by a
* MATERIALIZED VIEW off opt_analytics_event.
* This is what a dashboard time-series reads;
* long TTL, tiny footprint.
* opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED
* dataset: one row per (tenant, dataset, entity),
* ReplacingMergeTree so a refresh is idempotent.
*
* Why the split is load-bearing, not cosmetic:
* * different ORDER BY keys. A dashboard filtering (tenant, metric, day)
* and a projection looked up by (tenant, dataset, entity_key) cannot
* share one primary key without one of them degenerating to a full scan.
* * different TTLs. Raw events must expire fast; rollups must not.
* * different cardinality. `LowCardinality(String)` is a win on
* tenant/metric/dataset and a LOSS on a free-form entity key, so they
* cannot be the same column type in the same table.
*
* EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it
* is what makes the forced tenant predicate (BI-P011) a primary-key range
* scan instead of a filter applied after reading everyone's rows, and it is
* what makes a per-tenant row policy cheap.
*
* LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this
* schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts
* ClickHouse pipelines or quarantined market data.
*/
export interface AnalyticsSchemaOptions {
database: string;
/** Raw event retention. Deliberately short — rollups carry the history. */
eventRetentionDays?: number;
/** Rollup retention. */
rollupRetentionDays?: number;
/** Serving-projection retention. */
projectionRetentionDays?: number;
}
/** Physical tables this module owns (opt_metric_events is NOT one of them). */
export const ANALYTICS_TABLES = [
"opt_analytics_event",
"opt_metric_daily",
"opt_dataset_projection",
] as const;
export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number];
/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */
export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const;
/**
* DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so
* applying it to a live database is idempotent (the same contract
* `ClickHouseAnalytics.ensureSchema` already keeps).
*/
export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] {
const db = opts.database;
const eventTtl = Math.floor(opts.eventRetentionDays ?? 30);
const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095);
const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365);
return [
`CREATE DATABASE IF NOT EXISTS ${db}`,
// ── raw event stream ────────────────────────────────────────────────
// Dimensions as parallel arrays (not Map) so ClickHouse can prune on
// dimension VALUES via the bloom filter index; Map(String,String) forces
// reading the whole map. `event_id` makes deduplication possible on
// replay without a ReplacingMergeTree's merge latency.
`CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
metric LowCardinality(String),
event_id String,
event_time DateTime64(3, 'UTC'),
value Float64,
dim_names Array(LowCardinality(String)),
dim_values Array(String),
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4
)
ENGINE = MergeTree()
PARTITION BY toYYYYMM(event_time)
ORDER BY (tenant_id, project_id, dataset_id, metric, event_time)
TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY
SETTINGS index_granularity = 8192`,
// ── daily rollup (serving tier for time series) ─────────────────────
`CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
metric LowCardinality(String),
day Date,
sum_state AggregateFunction(sum, Float64),
count_state AggregateFunction(count, Float64),
min_state AggregateFunction(min, Float64),
max_state AggregateFunction(max, Float64)
)
ENGINE = AggregatingMergeTree()
PARTITION BY toYYYYMM(day)
ORDER BY (tenant_id, project_id, dataset_id, metric, day)
TTL day + INTERVAL ${rollupTtl} DAY`,
`CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv
TO ${db}.opt_metric_daily AS
SELECT
tenant_id,
project_id,
dataset_id,
metric,
toDate(event_time) AS day,
sumState(value) AS sum_state,
countState(value) AS count_state,
minState(value) AS min_state,
maxState(value) AS max_state
FROM ${db}.opt_analytics_event
GROUP BY tenant_id, project_id, dataset_id, metric, day`,
// ── serving projection for a CURATED dataset ────────────────────────
// ReplacingMergeTree(version) so refreshProjection is idempotent: the
// same (tenant, dataset, entity_key) re-ingested at a higher version
// supersedes rather than duplicates.
`CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection (
tenant_id LowCardinality(String),
project_id LowCardinality(String),
dataset_id LowCardinality(String),
entity_key String,
as_of DateTime64(3, 'UTC'),
version UInt64,
attributes Map(String, String),
measures Map(String, Float64),
INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4
)
ENGINE = ReplacingMergeTree(version)
PARTITION BY (tenant_id)
ORDER BY (tenant_id, project_id, dataset_id, entity_key)
TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`,
];
}
/**
* Statement that proves a table's ORDER BY leads with tenant_id — used by the
* BI-P011 test suite so "tenant-first primary key" is asserted against the
* live server, not against this file's own comments.
*/
export function sortingKeySql(db: string, table: string): string {
return (
`SELECT sorting_key FROM system.tables ` +
`WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
);
}