/** * src/analytics/engine/schema.ts * =============================== * BI-P010 — CLICKHOUSE PRODUCTION SCHEMA. * * BI-P010's explicit warning is "Avoid one giant unbounded table for every * analytics use case." The existing `opt_metric_events` (owned by * src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one * table for one job: low-cardinality capability metrics. It stays, and it is * NOT extended into the catch-all. * * Four tables, four jobs, four retention profiles: * * opt_metric_events (PRE-EXISTING, not defined here) — capability * metrics, Map(String,String) dimensions, 90d. * opt_analytics_event — high-volume per-tenant event stream. Wide * dimension arrays, SHORT TTL (raw events are not * the serving tier; they age into the rollups). * opt_metric_daily — AggregatingMergeTree rollup fed by a * MATERIALIZED VIEW off opt_analytics_event. * This is what a dashboard time-series reads; * long TTL, tiny footprint. * opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED * dataset: one row per (tenant, dataset, entity), * ReplacingMergeTree so a refresh is idempotent. * * Why the split is load-bearing, not cosmetic: * * different ORDER BY keys. A dashboard filtering (tenant, metric, day) * and a projection looked up by (tenant, dataset, entity_key) cannot * share one primary key without one of them degenerating to a full scan. * * different TTLs. Raw events must expire fast; rollups must not. * * different cardinality. `LowCardinality(String)` is a win on * tenant/metric/dataset and a LOSS on a free-form entity key, so they * cannot be the same column type in the same table. * * EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it * is what makes the forced tenant predicate (BI-P011) a primary-key range * scan instead of a filter applied after reading everyone's rows, and it is * what makes a per-tenant row policy cheap. * * LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this * schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts * ClickHouse pipelines or quarantined market data. */ export interface AnalyticsSchemaOptions { database: string; /** Raw event retention. Deliberately short — rollups carry the history. */ eventRetentionDays?: number; /** Rollup retention. */ rollupRetentionDays?: number; /** Serving-projection retention. */ projectionRetentionDays?: number; } /** Physical tables this module owns (opt_metric_events is NOT one of them). */ export const ANALYTICS_TABLES = [ "opt_analytics_event", "opt_metric_daily", "opt_dataset_projection", ] as const; export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number]; /** The materialized view wiring opt_analytics_event -> opt_metric_daily. */ export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const; /** * DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so * applying it to a live database is idempotent (the same contract * `ClickHouseAnalytics.ensureSchema` already keeps). */ export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] { const db = opts.database; const eventTtl = Math.floor(opts.eventRetentionDays ?? 30); const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095); const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365); return [ `CREATE DATABASE IF NOT EXISTS ${db}`, // ── raw event stream ──────────────────────────────────────────────── // Dimensions as parallel arrays (not Map) so ClickHouse can prune on // dimension VALUES via the bloom filter index; Map(String,String) forces // reading the whole map. `event_id` makes deduplication possible on // replay without a ReplacingMergeTree's merge latency. `CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event ( tenant_id LowCardinality(String), project_id LowCardinality(String), dataset_id LowCardinality(String), metric LowCardinality(String), event_id String, event_time DateTime64(3, 'UTC'), value Float64, dim_names Array(LowCardinality(String)), dim_values Array(String), ingested_at DateTime64(3, 'UTC') DEFAULT now64(3), INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4 ) ENGINE = MergeTree() PARTITION BY toYYYYMM(event_time) ORDER BY (tenant_id, project_id, dataset_id, metric, event_time) TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY SETTINGS index_granularity = 8192`, // ── daily rollup (serving tier for time series) ───────────────────── `CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily ( tenant_id LowCardinality(String), project_id LowCardinality(String), dataset_id LowCardinality(String), metric LowCardinality(String), day Date, sum_state AggregateFunction(sum, Float64), count_state AggregateFunction(count, Float64), min_state AggregateFunction(min, Float64), max_state AggregateFunction(max, Float64) ) ENGINE = AggregatingMergeTree() PARTITION BY toYYYYMM(day) ORDER BY (tenant_id, project_id, dataset_id, metric, day) TTL day + INTERVAL ${rollupTtl} DAY`, `CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv TO ${db}.opt_metric_daily AS SELECT tenant_id, project_id, dataset_id, metric, toDate(event_time) AS day, sumState(value) AS sum_state, countState(value) AS count_state, minState(value) AS min_state, maxState(value) AS max_state FROM ${db}.opt_analytics_event GROUP BY tenant_id, project_id, dataset_id, metric, day`, // ── serving projection for a CURATED dataset ──────────────────────── // ReplacingMergeTree(version) so refreshProjection is idempotent: the // same (tenant, dataset, entity_key) re-ingested at a higher version // supersedes rather than duplicates. `CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection ( tenant_id LowCardinality(String), project_id LowCardinality(String), dataset_id LowCardinality(String), entity_key String, as_of DateTime64(3, 'UTC'), version UInt64, attributes Map(String, String), measures Map(String, Float64), INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4 ) ENGINE = ReplacingMergeTree(version) PARTITION BY (tenant_id) ORDER BY (tenant_id, project_id, dataset_id, entity_key) TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`, ]; } /** * Statement that proves a table's ORDER BY leads with tenant_id — used by the * BI-P011 test suite so "tenant-first primary key" is asserted against the * live server, not against this file's own comments. */ export function sortingKeySql(db: string, table: string): string { return ( `SELECT sorting_key FROM system.tables ` + `WHERE database = '${db}' AND name = '${table}' FORMAT TSV` ); }