170 lines
7.4 KiB
TypeScript
170 lines
7.4 KiB
TypeScript
/**
|
|
* src/analytics/engine/schema.ts
|
|
* ===============================
|
|
* BI-P010 — CLICKHOUSE PRODUCTION SCHEMA.
|
|
*
|
|
* BI-P010's explicit warning is "Avoid one giant unbounded table for every
|
|
* analytics use case." The existing `opt_metric_events` (owned by
|
|
* src/analytics/clickhouse-analytics.ts, PRESERVED untouched) is exactly one
|
|
* table for one job: low-cardinality capability metrics. It stays, and it is
|
|
* NOT extended into the catch-all.
|
|
*
|
|
* Four tables, four jobs, four retention profiles:
|
|
*
|
|
* opt_metric_events (PRE-EXISTING, not defined here) — capability
|
|
* metrics, Map(String,String) dimensions, 90d.
|
|
* opt_analytics_event — high-volume per-tenant event stream. Wide
|
|
* dimension arrays, SHORT TTL (raw events are not
|
|
* the serving tier; they age into the rollups).
|
|
* opt_metric_daily — AggregatingMergeTree rollup fed by a
|
|
* MATERIALIZED VIEW off opt_analytics_event.
|
|
* This is what a dashboard time-series reads;
|
|
* long TTL, tiny footprint.
|
|
* opt_dataset_projection — the SERVING tier (BI-P003) for a CURATED
|
|
* dataset: one row per (tenant, dataset, entity),
|
|
* ReplacingMergeTree so a refresh is idempotent.
|
|
*
|
|
* Why the split is load-bearing, not cosmetic:
|
|
* * different ORDER BY keys. A dashboard filtering (tenant, metric, day)
|
|
* and a projection looked up by (tenant, dataset, entity_key) cannot
|
|
* share one primary key without one of them degenerating to a full scan.
|
|
* * different TTLs. Raw events must expire fast; rollups must not.
|
|
* * different cardinality. `LowCardinality(String)` is a win on
|
|
* tenant/metric/dataset and a LOSS on a free-form entity key, so they
|
|
* cannot be the same column type in the same table.
|
|
*
|
|
* EVERY table leads its ORDER BY with `tenant_id`. That is not stylistic: it
|
|
* is what makes the forced tenant predicate (BI-P011) a primary-key range
|
|
* scan instead of a filter applied after reading everyone's rows, and it is
|
|
* what makes a per-tenant row policy cheap.
|
|
*
|
|
* LEGAL BOUNDARY (inherited verbatim from clickhouse-analytics.ts): this
|
|
* schema is for generic 0pt analytics. It MUST NEVER be pointed at the b0ts
|
|
* ClickHouse pipelines or quarantined market data.
|
|
*/
|
|
|
|
export interface AnalyticsSchemaOptions {
|
|
database: string;
|
|
/** Raw event retention. Deliberately short — rollups carry the history. */
|
|
eventRetentionDays?: number;
|
|
/** Rollup retention. */
|
|
rollupRetentionDays?: number;
|
|
/** Serving-projection retention. */
|
|
projectionRetentionDays?: number;
|
|
}
|
|
|
|
/** Physical tables this module owns (opt_metric_events is NOT one of them). */
|
|
export const ANALYTICS_TABLES = [
|
|
"opt_analytics_event",
|
|
"opt_metric_daily",
|
|
"opt_dataset_projection",
|
|
] as const;
|
|
export type AnalyticsTable = (typeof ANALYTICS_TABLES)[number];
|
|
|
|
/** The materialized view wiring opt_analytics_event -> opt_metric_daily. */
|
|
export const ANALYTICS_MATERIALIZED_VIEWS = ["opt_metric_daily_mv"] as const;
|
|
|
|
/**
|
|
* DDL, in dependency order. Every statement is CREATE ... IF NOT EXISTS so
|
|
* applying it to a live database is idempotent (the same contract
|
|
* `ClickHouseAnalytics.ensureSchema` already keeps).
|
|
*/
|
|
export function analyticsSchemaDdl(opts: AnalyticsSchemaOptions): string[] {
|
|
const db = opts.database;
|
|
const eventTtl = Math.floor(opts.eventRetentionDays ?? 30);
|
|
const rollupTtl = Math.floor(opts.rollupRetentionDays ?? 1095);
|
|
const projectionTtl = Math.floor(opts.projectionRetentionDays ?? 365);
|
|
|
|
return [
|
|
`CREATE DATABASE IF NOT EXISTS ${db}`,
|
|
|
|
// ── raw event stream ────────────────────────────────────────────────
|
|
// Dimensions as parallel arrays (not Map) so ClickHouse can prune on
|
|
// dimension VALUES via the bloom filter index; Map(String,String) forces
|
|
// reading the whole map. `event_id` makes deduplication possible on
|
|
// replay without a ReplacingMergeTree's merge latency.
|
|
`CREATE TABLE IF NOT EXISTS ${db}.opt_analytics_event (
|
|
tenant_id LowCardinality(String),
|
|
project_id LowCardinality(String),
|
|
dataset_id LowCardinality(String),
|
|
metric LowCardinality(String),
|
|
event_id String,
|
|
event_time DateTime64(3, 'UTC'),
|
|
value Float64,
|
|
dim_names Array(LowCardinality(String)),
|
|
dim_values Array(String),
|
|
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
INDEX idx_dim_values dim_values TYPE bloom_filter(0.01) GRANULARITY 4
|
|
)
|
|
ENGINE = MergeTree()
|
|
PARTITION BY toYYYYMM(event_time)
|
|
ORDER BY (tenant_id, project_id, dataset_id, metric, event_time)
|
|
TTL toDateTime(event_time) + INTERVAL ${eventTtl} DAY
|
|
SETTINGS index_granularity = 8192`,
|
|
|
|
// ── daily rollup (serving tier for time series) ─────────────────────
|
|
`CREATE TABLE IF NOT EXISTS ${db}.opt_metric_daily (
|
|
tenant_id LowCardinality(String),
|
|
project_id LowCardinality(String),
|
|
dataset_id LowCardinality(String),
|
|
metric LowCardinality(String),
|
|
day Date,
|
|
sum_state AggregateFunction(sum, Float64),
|
|
count_state AggregateFunction(count, Float64),
|
|
min_state AggregateFunction(min, Float64),
|
|
max_state AggregateFunction(max, Float64)
|
|
)
|
|
ENGINE = AggregatingMergeTree()
|
|
PARTITION BY toYYYYMM(day)
|
|
ORDER BY (tenant_id, project_id, dataset_id, metric, day)
|
|
TTL day + INTERVAL ${rollupTtl} DAY`,
|
|
|
|
`CREATE MATERIALIZED VIEW IF NOT EXISTS ${db}.opt_metric_daily_mv
|
|
TO ${db}.opt_metric_daily AS
|
|
SELECT
|
|
tenant_id,
|
|
project_id,
|
|
dataset_id,
|
|
metric,
|
|
toDate(event_time) AS day,
|
|
sumState(value) AS sum_state,
|
|
countState(value) AS count_state,
|
|
minState(value) AS min_state,
|
|
maxState(value) AS max_state
|
|
FROM ${db}.opt_analytics_event
|
|
GROUP BY tenant_id, project_id, dataset_id, metric, day`,
|
|
|
|
// ── serving projection for a CURATED dataset ────────────────────────
|
|
// ReplacingMergeTree(version) so refreshProjection is idempotent: the
|
|
// same (tenant, dataset, entity_key) re-ingested at a higher version
|
|
// supersedes rather than duplicates.
|
|
`CREATE TABLE IF NOT EXISTS ${db}.opt_dataset_projection (
|
|
tenant_id LowCardinality(String),
|
|
project_id LowCardinality(String),
|
|
dataset_id LowCardinality(String),
|
|
entity_key String,
|
|
as_of DateTime64(3, 'UTC'),
|
|
version UInt64,
|
|
attributes Map(String, String),
|
|
measures Map(String, Float64),
|
|
INDEX idx_entity entity_key TYPE bloom_filter(0.01) GRANULARITY 4
|
|
)
|
|
ENGINE = ReplacingMergeTree(version)
|
|
PARTITION BY (tenant_id)
|
|
ORDER BY (tenant_id, project_id, dataset_id, entity_key)
|
|
TTL toDateTime(as_of) + INTERVAL ${projectionTtl} DAY`,
|
|
];
|
|
}
|
|
|
|
/**
|
|
* Statement that proves a table's ORDER BY leads with tenant_id — used by the
|
|
* BI-P011 test suite so "tenant-first primary key" is asserted against the
|
|
* live server, not against this file's own comments.
|
|
*/
|
|
export function sortingKeySql(db: string, table: string): string {
|
|
return (
|
|
`SELECT sorting_key FROM system.tables ` +
|
|
`WHERE database = '${db}' AND name = '${table}' FORMAT TSV`
|
|
);
|
|
}
|