feat(analytics): BI-P105 — a pre-aggregate serves only while it is provably undrifted
CI (SIAX Cloud) / security (push) Successful in 49s
Security Scan (SIAX Cloud) / security (push) Successful in 50s
CI (SIAX Cloud) / integration (push) Successful in 55s
CI (SIAX Cloud) / quality (push) Successful in 1m20s
CI (SIAX Cloud) / quality (pull_request) Successful in 1m21s
CI (SIAX Cloud) / security (pull_request) Successful in 42s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 37s
CI (SIAX Cloud) / integration (pull_request) Successful in 1m10s

A slow query is a complaint; a drifted pre-aggregate answers instantly, with
confidence, and is wrong. So the router is fail-closed on eight named reasons,
and the one people leave out — drift_never_verified — is the one that matters
on launch day: a brand-new aggregate is exactly as unproven as a drifted one.

Proved against ClickHouse 26.3.10.60, 2M rows, a real materialized view
(evidence/bi-p105/materialization-drift.json):

  after backfill      match, 48 buckets, 801 571 002 both sides
  GDPR erasure        ALTER TABLE ... DELETE on the source. An MV is an INSERT
                      trigger and never sees a mutation, so the aggregate kept
                      the deleted revenue: it would have reported 801 571 002
                      against a true 761 580 554 — 39 990 448 overstated,
                      5.251%, with NOTHING stale. Staleness at the moment of
                      the decision: 1 second.
  guarded router      base_table (state_not_serving)
  counterfactual      a staleness-only router SERVES it — measured, not assumed
  after rebuild       match, and routing returns to the materialization

Drift detection treats a MISSING bucket as drift even when every value present
is correct (late-arriving rows are the sneakiest failure), and an empty
comparison as insufficient_data rather than match — the same false green
BI-P025 found in a quality check over an empty table.

The measured speed-up is reported as a distribution, not a figure: p50 3.4x,
but the run-to-run range spans 0.6x-7.5x on one idle host, so quoting "3.4x"
as a property of the system would be misleading (BI-P104 doctrine).

19 unit tests + 4 against live ClickHouse.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Opus 5
2026-09-07 03:56:32 +02:00
parent 1a12e8a882
commit 96ff601248
5 changed files with 1229 additions and 0 deletions
@@ -0,0 +1,75 @@
{
"clickhouse_version": "26.3.10.60",
"recorded_at": "2026-09-07T01:55:42.821Z",
"source_rows": 2000000,
"after_backfill": {
"verdict": "match",
"buckets_compared": 48,
"revenue_total": {
"source": 801571002,
"aggregate": 801571002,
"delta": 0
}
},
"performance": {
"note": "ClickHouse caches marks and pages between repetitions, so these are WARM numbers on one host with nothing else running. They describe this shape of query on this data, not a system property.",
"repetitions": 9,
"base_table_ms": {
"n": 9,
"min": 48.70366200000012,
"p50": 76.67811500000016,
"p95": 115.04416399999991,
"max": 115.04416399999991,
"spread_ratio": 2.36
},
"pre_aggregate_ms": {
"n": 9,
"min": 15.281603999999788,
"p50": 22.727642999999944,
"p95": 84.41279499999973,
"max": 84.41279499999973,
"spread_ratio": 5.52
},
"speedup_p50": 3.4,
"speedup_range_min_max": [0.6, 7.5]
},
"gdpr_erasure_drill": {
"mutation": "ALTER TABLE sales DELETE WHERE tenant_id='t_alpha' AND customer_id < 250",
"verdict": "drifted",
"buckets_compared": 48,
"revenue_true_after_erasure": 761580554,
"revenue_pre_aggregate_would_report": 801571002,
"revenue_overstatement": 39990448,
"revenue_overstatement_pct": 5.251,
"worst_bucket": {
"key": "2026-08-28|FI",
"metric": "revenue",
"source_value": 15830779,
"aggregate_value": 16699145,
"absolute_delta": 868366,
"relative_delta": 0.05485301765630106
},
"guarded_router_decision": {
"source": "base_table",
"refusal": "state_not_serving",
"detail": "materialization state is \"drifted\""
},
"staleness_only_router_would_serve": true,
"staleness_seconds_at_decision": 1,
"note": "The aggregate was NOT stale by any clock: it had just been refreshed, and every freshness check passed. Only the source comparison caught it. This is why staleness alone is not a correctness guarantee."
},
"after_rebuild": {
"verdict": "match",
"revenue_total": {
"source": 761580554,
"aggregate": 761580554,
"delta": 0
},
"routing": {
"source": "materialization",
"materialization_id": "mv_sales_daily",
"target_ref": "opt_p105_mtql9p2f.sales_daily",
"staleness_seconds": 0
}
}
}
@@ -0,0 +1,170 @@
/**
* src/analytics/materialization/drift-detector.ts
* ================================================
* BI-P105 — is the pre-aggregate still telling the truth?
*
* The comparison is bucket-by-bucket between the aggregate and a recomputation
* from the source over the same window. Three things make it a real detector
* rather than a ceremony:
*
* 1. A MISSING BUCKET IS DRIFT. The nastiest pre-aggregate failure is not a
* wrong value, it is an absent row: late-arriving data, a materialized view
* that was created after the backfill, a refresh that failed silently. Every
* value it DOES hold is correct, so a value-only comparison passes while the
* total is wrong. Set difference is checked first, before any arithmetic.
*
* 2. ZERO BUCKETS IS `insufficient_data`, NEVER `match`. A comparison over an
* empty window finds no discrepancies — the same false green BI-P025 found
* in a quality check over an empty table. The router treats
* `insufficient_data` as "do not serve", not as "fine".
*
* 3. RELATIVE AND ABSOLUTE TOLERANCE BOTH HAVE TO HOLD. A 0.1% tolerance on a
* revenue bucket of 40M is 40 000 kr of invisible error; an absolute-only
* tolerance flags every rounding difference on a large bucket. Requiring
* both means a discrepancy has to be small in kronor AND small in
* proportion.
*/
import type { DriftTolerance, DriftVerdict, MaterializationStatus } from './materialization';
/** One aggregated row, keyed by its dimension tuple. */
export interface AggregateBucket {
/** Stable key for the dimension tuple, e.g. "2026-09-01|SE". */
key: string;
/** metric name -> value. */
values: Record<string, number>;
}
export interface DriftDiscrepancy {
key: string;
metric: string;
source_value: number;
aggregate_value: number;
absolute_delta: number;
/** |delta| / |source|, or Infinity when the source value is 0 and the aggregate is not. */
relative_delta: number;
}
export interface DriftReport {
verdict: DriftVerdict;
checked_at: string;
buckets_compared: number;
/** In the source recomputation, absent from the aggregate. Always drift. */
missing_in_aggregate: string[];
/** In the aggregate, absent from the source. Always drift (deleted rows). */
extra_in_aggregate: string[];
discrepancies: DriftDiscrepancy[];
worst?: DriftDiscrepancy;
tolerance: DriftTolerance;
/**
* What a caller would have been told if it had trusted the aggregate: the
* total the aggregate reports vs the total the source computes, per metric.
* This is the number that makes a drift report actionable — "0.4% off" means
* nothing until it is "1 284 500 kr reported, 1 271 900 kr true".
*/
totals: Record<string, { source: number; aggregate: number; delta: number }>;
}
const MAX_REPORTED_DISCREPANCIES = 50;
function relative(sourceValue: number, delta: number): number {
if (sourceValue === 0) return delta === 0 ? 0 : Number.POSITIVE_INFINITY;
return Math.abs(delta / sourceValue);
}
/**
* Compare a recomputation from source against the pre-aggregate.
*
* `fromSource` is the truth. Both sides must cover the same window; the caller
* is responsible for that, and `missing_in_aggregate` will catch it loudly if
* they do not.
*/
export function detectDrift(args: {
fromSource: AggregateBucket[];
fromAggregate: AggregateBucket[];
tolerance: DriftTolerance;
metrics: string[];
now?: Date;
}): DriftReport {
const { fromSource, fromAggregate, tolerance, metrics } = args;
const checked_at = (args.now ?? new Date()).toISOString();
const sourceByKey = new Map(fromSource.map((b) => [b.key, b]));
const aggByKey = new Map(fromAggregate.map((b) => [b.key, b]));
const missing_in_aggregate = [...sourceByKey.keys()].filter((k) => !aggByKey.has(k)).sort();
const extra_in_aggregate = [...aggByKey.keys()].filter((k) => !sourceByKey.has(k)).sort();
const discrepancies: DriftDiscrepancy[] = [];
let buckets_compared = 0;
const totals: DriftReport['totals'] = {};
for (const m of metrics) totals[m] = { source: 0, aggregate: 0, delta: 0 };
for (const [key, sourceBucket] of sourceByKey) {
for (const m of metrics) totals[m].source += sourceBucket.values[m] ?? 0;
const aggBucket = aggByKey.get(key);
if (!aggBucket) continue;
buckets_compared += 1;
for (const m of metrics) {
const sv = sourceBucket.values[m] ?? 0;
const av = aggBucket.values[m] ?? 0;
const delta = av - sv;
const rel = relative(sv, delta);
if (Math.abs(delta) > tolerance.absolute || rel > tolerance.relative) {
discrepancies.push({
key,
metric: m,
source_value: sv,
aggregate_value: av,
absolute_delta: delta,
relative_delta: rel,
});
}
}
}
for (const bucket of aggByKey.values()) {
for (const m of metrics) totals[m].aggregate += bucket.values[m] ?? 0;
}
for (const m of metrics) totals[m].delta = totals[m].aggregate - totals[m].source;
discrepancies.sort((a, b) => Math.abs(b.absolute_delta) - Math.abs(a.absolute_delta));
// A comparison that compared nothing has not proved anything. This is the
// BI-P025 false green, restated for pre-aggregates.
const verdict: DriftVerdict =
buckets_compared === 0 && missing_in_aggregate.length === 0 && extra_in_aggregate.length === 0
? 'insufficient_data'
: discrepancies.length > 0 || missing_in_aggregate.length > 0 || extra_in_aggregate.length > 0
? 'drifted'
: 'match';
const report: DriftReport = {
verdict,
checked_at,
buckets_compared,
missing_in_aggregate,
extra_in_aggregate,
discrepancies: discrepancies.slice(0, MAX_REPORTED_DISCREPANCIES),
tolerance,
totals,
};
if (discrepancies.length > 0) report.worst = discrepancies[0];
return report;
}
/**
* Fold a drift report into the runtime status. A drifted aggregate is taken
* OUT of service here rather than merely annotated: leaving it `serving` with
* a bad verdict relies on every future reader remembering to check the
* verdict, and one that forgets serves the wrong number.
*
* `insufficient_data` also stops service. It means the check could not prove
* anything, and an unproven aggregate is not a current one.
*/
export function applyDriftReport(status: MaterializationStatus, report: DriftReport): MaterializationStatus {
return {
...status,
state: report.verdict === 'match' ? status.state : 'drifted',
last_drift_check_at: report.checked_at,
last_drift_verdict: report.verdict,
};
}
@@ -0,0 +1,309 @@
/**
* src/analytics/materialization/materialization.ts
* =================================================
* BI-P105 — MATERIALIZATION / PRE-AGGREGATION.
*
* A pre-aggregate is the one optimisation that fails SILENTLY. A slow query is
* a complaint; a pre-aggregate that has drifted from its source answers
* instantly, with confidence, and is wrong — no error, no timeout, nothing in
* a log. So this module is built around one rule:
*
* A MATERIALIZATION SERVES ONLY WHILE IT IS PROVABLY CURRENT AND PROVABLY
* UNDRIFTED. Every unknown routes to the source table.
*
* Six refusal reasons, each of which is a real way the aggregate becomes a
* confident liar:
*
* coverage_miss it does not carry the metric/dimension/grain
* asked for (a filter on a dimension the
* aggregate collapsed cannot be applied after
* the fact, and applying it anyway is how a
* pre-aggregate returns another slice's number)
* watermark_unknown nobody knows how current it is. BI-P026 already
* refuses to render UNKNOWN age as current; the
* same doctrine applies here.
* staleness_exceeded it is older than its own freshness contract
* window_beyond_watermark the question reaches past the newest data the
* aggregate contains — it would silently answer
* with a shorter period
* drift_never_verified it has never been compared to its source
* drift_verification_stale the last comparison is older than the interval
* the spec requires
* drift_detected the last comparison FAILED
*
* `drift_never_verified` is the one people leave out, and it is the one that
* matters most on day one: a brand-new materialization is exactly as unproven
* as a drifted one, and a router that defaults new aggregates to "serving"
* ships the bug on the day the feature launches.
*
* The DoD also asks for measured improvement and documented freshness
* semantics: scripts/bi-p105-materialization-drift.mjs measures both against a
* live ClickHouse and writes evidence/bi-p105/materialization-drift.json,
* including what happens WITHOUT this router (the drifted aggregate served).
*/
import { SiaxError } from '../../platform/errors';
import type { AnalyticsQueryRequest, TimeGrain } from '../contract/query-request';
export const MATERIALIZATION_STRATEGIES = [
'clickhouse_materialized_view',
'clickhouse_projection',
'dbt_incremental',
'cube_preaggregation',
] as const;
export type MaterializationStrategy = (typeof MATERIALIZATION_STRATEGIES)[number];
export const MATERIALIZATION_STATES = ['building', 'serving', 'drifted', 'disabled'] as const;
export type MaterializationState = (typeof MATERIALIZATION_STATES)[number];
/**
* Grains ordered coarsest-last. An aggregate at `day` can answer `day` and
* anything coarser by re-aggregating; it can NEVER answer `hour`, because the
* detail was thrown away when the rows were rolled up.
*/
const GRAIN_ORDER: readonly TimeGrain[] = ['hour', 'day', 'week', 'month', 'quarter', 'year'];
/**
* `week` does not divide `month` and `month` does not divide `quarter` by row
* count, but both roll up cleanly from `day`. Grains that CANNOT be derived by
* re-aggregating a coarser one are listed here so the router refuses rather
* than quietly returning a mis-bucketed series.
*/
function canRollUp(from: TimeGrain, to: TimeGrain): boolean {
if (from === to) return true;
const fi = GRAIN_ORDER.indexOf(from);
const ti = GRAIN_ORDER.indexOf(to);
if (fi < 0 || ti < 0 || ti < fi) return false;
// Weekly buckets do not compose into calendar months/quarters/years.
if (from === 'week' && to !== 'week') return false;
return true;
}
export interface DriftTolerance {
/** Absolute delta allowed per metric per bucket. */
absolute: number;
/** Relative delta allowed (0.001 = 0.1%). Both must hold. */
relative: number;
}
/** What a materialization IS. Declarative; the engine adapters read it. */
export interface MaterializationSpec {
materialization_id: string;
tenant_id: string;
project_id: string;
/** The dataset it pre-aggregates (BI-P003). */
dataset: string;
strategy: MaterializationStrategy;
/** Physical target (ClickHouse table, dbt model, Cube pre-agg name). */
target_ref: string;
metrics: string[];
dimensions: string[];
time_dimension?: string;
grain?: TimeGrain;
/** The freshness contract. Older than this and it stops serving. */
max_staleness_seconds: number;
/** How often the aggregate must be re-proved against its source. */
drift_check_interval_seconds: number;
tolerance: DriftTolerance;
}
export const DRIFT_VERDICTS = ['match', 'drifted', 'insufficient_data'] as const;
export type DriftVerdict = (typeof DRIFT_VERDICTS)[number];
/** Runtime state. Separate from the spec: the spec is config, this is fact. */
export interface MaterializationStatus {
materialization_id: string;
state: MaterializationState;
/**
* The newest source timestamp the aggregate represents. UNDEFINED means
* UNKNOWN, and unknown is never treated as current.
*/
data_watermark?: string;
last_refresh_at?: string;
last_drift_check_at?: string;
last_drift_verdict?: DriftVerdict;
}
export const ROUTING_REFUSALS = [
'coverage_miss',
'watermark_unknown',
'staleness_exceeded',
'window_beyond_watermark',
'drift_never_verified',
'drift_verification_stale',
'drift_detected',
'state_not_serving',
] as const;
export type RoutingRefusal = (typeof ROUTING_REFUSALS)[number];
export type RoutingDecision =
| {
source: 'materialization';
materialization_id: string;
target_ref: string;
/** Reported so a caller can show it; never inferred, always measured. */
staleness_seconds: number;
}
| { source: 'base_table'; refusal: RoutingRefusal; detail: string };
function refuse(refusal: RoutingRefusal, detail: string): RoutingDecision {
return { source: 'base_table', refusal, detail };
}
/**
* Staleness of an aggregate, in seconds, or `undefined` when it cannot be
* computed. Deliberately NOT defaulted to 0: "we could not tell" and "it is
* current" are different facts, and conflating them is the bug.
*/
export function materializationStaleness(status: MaterializationStatus, now: Date): number | undefined {
if (!status.data_watermark) return undefined;
const t = Date.parse(status.data_watermark);
if (Number.isNaN(t)) return undefined;
return Math.max(0, Math.floor((now.getTime() - t) / 1000));
}
/**
* Decide whether a request may be answered from the pre-aggregate.
*
* Fail-closed by construction: the function returns `materialization` only
* after every check has passed, and every unknown is a refusal with a named
* reason. Callers get the reason so it can be logged and counted (BI-P103
* reads the counts).
*/
export function selectQuerySource(args: {
request: AnalyticsQueryRequest;
spec: MaterializationSpec;
status: MaterializationStatus;
now: Date;
}): RoutingDecision {
const { request, spec, status, now } = args;
if (request.dataset !== spec.dataset) {
return refuse('coverage_miss', `materialization covers "${spec.dataset}", not "${request.dataset}"`);
}
if (status.state !== 'serving') {
return refuse('state_not_serving', `materialization state is "${status.state}"`);
}
// ── coverage ────────────────────────────────────────────────────────────
const metrics = new Set(spec.metrics);
for (const m of request.metrics) {
if (!metrics.has(m)) return refuse('coverage_miss', `metric "${m}" is not in the pre-aggregate`);
}
const dimensions = new Set(spec.dimensions);
for (const d of request.dimensions) {
if (!dimensions.has(d)) return refuse('coverage_miss', `dimension "${d}" is not in the pre-aggregate`);
}
for (const f of request.filters) {
if (!dimensions.has(f.dimension)) {
// The aggregate summed across this dimension. Filtering afterwards would
// return a number computed over rows the filter should have excluded.
return refuse(
'coverage_miss',
`filter on "${f.dimension}" cannot be applied: the pre-aggregate has already summed across it`,
);
}
}
const requestedGrain = request.timeRange?.grain;
if (requestedGrain) {
if (!spec.grain) return refuse('coverage_miss', 'the pre-aggregate has no time grain');
if (!canRollUp(spec.grain, requestedGrain)) {
return refuse(
'coverage_miss',
`a "${spec.grain}" pre-aggregate cannot answer at "${requestedGrain}" grain`,
);
}
}
// ── freshness ───────────────────────────────────────────────────────────
const staleness = materializationStaleness(status, now);
if (staleness === undefined) {
return refuse(
'watermark_unknown',
'the pre-aggregate has no data watermark — unknown age is not current age',
);
}
if (staleness > spec.max_staleness_seconds) {
return refuse(
'staleness_exceeded',
`${staleness}s stale, contract allows ${spec.max_staleness_seconds}s`,
);
}
const to = request.timeRange?.to;
if (to) {
const toMs = Date.parse(to);
const watermarkMs = Date.parse(status.data_watermark!);
if (!Number.isNaN(toMs) && toMs > watermarkMs) {
return refuse(
'window_beyond_watermark',
`the question runs to ${to} but the pre-aggregate only contains data to ${status.data_watermark}`,
);
}
}
// ── drift ───────────────────────────────────────────────────────────────
if (!status.last_drift_check_at || !status.last_drift_verdict) {
return refuse(
'drift_never_verified',
'the pre-aggregate has never been compared to its source; a new aggregate is as unproven as a drifted one',
);
}
if (status.last_drift_verdict !== 'match') {
return refuse('drift_detected', `last drift check returned "${status.last_drift_verdict}"`);
}
const checkedAgo = Math.floor((now.getTime() - Date.parse(status.last_drift_check_at)) / 1000);
if (Number.isNaN(checkedAgo)) {
return refuse('drift_never_verified', 'last_drift_check_at is not a timestamp');
}
if (checkedAgo > spec.drift_check_interval_seconds) {
return refuse(
'drift_verification_stale',
`last verified ${checkedAgo}s ago, spec requires every ${spec.drift_check_interval_seconds}s`,
);
}
return {
source: 'materialization',
materialization_id: spec.materialization_id,
target_ref: spec.target_ref,
staleness_seconds: staleness,
};
}
/**
* The routing decision as a freshness envelope for the result (BI-P026 shape):
* a consumer must be able to see WHICH source answered and how old it was.
* A result that does not say is a result nobody can audit.
*/
export function describeRouting(decision: RoutingDecision): {
served_from: 'materialization' | 'base_table';
materialization_id?: string;
staleness_seconds?: number;
refusal?: RoutingRefusal;
detail?: string;
} {
return decision.source === 'materialization'
? {
served_from: 'materialization',
materialization_id: decision.materialization_id,
staleness_seconds: decision.staleness_seconds,
}
: { served_from: 'base_table', refusal: decision.refusal, detail: decision.detail };
}
export function assertSpec(spec: MaterializationSpec): void {
const invalid = (d: string): never => {
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.materialization', d);
};
if (!spec.materialization_id?.trim()) invalid('materialization_id is required');
if (!spec.tenant_id?.trim() || !spec.project_id?.trim()) invalid('a materialization is tenant-scoped');
if (spec.metrics.length === 0) invalid('a materialization with no metrics pre-aggregates nothing');
if (spec.max_staleness_seconds <= 0) invalid('max_staleness_seconds must be positive');
if (spec.drift_check_interval_seconds <= 0) {
invalid(
'drift_check_interval_seconds must be positive — an aggregate that is never re-verified is one ' +
'source mutation away from serving wrong numbers forever',
);
}
if (spec.tolerance.absolute < 0 || spec.tolerance.relative < 0) invalid('tolerance must be non-negative');
if (spec.grain && !GRAIN_ORDER.includes(spec.grain)) invalid('grain is not a time grain');
}
@@ -0,0 +1,327 @@
/**
* tests/analytics/materialization-drift.test.ts
* ==============================================
* BI-P105 — MATERIALIZATION / PRE-AGGREGATION.
*
* The DoD asks for measured improvement and documented freshness semantics.
* The failure mode it does not name, and the one that actually costs money, is
* a pre-aggregate that returns a confident wrong number. So every test here is
* written as an attack on the router:
*
* - a drifted aggregate must NOT serve, and the counterfactual (a naive
* router that only checks staleness) must serve it — otherwise the guard
* is not known to be doing anything;
* - a MISSING bucket must count as drift even though every value present is
* correct;
* - a comparison over an empty window must be `insufficient_data`, never
* `match`;
* - an aggregate that has never been verified must not serve, because on
* launch day it is exactly as unproven as a drifted one.
*/
import { describe, expect, it } from 'vitest';
import {
applyDriftReport,
detectDrift,
type AggregateBucket,
} from '@/src/analytics/materialization/drift-detector';
import {
assertSpec,
describeRouting,
materializationStaleness,
selectQuerySource,
type MaterializationSpec,
type MaterializationStatus,
} from '@/src/analytics/materialization/materialization';
import type { AnalyticsQueryRequest } from '@/src/analytics/contract/query-request';
const NOW = new Date('2026-09-07T12:00:00.000Z');
const TOLERANCE = { absolute: 0.5, relative: 0.001 };
const SPEC: MaterializationSpec = {
materialization_id: 'mv_sales_daily',
tenant_id: 't_alpha',
project_id: 'p_main',
dataset: 'ds_sales',
strategy: 'clickhouse_materialized_view',
target_ref: 'opt_sales_daily',
metrics: ['revenue', 'orders'],
dimensions: ['order_date', 'region'],
time_dimension: 'order_date',
grain: 'day',
max_staleness_seconds: 3600,
drift_check_interval_seconds: 86_400,
tolerance: TOLERANCE,
};
/** Fresh, recently verified, matching. The only state that may serve. */
function healthyStatus(): MaterializationStatus {
return {
materialization_id: SPEC.materialization_id,
state: 'serving',
data_watermark: '2026-09-07T11:45:00.000Z',
last_refresh_at: '2026-09-07T11:46:00.000Z',
last_drift_check_at: '2026-09-07T06:00:00.000Z',
last_drift_verdict: 'match',
};
}
const REQUEST: AnalyticsQueryRequest = {
dataset: 'ds_sales',
metrics: ['revenue'],
dimensions: ['order_date'],
filters: [{ dimension: 'region', operator: 'eq', values: ['SE'] }],
timeRange: { from: '2026-09-01T00:00:00.000Z', to: '2026-09-07T00:00:00.000Z', grain: 'day' },
};
/**
* The router someone writes when they have thought about staleness but not
* about correctness. It exists here as the counterfactual: it must SERVE the
* drifted aggregate that `selectQuerySource` refuses.
*/
function naiveRouter(status: MaterializationStatus, now: Date): 'materialization' | 'base_table' {
const staleness = materializationStaleness(status, now) ?? 0;
return staleness <= SPEC.max_staleness_seconds ? 'materialization' : 'base_table';
}
function buckets(spec: Record<string, [number, number]>): AggregateBucket[] {
return Object.entries(spec).map(([key, [revenue, orders]]) => ({ key, values: { revenue, orders } }));
}
const SOURCE = buckets({
'2026-09-01|SE': [120_000, 410],
'2026-09-02|SE': [131_500, 442],
'2026-09-03|SE': [98_250, 305],
'2026-09-04|SE': [140_000, 480],
});
describe('BI-P105 drift detection', () => {
it('an identical aggregate matches', () => {
const report = detectDrift({
fromSource: SOURCE,
fromAggregate: SOURCE,
tolerance: TOLERANCE,
metrics: ['revenue', 'orders'],
now: NOW,
});
expect(report.verdict).toBe('match');
expect(report.buckets_compared).toBe(4);
expect(report.totals.revenue.delta).toBe(0);
});
it('a value beyond tolerance is drift, and the report says what a caller would have been told', () => {
const drifted = buckets({
'2026-09-01|SE': [120_000, 410],
'2026-09-02|SE': [131_500, 442],
'2026-09-03|SE': [98_250, 305],
// A source row was deleted (GDPR erasure) but the aggregate still counts it.
'2026-09-04|SE': [152_400, 495],
});
const report = detectDrift({
fromSource: SOURCE,
fromAggregate: drifted,
tolerance: TOLERANCE,
metrics: ['revenue', 'orders'],
now: NOW,
});
expect(report.verdict).toBe('drifted');
expect(report.worst?.key).toBe('2026-09-04|SE');
expect(report.worst?.absolute_delta).toBe(12_400);
expect(report.totals.revenue.aggregate - report.totals.revenue.source).toBe(12_400);
});
it('a MISSING bucket is drift even though every value present is correct', () => {
const lateArriving = SOURCE.filter((b) => b.key !== '2026-09-04|SE');
const report = detectDrift({
fromSource: SOURCE,
fromAggregate: lateArriving,
tolerance: TOLERANCE,
metrics: ['revenue', 'orders'],
now: NOW,
});
expect(report.discrepancies).toHaveLength(0); // every compared value agrees
expect(report.verdict).toBe('drifted'); // and it is still wrong
expect(report.missing_in_aggregate).toEqual(['2026-09-04|SE']);
expect(report.totals.revenue.delta).toBe(-140_000);
});
it('an EXTRA bucket is drift — the source rows are gone, the aggregate is not', () => {
const stale = [...SOURCE, { key: '2026-08-31|SE', values: { revenue: 90_000, orders: 300 } }];
const report = detectDrift({
fromSource: SOURCE,
fromAggregate: stale,
tolerance: TOLERANCE,
metrics: ['revenue', 'orders'],
now: NOW,
});
expect(report.verdict).toBe('drifted');
expect(report.extra_in_aggregate).toEqual(['2026-08-31|SE']);
});
it('an EMPTY comparison is insufficient_data, not match', () => {
const report = detectDrift({
fromSource: [],
fromAggregate: [],
tolerance: TOLERANCE,
metrics: ['revenue'],
now: NOW,
});
expect(report.verdict).toBe('insufficient_data');
expect(report.buckets_compared).toBe(0);
// And it must not be allowed to serve on the strength of that.
const status = applyDriftReport(healthyStatus(), report);
expect(status.state).toBe('drifted');
});
it('both tolerances must hold — a large bucket cannot hide behind a relative bound', () => {
const source = buckets({ 'b|SE': [40_000_000, 1] });
const off = buckets({ 'b|SE': [40_020_000, 1] }); // 0.05%: inside relative, far outside absolute
const report = detectDrift({
fromSource: source,
fromAggregate: off,
tolerance: { absolute: 1000, relative: 0.001 },
metrics: ['revenue'],
now: NOW,
});
expect(report.verdict).toBe('drifted');
expect(report.worst?.relative_delta).toBeCloseTo(0.0005, 6);
});
it('a bucket that should be zero and is not is drift, not a division error', () => {
const report = detectDrift({
fromSource: buckets({ 'b|SE': [0, 0] }),
fromAggregate: buckets({ 'b|SE': [12, 1] }),
tolerance: TOLERANCE,
metrics: ['revenue'],
now: NOW,
});
expect(report.verdict).toBe('drifted');
expect(report.worst?.relative_delta).toBe(Number.POSITIVE_INFINITY);
});
});
describe('BI-P105 routing — a pre-aggregate serves only while it is provably current and undrifted', () => {
it('serves when everything is proved', () => {
const decision = selectQuerySource({ request: REQUEST, spec: SPEC, status: healthyStatus(), now: NOW });
expect(decision.source).toBe('materialization');
expect(describeRouting(decision)).toMatchObject({
served_from: 'materialization',
materialization_id: 'mv_sales_daily',
staleness_seconds: 900,
});
});
it('a DRIFTED aggregate is refused — and the naive router serves it', () => {
const drifted = applyDriftReport(healthyStatus(), {
verdict: 'drifted',
checked_at: '2026-09-07T11:00:00.000Z',
buckets_compared: 4,
missing_in_aggregate: [],
extra_in_aggregate: [],
discrepancies: [],
tolerance: TOLERANCE,
totals: {},
});
const guarded = selectQuerySource({ request: REQUEST, spec: SPEC, status: drifted, now: NOW });
expect(guarded.source).toBe('base_table');
expect(guarded).toMatchObject({ refusal: 'state_not_serving' });
// The counterfactual: the same state, routed by staleness alone.
expect(naiveRouter(drifted, NOW)).toBe('materialization');
});
it('a NEVER-VERIFIED aggregate is refused — and the naive router serves it', () => {
const brandNew: MaterializationStatus = {
materialization_id: SPEC.materialization_id,
state: 'serving',
data_watermark: '2026-09-07T11:59:00.000Z',
last_refresh_at: '2026-09-07T11:59:00.000Z',
};
const guarded = selectQuerySource({ request: REQUEST, spec: SPEC, status: brandNew, now: NOW });
expect(guarded).toMatchObject({ source: 'base_table', refusal: 'drift_never_verified' });
expect(naiveRouter(brandNew, NOW)).toBe('materialization');
});
it('a verification older than the required interval is refused', () => {
const status = { ...healthyStatus(), last_drift_check_at: '2026-09-05T06:00:00.000Z' };
expect(selectQuerySource({ request: REQUEST, spec: SPEC, status, now: NOW })).toMatchObject({
refusal: 'drift_verification_stale',
});
});
it('an UNKNOWN watermark is refused rather than treated as current', () => {
const status = { ...healthyStatus(), data_watermark: undefined };
expect(materializationStaleness(status, NOW)).toBeUndefined();
expect(selectQuerySource({ request: REQUEST, spec: SPEC, status, now: NOW })).toMatchObject({
refusal: 'watermark_unknown',
});
});
it('an aggregate older than its freshness contract is refused', () => {
const status = { ...healthyStatus(), data_watermark: '2026-09-07T09:00:00.000Z' };
expect(selectQuerySource({ request: REQUEST, spec: SPEC, status, now: NOW })).toMatchObject({
refusal: 'staleness_exceeded',
});
});
it('a question that reaches past the watermark is refused, not silently shortened', () => {
const request = { ...REQUEST, timeRange: { ...REQUEST.timeRange, to: '2026-09-07T12:00:00.000Z' } };
expect(selectQuerySource({ request, spec: SPEC, status: healthyStatus(), now: NOW })).toMatchObject({
refusal: 'window_beyond_watermark',
});
});
it('a filter on a dimension the aggregate summed across is refused', () => {
const request = {
...REQUEST,
filters: [{ dimension: 'sales_channel', operator: 'eq' as const, values: ['web'] }],
};
const decision = selectQuerySource({ request, spec: SPEC, status: healthyStatus(), now: NOW });
expect(decision).toMatchObject({ source: 'base_table', refusal: 'coverage_miss' });
expect((decision as { detail: string }).detail).toMatch(/already summed across it/);
});
it('a metric or dimension the aggregate does not carry is refused', () => {
expect(
selectQuerySource({
request: { ...REQUEST, metrics: ['margin'] },
spec: SPEC,
status: healthyStatus(),
now: NOW,
}),
).toMatchObject({ refusal: 'coverage_miss' });
expect(
selectQuerySource({
request: { ...REQUEST, dimensions: ['product'] },
spec: SPEC,
status: healthyStatus(),
now: NOW,
}),
).toMatchObject({ refusal: 'coverage_miss' });
});
it('a daily aggregate cannot answer an hourly question, but can answer a monthly one', () => {
const hourly = { ...REQUEST, timeRange: { ...REQUEST.timeRange, grain: 'hour' as const } };
expect(
selectQuerySource({ request: hourly, spec: SPEC, status: healthyStatus(), now: NOW }),
).toMatchObject({ refusal: 'coverage_miss' });
const monthly = { ...REQUEST, timeRange: { ...REQUEST.timeRange, grain: 'month' as const } };
expect(
selectQuerySource({ request: monthly, spec: SPEC, status: healthyStatus(), now: NOW }).source,
).toBe('materialization');
});
it('a WEEKLY aggregate cannot answer a monthly question — weeks do not compose into months', () => {
const weekly = { ...SPEC, grain: 'week' as const };
const monthly = { ...REQUEST, timeRange: { ...REQUEST.timeRange, grain: 'month' as const } };
expect(
selectQuerySource({ request: monthly, spec: weekly, status: healthyStatus(), now: NOW }),
).toMatchObject({ refusal: 'coverage_miss' });
});
it('a spec with no drift check interval is refused at declaration time', () => {
expect(() => assertSpec({ ...SPEC, drift_check_interval_seconds: 0 })).toThrow(/never re-verified/);
expect(() => assertSpec(SPEC)).not.toThrow();
});
});
@@ -0,0 +1,348 @@
/**
* tests/integration/materialization-drift-clickhouse.integration.test.ts
* =======================================================================
* BI-P105 against a LIVE ClickHouse. Skips honestly when CLICKHOUSE_URL is
* unset; with it set and the server down, it FAILS rather than skipping.
*
* BI-P105's DoD is "performance improvement measured and freshness semantics
* documented", and both halves are done here against real data rather than
* asserted. What the DoD does not say, and what this file is really about, is
* that a pre-aggregate fails SILENTLY: it answers instantly and wrongly.
*
* The drill drives a real ClickHouse materialized view through the exact
* sequence that produces a confident wrong number in production:
*
* 1. 2M source rows, a SummingMergeTree target and a materialized view.
* 2. Backfill, verify with the SHIPPED detector: verdict `match`.
* 3. Measure both paths repeatedly, and report the DISTRIBUTION — a single
* speed-up figure quoted as a system property is misleading (BI-P104).
* 4. `ALTER TABLE ... DELETE` on the source: a GDPR erasure, the single most
* common real cause of pre-aggregate drift. A ClickHouse materialized
* view is an INSERT trigger; it does not see mutations, so the target
* keeps the deleted revenue forever and nothing errors.
* 5. The detector must catch it, and `selectQuerySource` must refuse to
* serve. The counterfactual — a router that checks only staleness — is
* run on the same state and MUST serve, with the wrong number recorded
* in kronor.
*
* Everything measured lands in evidence/bi-p105/materialization-drift.json.
*
* docker run -d --name ch -p 28123:8123 -e CLICKHOUSE_PASSWORD=test \
* clickhouse/clickhouse-server:26.3
* CLICKHOUSE_URL=http://127.0.0.1:28123 CLICKHOUSE_AUTH=default:test \
* pnpm vitest run tests/integration/materialization-drift-clickhouse.integration.test.ts
*
* It CREATEs and DROPs its own database. Never point it at the b0ts
* ClickHouse or any quarantined market data.
*/
import fs from 'node:fs';
import path from 'node:path';
import { afterAll, beforeAll, describe, expect, it } from 'vitest';
import {
applyDriftReport,
detectDrift,
type AggregateBucket,
type DriftReport,
} from '@/src/analytics/materialization/drift-detector';
import {
materializationStaleness,
selectQuerySource,
type MaterializationSpec,
type MaterializationStatus,
} from '@/src/analytics/materialization/materialization';
import type { AnalyticsQueryRequest } from '@/src/analytics/contract/query-request';
const CLICKHOUSE_URL = process.env.CLICKHOUSE_URL;
const CLICKHOUSE_AUTH = process.env.CLICKHOUSE_AUTH;
const run = CLICKHOUSE_URL ? describe : describe.skip;
const DB = `opt_p105_${Date.now().toString(36)}`;
const ROWS = 2_000_000;
const REPETITIONS = 9;
const headers: Record<string, string> = CLICKHOUSE_AUTH
? { Authorization: `Basic ${Buffer.from(CLICKHOUSE_AUTH).toString('base64')}` }
: {};
async function ch(query: string): Promise<string> {
const res = await fetch(CLICKHOUSE_URL ?? 'http://skipped.invalid', {
method: 'POST',
headers,
body: query,
});
const text = await res.text();
if (!res.ok) throw new Error(`ClickHouse ${res.status}: ${text.slice(0, 400)}`);
return text;
}
/** Rows as buckets keyed "date|region", which is what the detector compares. */
async function bucketsFrom(query: string): Promise<AggregateBucket[]> {
const text = await ch(`${query} FORMAT JSONEachRow`);
return text
.trim()
.split('\n')
.filter(Boolean)
.map((line) => {
const r = JSON.parse(line) as { d: string; region: string; revenue: number; orders: number };
return { key: `${r.d}|${r.region}`, values: { revenue: Number(r.revenue), orders: Number(r.orders) } };
});
}
/** Median and the spread around it. A single number is not a system property. */
function distribution(samples: number[]): {
n: number;
min: number;
p50: number;
p95: number;
max: number;
spread_ratio: number;
} {
const s = [...samples].sort((a, b) => a - b);
const at = (q: number) => s[Math.min(s.length - 1, Math.floor(q * s.length))];
return {
n: s.length,
min: s[0],
p50: at(0.5),
p95: at(0.95),
max: s[s.length - 1],
spread_ratio: Number((s[s.length - 1] / Math.max(s[0], 1e-9)).toFixed(2)),
};
}
async function timeQuery(query: string): Promise<number> {
const t0 = performance.now();
await ch(`${query} FORMAT JSONEachRow`);
return performance.now() - t0;
}
const SOURCE_AGG = `
SELECT toString(order_date) AS d, region, sum(amount) AS revenue, count() AS orders
FROM ${DB}.sales WHERE tenant_id = 't_alpha' GROUP BY d, region ORDER BY d, region`;
const TARGET_AGG = `
SELECT toString(order_date) AS d, region, sum(revenue) AS revenue, sum(orders) AS orders
FROM ${DB}.sales_daily WHERE tenant_id = 't_alpha' GROUP BY d, region ORDER BY d, region`;
const SPEC: MaterializationSpec = {
materialization_id: 'mv_sales_daily',
tenant_id: 't_alpha',
project_id: 'p_main',
dataset: 'ds_sales',
strategy: 'clickhouse_materialized_view',
target_ref: `${DB}.sales_daily`,
metrics: ['revenue', 'orders'],
dimensions: ['order_date', 'region'],
time_dimension: 'order_date',
grain: 'day',
max_staleness_seconds: 3600,
drift_check_interval_seconds: 86_400,
tolerance: { absolute: 0.5, relative: 0.0001 },
};
const REQUEST: AnalyticsQueryRequest = {
dataset: 'ds_sales',
metrics: ['revenue'],
dimensions: ['order_date'],
filters: [],
timeRange: { grain: 'day' },
};
const evidence: Record<string, unknown> = {};
let status: MaterializationStatus;
run('BI-P105 materialization against a LIVE ClickHouse', () => {
beforeAll(async () => {
evidence.clickhouse_version = (await ch('SELECT version()')).trim();
evidence.recorded_at = new Date().toISOString();
evidence.source_rows = ROWS;
await ch(`CREATE DATABASE IF NOT EXISTS ${DB}`);
await ch(`
CREATE TABLE ${DB}.sales (
tenant_id LowCardinality(String),
order_date Date,
region LowCardinality(String),
customer_id UInt32,
amount Decimal(18, 2)
) ENGINE = MergeTree ORDER BY (tenant_id, order_date, region)`);
await ch(`
INSERT INTO ${DB}.sales
SELECT
if(number % 5 = 0, 't_beta', 't_alpha') AS tenant_id,
toDate('2026-08-01') + toIntervalDay(number % 30) AS order_date,
['SE','NO','DK','FI'][(number % 4) + 1] AS region,
toUInt32(number % 5000) AS customer_id,
toDecimal64((number % 977) + 13, 2) AS amount
FROM numbers(${ROWS})`);
await ch(`
CREATE TABLE ${DB}.sales_daily (
tenant_id LowCardinality(String),
order_date Date,
region LowCardinality(String),
revenue Decimal(38, 2),
orders UInt64
) ENGINE = SummingMergeTree ORDER BY (tenant_id, order_date, region)`);
await ch(`
CREATE MATERIALIZED VIEW ${DB}.sales_daily_mv TO ${DB}.sales_daily AS
SELECT tenant_id, order_date, region, sum(amount) AS revenue, count() AS orders
FROM ${DB}.sales GROUP BY tenant_id, order_date, region`);
// The MV is an INSERT trigger: it saw nothing that was already there, so
// the backfill is explicit. (This is drift cause #1, and it is why the
// first check below has to be able to fail.)
await ch(`
INSERT INTO ${DB}.sales_daily
SELECT tenant_id, order_date, region, sum(amount) AS revenue, count() AS orders
FROM ${DB}.sales GROUP BY tenant_id, order_date, region`);
await ch(`OPTIMIZE TABLE ${DB}.sales_daily FINAL`);
status = {
materialization_id: SPEC.materialization_id,
state: 'serving',
data_watermark: new Date().toISOString(),
last_refresh_at: new Date().toISOString(),
};
}, 300_000);
afterAll(async () => {
if (!CLICKHOUSE_URL) return;
try {
await ch(`DROP DATABASE IF EXISTS ${DB}`);
} finally {
const dir = path.join(process.cwd(), 'evidence/bi-p105');
fs.mkdirSync(dir, { recursive: true });
fs.writeFileSync(
path.join(dir, 'materialization-drift.json'),
`${JSON.stringify(evidence, null, 2)}\n`,
);
}
}, 60_000);
it('the freshly backfilled aggregate MATCHES its source', async () => {
const report = detectDrift({
fromSource: await bucketsFrom(SOURCE_AGG),
fromAggregate: await bucketsFrom(TARGET_AGG),
tolerance: SPEC.tolerance,
metrics: ['revenue', 'orders'],
});
expect(report.buckets_compared).toBeGreaterThan(0); // an empty compare proves nothing
expect(report.verdict).toBe('match');
status = applyDriftReport(status, report);
evidence.after_backfill = {
verdict: report.verdict,
buckets_compared: report.buckets_compared,
revenue_total: report.totals.revenue,
};
}, 120_000);
it('measures the improvement as a DISTRIBUTION, not a single number', async () => {
const base: number[] = [];
const agg: number[] = [];
for (let i = 0; i < REPETITIONS; i += 1) {
// Interleaved so a machine that gets busy halfway through penalises both.
base.push(await timeQuery(SOURCE_AGG));
agg.push(await timeQuery(TARGET_AGG));
}
const baseD = distribution(base);
const aggD = distribution(agg);
evidence.performance = {
note:
'ClickHouse caches marks and pages between repetitions, so these are WARM numbers on one host ' +
'with nothing else running. They describe this shape of query on this data, not a system property.',
repetitions: REPETITIONS,
base_table_ms: baseD,
pre_aggregate_ms: aggD,
speedup_p50: Number((baseD.p50 / Math.max(aggD.p50, 1e-9)).toFixed(1)),
speedup_range_min_max: [
Number((baseD.min / Math.max(aggD.max, 1e-9)).toFixed(1)),
Number((baseD.max / Math.max(aggD.min, 1e-9)).toFixed(1)),
],
};
// The claim under test is only that the pre-aggregate is not SLOWER; the
// magnitude is reported, never asserted as a threshold.
expect(aggD.p50).toBeLessThanOrEqual(baseD.p50);
}, 300_000);
it('a GDPR erasure on the source DRIFTS the aggregate, and the drift is caught', async () => {
const before = await bucketsFrom(TARGET_AGG);
// The mutation a right-to-erasure request produces. A materialized view is
// an insert trigger — it never sees this.
await ch(
`ALTER TABLE ${DB}.sales DELETE WHERE tenant_id = 't_alpha' AND customer_id < 250 SETTINGS mutations_sync = 2`,
);
const fromSource = await bucketsFrom(SOURCE_AGG);
const fromAggregate = await bucketsFrom(TARGET_AGG);
expect(fromAggregate).toEqual(before); // the aggregate genuinely did not move
const report: DriftReport = detectDrift({
fromSource,
fromAggregate,
tolerance: SPEC.tolerance,
metrics: ['revenue', 'orders'],
});
expect(report.verdict).toBe('drifted');
expect(report.worst).toBeDefined();
const drifted = applyDriftReport(status, report);
const guarded = selectQuerySource({ request: REQUEST, spec: SPEC, status: drifted, now: new Date() });
expect(guarded.source).toBe('base_table');
// COUNTERFACTUAL: a router that checks only staleness serves it, and this
// is the number a dashboard would have shown.
const stalenessOnlyServes =
(materializationStaleness(drifted, new Date()) ?? 0) <= SPEC.max_staleness_seconds;
expect(stalenessOnlyServes).toBe(true);
evidence.gdpr_erasure_drill = {
mutation: "ALTER TABLE sales DELETE WHERE tenant_id='t_alpha' AND customer_id < 250",
verdict: report.verdict,
buckets_compared: report.buckets_compared,
revenue_true_after_erasure: report.totals.revenue.source,
revenue_pre_aggregate_would_report: report.totals.revenue.aggregate,
revenue_overstatement: report.totals.revenue.delta,
revenue_overstatement_pct: Number(
((report.totals.revenue.delta / report.totals.revenue.source) * 100).toFixed(3),
),
worst_bucket: report.worst,
guarded_router_decision: guarded,
staleness_only_router_would_serve: stalenessOnlyServes,
staleness_seconds_at_decision: materializationStaleness(drifted, new Date()),
note:
'The aggregate was NOT stale by any clock: it had just been refreshed, and every freshness check ' +
'passed. Only the source comparison caught it. This is why staleness alone is not a correctness ' +
'guarantee.',
};
status = drifted;
}, 300_000);
it('and a refresh that re-derives the aggregate clears the drift', async () => {
await ch(`TRUNCATE TABLE ${DB}.sales_daily`);
await ch(`
INSERT INTO ${DB}.sales_daily
SELECT tenant_id, order_date, region, sum(amount) AS revenue, count() AS orders
FROM ${DB}.sales GROUP BY tenant_id, order_date, region`);
await ch(`OPTIMIZE TABLE ${DB}.sales_daily FINAL`);
const report = detectDrift({
fromSource: await bucketsFrom(SOURCE_AGG),
fromAggregate: await bucketsFrom(TARGET_AGG),
tolerance: SPEC.tolerance,
metrics: ['revenue', 'orders'],
});
expect(report.verdict).toBe('match');
const healed: MaterializationStatus = applyDriftReport(
{ ...status, state: 'serving', data_watermark: new Date().toISOString() },
report,
);
const decision = selectQuerySource({ request: REQUEST, spec: SPEC, status: healed, now: new Date() });
expect(decision.source).toBe('materialization');
evidence.after_rebuild = {
verdict: report.verdict,
revenue_total: report.totals.revenue,
routing: decision,
};
}, 300_000);
});