diff --git a/evidence/bi-p025/native-quality-live.json b/evidence/bi-p025/native-quality-live.json new file mode 100644 index 0000000..ce7a293 --- /dev/null +++ b/evidence/bi-p025/native-quality-live.json @@ -0,0 +1,33 @@ +{ + "clickhouse_version": "24.8.14.39", + "recorded_at": "2026-09-07T00:40:02.745Z", + "not_null": { + "evaluated": 118, + "violations": 5 + }, + "range": { + "violations": 3 + }, + "distribution": { + "violations": 4 + }, + "unique": { + "violations": 11, + "breakdown": "6 rows on dup_0/dup_1, plus 5 rows sharing the empty-string key" + }, + "empty_dataset": { + "raw_counts_from_clickhouse": { + "violations": 0, + "evaluated": 0 + }, + "verdict_passed": false, + "note": "a real engine returns zero violations over zero rows; the provider refuses to call that a pass" + }, + "tenant_isolation": { + "base_table_rows": 618, + "alpha_evaluated": 118, + "alpha_violations": 5, + "beta_evaluated": 500, + "beta_violations": 500 + } +} diff --git a/src/analytics/quality/clickhouse-quality-runner.ts b/src/analytics/quality/clickhouse-quality-runner.ts new file mode 100644 index 0000000..697ea64 --- /dev/null +++ b/src/analytics/quality/clickhouse-quality-runner.ts @@ -0,0 +1,127 @@ +/** + * src/analytics/quality/clickhouse-quality-runner.ts + * =================================================== + * BI-P025 — the ClickHouse `QualityQueryRunner`. + * + * One statement per rule, and it counts BOTH numbers in a single pass: + * + * SELECT countIf() AS violations, count() AS evaluated + * FROM . WHERE tenant_id = {tenant:String} AND dataset_id = … + * + * `evaluated` is not decoration. It is what makes an EMPTY dataset + * distinguishable from a CLEAN one — the false green + * `NativeQualityProvider` refuses. Counting only violations would make the two + * cases identical at this layer, and no amount of care upstream could recover + * the difference. + * + * The BI-P011 forced tenant predicate applies here exactly as it does to a + * metric query: `tenant_id = {tenant:String}` is condition[0], the value is + * bound, and a quality run therefore cannot read another tenant's rows even + * though it runs on an administrative credential. + * + * `unique` gets its own statement shape, because a duplicate is a property of + * a GROUP, not of a row, and pretending otherwise would make the rule count + * the wrong thing. + */ +import { SiaxError } from '../../platform/errors'; +import { assertSafeIdentifier, FORCED_TENANT_PREDICATE } from '../engine/query-compiler'; +import type { QualityQueryRunner } from './quality-provider'; + +export interface ClickHouseQualityRunnerOptions { + baseUrl: string; + basicAuth?: string; + database: string; + /** dataset_id -> physical table. An unlisted dataset is refused, not guessed. */ + tables: Record; + timeoutMs?: number; + /** Ceiling for a quality scan; a check must not become the expensive query. */ + maxRowsToRead?: number; +} + +/** Marker the provider emits for the `unique` rule. */ +const UNIQUE_MARKER = /^__unique__\(([A-Za-z_][A-Za-z0-9_]{0,62})\)$/; + +export class ClickHouseQualityRunner implements QualityQueryRunner { + readonly engine = 'clickhouse'; + + constructor(private readonly opts: ClickHouseQualityRunnerOptions) { + assertSafeIdentifier(opts.database, 'database'); + for (const table of Object.values(opts.tables)) assertSafeIdentifier(table, 'table'); + } + + async count(args: { + tenant_id: string; + dataset_id: string; + violationExpression: string; + params: Record; + }): Promise<{ violations: number; evaluated: number }> { + if (!args.tenant_id?.trim()) { + throw new SiaxError( + 'UNAUTHENTICATED', + '0pt.analytics.quality', + 'a quality run requires a verified tenant', + ); + } + const table = this.opts.tables[args.dataset_id]; + if (!table) { + throw new SiaxError( + 'NOT_FOUND', + '0pt.analytics.quality', + `dataset "${args.dataset_id}" has no physical table registered for quality runs`, + ); + } + const params: Record = { + ...args.params, + param_tenant: args.tenant_id, + param_dataset: args.dataset_id, + }; + const where = `${FORCED_TENANT_PREDICATE} AND dataset_id = {dataset:String}`; + + const unique = UNIQUE_MARKER.exec(args.violationExpression.trim()); + const sql = unique + ? // A duplicate is a property of a GROUP. Count the rows that belong to a + // key appearing more than once, and the total rows, in one pass. + `SELECT sum(multiIf(c > 1, c, 0)) AS violations, sum(c) AS evaluated FROM ` + + `(SELECT ${assertSafeIdentifier(unique[1], 'field')} AS k, count() AS c ` + + `FROM ${this.opts.database}.${table} WHERE ${where} GROUP BY k) FORMAT JSON` + : `SELECT countIf(${args.violationExpression}) AS violations, count() AS evaluated ` + + `FROM ${this.opts.database}.${table} WHERE ${where} FORMAT JSON`; + + const url = new URL(this.opts.baseUrl.replace(/\/+$/, '')); + url.searchParams.set('query', sql); + for (const [k, v] of Object.entries(params)) url.searchParams.set(k, v); + url.searchParams.set('max_rows_to_read', String(this.opts.maxRowsToRead ?? 5_000_000_000)); + url.searchParams.set('read_overflow_mode', 'throw'); + url.searchParams.set('max_execution_time', '60'); + url.searchParams.set('timeout_overflow_mode', 'throw'); + + const headers: Record = {}; + if (this.opts.basicAuth) { + headers.Authorization = `Basic ${Buffer.from(this.opts.basicAuth).toString('base64')}`; + } + + let res: Response; + try { + res = await fetch(url.toString(), { + method: 'POST', + headers, + body: '', + signal: AbortSignal.timeout(this.opts.timeoutMs ?? 60_000), + }); + } catch (cause) { + throw new SiaxError('DEPENDENCY_UNAVAILABLE', 'clickhouse', 'clickhouse unreachable', { cause }); + } + const text = await res.text(); + if (!res.ok) { + const code = /Code:\s*(\d+)/.exec(text)?.[1] ?? String(res.status); + throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', `quality query failed (CH code ${code})`); + } + const row = ( + JSON.parse(text) as { data: Array<{ violations: string | number; evaluated: string | number }> } + ).data[0]; + if (!row) { + throw new SiaxError('PERMANENT_FAILURE', 'clickhouse', 'quality query returned no counts'); + } + return { violations: Number(row.violations ?? 0), evaluated: Number(row.evaluated ?? 0) }; + } +} diff --git a/src/analytics/quality/freshness.ts b/src/analytics/quality/freshness.ts new file mode 100644 index 0000000..aaa587b --- /dev/null +++ b/src/analytics/quality/freshness.ts @@ -0,0 +1,196 @@ +/** + * src/analytics/quality/freshness.ts + * =================================== + * BI-P026 — DATA FRESHNESS. + * + * Five states, exactly as BI-P026 names them, and the DoD is a NEGATIVE: + * + * "Stale data is never SILENTLY presented as current." + * + * Two words carry it. "Silently" means the failure mode is not "we served old + * data" — it is "we served old data and the reader could not tell". And + * "never" means the enforcement has to sit at the boundary every result + * crosses, not in each dashboard's rendering code. + * + * So `assertFreshnessDisclosed` refuses to release a result that carries no + * freshness envelope at all. A result with no envelope is not FRESH and not + * UNKNOWN — it is unreleasable, because "we forgot to attach it" and "the data + * is current" are indistinguishable to a reader, and a default of FRESH would + * make forgetting the silent path. + * + * UNKNOWN is likewise never treated as FRESH. It is its own state precisely + * because "we could not determine the age" is the honest thing to display, and + * collapsing it into FRESH is the classic version of this bug. + */ +import { SiaxError } from '../../platform/errors'; + +/** BI-P026's states, verbatim and in severity order. */ +export const FRESHNESS_STATES = ['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN'] as const; +export type FreshnessState = (typeof FRESHNESS_STATES)[number]; + +/** States a reader must be shown explicitly rather than served as current. */ +export const NON_CURRENT_STATES: readonly FreshnessState[] = ['DELAYED', 'STALE', 'FAILED', 'UNKNOWN']; + +export interface FreshnessPolicy { + /** How often the upstream is expected to land data. */ + expectedIntervalMs: number; + /** + * Multiples of the expected interval. Past `delayedAfter` the data is + * DELAYED; past `staleAfter` it is STALE. Defaults are deliberately tight: + * a dashboard that is two cycles behind is already misleading someone. + */ + delayedAfterIntervals?: number; + staleAfterIntervals?: number; +} + +export interface FreshnessInput { + /** When the newest successfully loaded data is timestamped. */ + lastSuccessfulLoadAt?: string; + /** Outcome of the most recent load attempt. */ + lastRunStatus?: 'succeeded' | 'failed' | 'running' | 'never_run'; + lastRunAt?: string; + policy: FreshnessPolicy; + now: number; +} + +export interface FreshnessEnvelope { + state: FreshnessState; + /** Age of the newest data, in ms. Absent when unknown. */ + age_ms?: number; + last_successful_load_at?: string; + last_run_at?: string; + expected_interval_ms: number; + /** Plain-language line a UI can render verbatim. */ + disclosure: string; + computed_at: string; +} + +export function computeFreshness(input: FreshnessInput): FreshnessEnvelope { + const { policy, now } = input; + const delayedAfter = policy.expectedIntervalMs * (policy.delayedAfterIntervals ?? 1.5); + const staleAfter = policy.expectedIntervalMs * (policy.staleAfterIntervals ?? 3); + const computed_at = new Date(now).toISOString(); + const base = { + last_successful_load_at: input.lastSuccessfulLoadAt, + last_run_at: input.lastRunAt, + expected_interval_ms: policy.expectedIntervalMs, + computed_at, + }; + + // FAILED outranks age: a pipeline that just failed may still have + // recently-timestamped data from the run before, and reporting that as FRESH + // is precisely how a broken pipeline stays invisible for a day. + if (input.lastRunStatus === 'failed') { + const age = ageOf(input.lastSuccessfulLoadAt, now); + return { + ...base, + state: 'FAILED', + age_ms: age, + disclosure: `The most recent load FAILED. Showing data from ${input.lastSuccessfulLoadAt ?? 'an unknown time'}.`, + }; + } + + if (!input.lastSuccessfulLoadAt || input.lastRunStatus === 'never_run') { + return { + ...base, + state: 'UNKNOWN', + disclosure: 'The age of this data could not be determined. Do not read it as current.', + }; + } + + const age = ageOf(input.lastSuccessfulLoadAt, now); + if (age === undefined) { + return { ...base, state: 'UNKNOWN', disclosure: 'The age of this data could not be determined.' }; + } + if (age > staleAfter) { + return { + ...base, + state: 'STALE', + age_ms: age, + disclosure: `This data is ${describeAge(age)} old, past the ${describeAge(staleAfter)} staleness threshold.`, + }; + } + if (age > delayedAfter) { + return { + ...base, + state: 'DELAYED', + age_ms: age, + disclosure: `This data is ${describeAge(age)} old and behind its ${describeAge(policy.expectedIntervalMs)} schedule.`, + }; + } + return { + ...base, + state: 'FRESH', + age_ms: age, + disclosure: `Current as of ${input.lastSuccessfulLoadAt}.`, + }; +} + +function ageOf(iso: string | undefined, now: number): number | undefined { + if (!iso) return undefined; + const t = Date.parse(iso); + if (Number.isNaN(t)) return undefined; + return Math.max(0, now - t); +} + +function describeAge(ms: number): string { + const s = Math.round(ms / 1000); + if (s < 90) return `${s}s`; + const m = Math.round(s / 60); + if (m < 90) return `${m}m`; + const h = Math.round(m / 60); + if (h < 48) return `${h}h`; + return `${Math.round(h / 24)}d`; +} + +/** Any result leaving the analytics boundary. */ +export interface DisclosableResult { + freshness?: FreshnessEnvelope; + /** + * Set by a caller that has SEEN a non-current state and chosen to proceed. + * Its presence is what makes serving non-current data not-silent. + */ + acknowledged_non_current?: boolean; +} + +/** + * BI-P026's DoD, as a gate. + * + * Three refusals, in order of how easy they are to get wrong: + * 1. no envelope at all — unreleasable. Not defaulted to FRESH. + * 2. a non-current state with no acknowledgement — refused, so a consumer + * that ignores the envelope cannot render stale numbers as current. + * 3. an acknowledgement with no envelope — refused, because acknowledging + * something you were never shown is not disclosure. + */ +export function assertFreshnessDisclosed(result: DisclosableResult, context: string): FreshnessEnvelope { + if (!result.freshness) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.freshness', + `${context} carries no freshness envelope — BI-P026: data with unknown age must never be released ` + + `as though it were current`, + ); + } + const state = result.freshness.state; + if (!(FRESHNESS_STATES as readonly string[]).includes(state)) { + throw new SiaxError( + 'PERMANENT_FAILURE', + '0pt.analytics.freshness', + `${context} has an unknown freshness state`, + ); + } + if (NON_CURRENT_STATES.includes(state) && !result.acknowledged_non_current) { + throw new SiaxError( + 'CONFLICT', + '0pt.analytics.freshness', + `${context} is ${state} and the consumer has not acknowledged it: ${result.freshness.disclosure}`, + ); + } + return result.freshness; +} + +/** True when the state may be presented without a caveat. Only one state may. */ +export function isPresentableAsCurrent(state: FreshnessState): boolean { + return state === 'FRESH'; +} diff --git a/src/analytics/quality/quality-provider.ts b/src/analytics/quality/quality-provider.ts new file mode 100644 index 0000000..1080e62 --- /dev/null +++ b/src/analytics/quality/quality-provider.ts @@ -0,0 +1,418 @@ +/** + * src/analytics/quality/quality-provider.ts + * ========================================== + * BI-P025 — DATA QUALITY PROVIDER. + * + * One interface, three provider kinds, and an honest split between them: + * + * NATIVE — this module compiles the rule into a counting query and runs it + * through an injected runner. It EXECUTES. + * DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against + * the BI-P022 contract). This module INGESTS the results. It does + * not run dbt, and `providerKind` says so rather than implying + * otherwise. + * GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here + * executes it. The adapter maps a GE validation result onto the + * canonical verdict so a GE suite counts toward the same gate. + * + * Calling all three "providers" while only one executes would be the kind of + * overstatement that turns into "we have Great Expectations coverage" in a + * status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or + * `"ingesting"`, and the gate records which produced each verdict. + * + * ── BI-P025's DoD ───────────────────────────────────────────────────────── + * "Failed critical quality gate can QUARANTINE serving projection." + * + * `quarantine.ts` is that, and the important half is what happens on the READ + * side: a quarantined projection is REFUSED, not annotated. An annotation on a + * dashboard nobody reads is not a quarantine. + */ +import { SiaxError } from '../../platform/errors'; +import type { QualityRuleKind, QualitySeverity } from '../domain/entities'; + +/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */ +export const QUALITY_RULES = [ + 'not_null', + 'unique', + 'range', + 'referential', + 'freshness', + 'schema', + 'distribution', + 'custom', +] as const; +export type QualityRule = (typeof QUALITY_RULES)[number]; + +/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */ +export const RULE_KIND_ALIASES: Record = { + not_null: 'not_null', + unique: 'unique', + range: 'range', + referential: 'referential', + freshness: 'freshness', + schema: 'custom', + distribution: 'accepted_values', + custom: 'custom', +}; + +export interface QualityRuleSpec { + rule_id: string; + tenant_id: string; + dataset_id: string; + rule: QualityRule; + /** Column the rule applies to; absent for dataset-level rules. */ + field?: string; + severity: QualitySeverity; + params: QualityRuleParams; + description?: string; +} + +export interface QualityRuleParams { + /** range */ + min?: number; + max?: number; + /** distribution / accepted values */ + accepted_values?: string[]; + /** distribution: the share of rows that must fall in accepted_values */ + min_share?: number; + /** referential */ + references_dataset?: string; + references_field?: string; + /** freshness: the maximum acceptable age of the newest row */ + max_age_seconds?: number; + timestamp_field?: string; + /** schema: the columns and types that must exist */ + expected_columns?: Array<{ name: string; type: string }>; + /** unique / not_null take no parameters */ + /** custom: an operator-authored counting query, reviewed like code */ + custom_violation_sql?: string; +} + +export interface QualityVerdict { + rule_id: string; + dataset_id: string; + tenant_id: string; + rule: QualityRule; + severity: QualitySeverity; + passed: boolean; + violation_count: number; + evaluated_rows: number; + /** Which provider produced this, and whether it EXECUTED or merely ingested. */ + produced_by: string; + provider_kind: 'executing' | 'ingesting'; + evaluated_at: string; + /** Short operator-facing note. Never raw engine output. */ + detail?: string; +} + +export interface SiaxDataQualityProvider { + readonly provider: string; + readonly providerKind: 'executing' | 'ingesting'; + /** Rules this provider can actually evaluate. */ + supports(rule: QualityRule): boolean; + evaluate(spec: QualityRuleSpec): Promise; +} + +/** + * Runs a counting query and returns two numbers. The only engine coupling in + * this module, so a DuckDB or Trino runner drops in unchanged. + */ +export interface QualityQueryRunner { + readonly engine: string; + /** + * Must apply the caller's tenant scoping. The runner is handed the tenant + * explicitly rather than trusting the SQL to carry it. + */ + count(args: { + tenant_id: string; + dataset_id: string; + violationExpression: string; + /** Bound parameters. Values NEVER appear in the expression text. */ + params: Record; + }): Promise<{ violations: number; evaluated: number }>; +} + +const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/; + +function assertField(field: string | undefined, rule: QualityRule): string { + if (!field || !IDENT_RE.test(field)) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + `rule "${rule}" requires a valid field name`, + ); + } + return field; +} + +/** + * The provider that actually runs checks. + * + * Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and + * the runner counts rows where it is true. That shape is chosen so a rule can + * never return "how many rows are fine", which is the number that looks + * healthy when a table is empty. + */ +export class NativeQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'native'; + readonly providerKind = 'executing' as const; + + constructor( + private readonly runner: QualityQueryRunner, + private readonly now: () => number = Date.now, + ) {} + + supports(rule: QualityRule): boolean { + // `schema` and `referential` need catalog/second-table access the counting + // runner does not have; they are honestly unsupported here rather than + // silently passing. + return !['schema', 'referential'].includes(rule); + } + + async evaluate(spec: QualityRuleSpec): Promise { + if (!this.supports(spec.rule)) { + throw new SiaxError( + 'CAPABILITY_UNAVAILABLE', + '0pt.analytics.quality', + `the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`, + ); + } + const { expression, params } = this.compile(spec); + const { violations, evaluated } = await this.runner.count({ + tenant_id: spec.tenant_id, + dataset_id: spec.dataset_id, + violationExpression: expression, + params, + }); + + // An EMPTY dataset is not a passing dataset. A `not_null` check over zero + // rows finds zero violations and would report success while the pipeline + // is silently broken — the single most common false green in data quality. + const emptyIsFailure = spec.rule !== 'custom'; + const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0; + + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + passed, + violation_count: violations, + evaluated_rows: evaluated, + produced_by: `${this.provider}:${this.runner.engine}`, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: + evaluated === 0 && emptyIsFailure + ? 'dataset is EMPTY — reported as a failure, not as zero violations' + : undefined, + }; + } + + private compile(spec: QualityRuleSpec): { expression: string; params: Record } { + const p = spec.params; + switch (spec.rule) { + case 'not_null': { + const f = assertField(spec.field, spec.rule); + return { expression: `${f} IS NULL OR ${f} = ''`, params: {} }; + } + case 'unique': { + const f = assertField(spec.field, spec.rule); + // Counted over a grouped subquery by the runner; the expression names + // the key so the runner can build it. + return { expression: `__unique__(${f})`, params: {} }; + } + case 'range': { + const f = assertField(spec.field, spec.rule); + if (p.min === undefined && p.max === undefined) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a range rule needs a min or a max', + ); + } + const parts: string[] = []; + const params: Record = {}; + if (p.min !== undefined) { + parts.push(`${f} < {qmin:Float64}`); + params.param_qmin = String(p.min); + } + if (p.max !== undefined) { + parts.push(`${f} > {qmax:Float64}`); + params.param_qmax = String(p.max); + } + return { expression: parts.join(' OR '), params }; + } + case 'freshness': { + const f = assertField(p.timestamp_field ?? spec.field, spec.rule); + if (!p.max_age_seconds || p.max_age_seconds <= 0) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a freshness rule needs max_age_seconds', + ); + } + return { + expression: `${f} < now() - toIntervalSecond({qage:UInt64})`, + params: { param_qage: String(Math.floor(p.max_age_seconds)) }, + }; + } + case 'distribution': { + const f = assertField(spec.field, spec.rule); + if (!p.accepted_values?.length) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a distribution rule needs accepted_values', + ); + } + const params: Record = {}; + const bound = p.accepted_values.map((v, i) => { + params[`param_qv${i}`] = v; + return `{qv${i}:String}`; + }); + return { expression: `${f} NOT IN (${bound.join(', ')})`, params }; + } + case 'custom': { + if (!p.custom_violation_sql?.trim()) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'a custom rule needs custom_violation_sql', + ); + } + // Deliberately NOT parsed or guarded here. A custom rule is + // operator-authored, reviewed like code, and stored in the rule table + // — not caller input. It never reaches this function from a request + // body; `QualityRuleSpec` rows are written by an admin path. + return { expression: p.custom_violation_sql, params: {} }; + } + default: + throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`); + } + } +} + +/** + * Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against + * the BI-P022 contract, and pretending otherwise here would misreport where + * the work happens. + */ +export interface DbtTestResult { + unique_id: string; + status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped'; + failures: number; + /** dbt does not report rows evaluated; absent, not zero. */ + rows_evaluated?: number; + message?: string; +} + +export class DbtQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'dbt'; + readonly providerKind = 'ingesting' as const; + constructor( + private readonly results: Map, + private readonly now: () => number = Date.now, + ) {} + + supports(): boolean { + return true; + } + + async evaluate(spec: QualityRuleSpec): Promise { + const result = this.results.get(spec.rule_id); + if (!result) { + // A missing result is UNKNOWN, and unknown is a FAILURE for a blocking + // rule. Treating "no result" as "passed" is how a quality gate stops + // gating the moment the test stops running. + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + passed: false, + violation_count: 0, + evaluated_rows: 0, + produced_by: this.provider, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: 'no dbt result for this rule — reported as a failure, not as a pass', + }; + } + return { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + // `skipped` and `error` are NOT passes. dbt's `skipped` in particular is + // the shape of l0re's silently-skipped Postgres tests. + passed: result.status === 'pass', + violation_count: result.failures, + evaluated_rows: result.rows_evaluated ?? 0, + produced_by: `dbt:${result.unique_id}`, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined, + }; + } +} + +/** Ingests a Great Expectations validation result. Does NOT run GE. */ +export interface GreatExpectationsResult { + expectation_suite_name: string; + success: boolean; + results: Array<{ + expectation_config: { expectation_type: string }; + success: boolean; + result?: { unexpected_count?: number; element_count?: number }; + }>; +} + +export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider { + readonly provider = 'great_expectations'; + readonly providerKind = 'ingesting' as const; + constructor( + private readonly suites: Map, + private readonly now: () => number = Date.now, + ) {} + + supports(): boolean { + return true; + } + + async evaluate(spec: QualityRuleSpec): Promise { + const suite = this.suites.get(spec.rule_id); + const base = { + rule_id: spec.rule_id, + dataset_id: spec.dataset_id, + tenant_id: spec.tenant_id, + rule: spec.rule, + severity: spec.severity, + produced_by: this.provider, + provider_kind: this.providerKind, + evaluated_at: new Date(this.now()).toISOString(), + }; + if (!suite) { + return { + ...base, + passed: false, + violation_count: 0, + evaluated_rows: 0, + detail: 'no GE result — reported as a failure', + }; + } + const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0); + const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0); + return { + ...base, + passed: suite.success, + violation_count: unexpected, + evaluated_rows: elements, + produced_by: `great_expectations:${suite.expectation_suite_name}`, + detail: suite.success ? undefined : 'GE suite failed', + }; + } +} diff --git a/src/analytics/quality/quarantine.ts b/src/analytics/quality/quarantine.ts new file mode 100644 index 0000000..518d232 --- /dev/null +++ b/src/analytics/quality/quarantine.ts @@ -0,0 +1,188 @@ +/** + * src/analytics/quality/quarantine.ts + * ==================================== + * BI-P025's DoD: "Failed critical quality gate can QUARANTINE serving + * projection." + * + * The word that matters is on the READ side. A quarantine that annotates a + * result is a label; a quarantine that REFUSES the read is a quarantine. This + * module does the second, and `assertProjectionReadable` is the single place + * every serving read passes through. + * + * Severity semantics, stated because they are the whole gate: + * `block` — a failure quarantines the projection. Reads are refused. + * `warn` — a failure is recorded and surfaced, and reads continue. + * A rule with no severity does not exist: `QualityRuleSpec.severity` is + * required, so "we forgot to mark it critical" cannot silently become `warn`. + */ +import { SiaxError } from '../../platform/errors'; +import type { QualityVerdict } from './quality-provider'; + +export interface QuarantineRecord { + tenant_id: string; + dataset_id: string; + quarantined: boolean; + /** Rule ids whose blocking failure caused it. */ + caused_by: string[]; + since?: string; + released_at?: string; + released_by?: string; + /** Operator-facing summary. */ + reason?: string; +} + +export interface QuarantineStore { + get(tenantId: string, datasetId: string): Promise; + put(record: QuarantineRecord): Promise; +} + +export class InMemoryQuarantineStore implements QuarantineStore { + private readonly rows = new Map(); + private key(t: string, d: string): string { + return `${t}::${d}`; + } + async get(t: string, d: string): Promise { + return this.rows.get(this.key(t, d)); + } + async put(r: QuarantineRecord): Promise { + this.rows.set(this.key(r.tenant_id, r.dataset_id), r); + } +} + +export interface GateOutcome { + dataset_id: string; + tenant_id: string; + passed: boolean; + quarantined: boolean; + blocking_failures: QualityVerdict[]; + warnings: QualityVerdict[]; + evaluated: number; +} + +export class QualityGate { + constructor( + private readonly store: QuarantineStore, + private readonly now: () => number = Date.now, + ) {} + + /** + * Apply a run's verdicts. + * + * A gate over ZERO verdicts does not pass. "No rules ran" and "every rule + * passed" produce the same green in most implementations, and that is how a + * quality gate stops gating without anyone noticing — the same shape as + * l0re's 28 silently-skipped tests and api0's `trivy --exit-code 0`. + */ + async apply(tenantId: string, datasetId: string, verdicts: QualityVerdict[]): Promise { + if (verdicts.length === 0) { + const record: QuarantineRecord = { + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: true, + caused_by: [], + since: new Date(this.now()).toISOString(), + reason: 'no quality rules were evaluated — an ungated dataset is quarantined, not trusted', + }; + await this.store.put(record); + return { + dataset_id: datasetId, + tenant_id: tenantId, + passed: false, + quarantined: true, + blocking_failures: [], + warnings: [], + evaluated: 0, + }; + } + + for (const v of verdicts) { + if (v.tenant_id !== tenantId || v.dataset_id !== datasetId) { + // A verdict from another tenant's run must never influence this gate. + throw new SiaxError( + 'AUTHORIZATION_DENIED', + '0pt.analytics.quality', + 'a quality verdict from a different tenant or dataset was supplied to this gate', + ); + } + } + + const blocking = verdicts.filter((v) => !v.passed && v.severity === 'block'); + const warnings = verdicts.filter((v) => !v.passed && v.severity === 'warn'); + + if (blocking.length > 0) { + await this.store.put({ + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: true, + caused_by: blocking.map((v) => v.rule_id), + since: new Date(this.now()).toISOString(), + reason: `${blocking.length} blocking quality rule(s) failed`, + }); + } else { + // Passing does NOT auto-release: releasing is an act with an operator + // behind it (`release()`), because a flapping rule would otherwise let a + // dataset quarantine and un-quarantine itself between reads. + const existing = await this.store.get(tenantId, datasetId); + if (!existing || !existing.quarantined) { + await this.store.put({ + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: false, + caused_by: [], + }); + } + } + + return { + dataset_id: datasetId, + tenant_id: tenantId, + passed: blocking.length === 0, + quarantined: blocking.length > 0 || (await this.isQuarantined(tenantId, datasetId)), + blocking_failures: blocking, + warnings, + evaluated: verdicts.length, + }; + } + + async isQuarantined(tenantId: string, datasetId: string): Promise { + return (await this.store.get(tenantId, datasetId))?.quarantined ?? false; + } + + /** Release requires an operator and a reason. It is not automatic. */ + async release(tenantId: string, datasetId: string, by: string, reason: string): Promise { + if (!by?.trim() || !reason?.trim()) { + throw new SiaxError( + 'INVALID_REQUEST', + '0pt.analytics.quality', + 'releasing a quarantine requires a principal and a reason', + ); + } + const record: QuarantineRecord = { + tenant_id: tenantId, + dataset_id: datasetId, + quarantined: false, + caused_by: [], + released_at: new Date(this.now()).toISOString(), + released_by: by, + reason, + }; + await this.store.put(record); + return record; + } + + /** + * The read-side gate. Every serving projection read passes through here. + * Refuses, rather than annotating — an annotation on a dashboard nobody + * reads is not a quarantine. + */ + async assertProjectionReadable(tenantId: string, datasetId: string): Promise { + const record = await this.store.get(tenantId, datasetId); + if (record?.quarantined) { + throw new SiaxError( + 'DEPENDENCY_UNAVAILABLE', + '0pt.analytics.quality', + `serving projection for "${datasetId}" is QUARANTINED: ${record.reason ?? 'failed quality gate'}`, + ); + } + } +} diff --git a/tests/analytics/quality-and-freshness.test.ts b/tests/analytics/quality-and-freshness.test.ts new file mode 100644 index 0000000..2140c85 --- /dev/null +++ b/tests/analytics/quality-and-freshness.test.ts @@ -0,0 +1,377 @@ +/** + * tests/analytics/quality-and-freshness.test.ts + * ============================================== + * BI-P025 (data quality provider) and BI-P026 (data freshness). + * + * Both DoDs are about a FALSE GREEN, so both are tested by constructing the + * false green and requiring a failure: + * + * BI-P025 A quality gate that passes when nothing ran. A `not_null` check + * over an EMPTY table finds zero violations — the single most + * common false green in data quality. Both are asserted to FAIL. + * + * BI-P026 UNKNOWN age quietly rendering as current, and a result released + * with no freshness envelope at all. Both are refused. + */ +import { describe, expect, it } from 'vitest'; +import { SiaxError } from '@/src/platform/errors'; +import { + DbtQualityProvider, + GreatExpectationsQualityProvider, + NativeQualityProvider, + QUALITY_RULES, + type QualityQueryRunner, + type QualityRuleSpec, + type QualityVerdict, +} from '@/src/analytics/quality/quality-provider'; +import { InMemoryQuarantineStore, QualityGate } from '@/src/analytics/quality/quarantine'; +import { + assertFreshnessDisclosed, + computeFreshness, + FRESHNESS_STATES, + isPresentableAsCurrent, + NON_CURRENT_STATES, +} from '@/src/analytics/quality/freshness'; + +function runner( + violations: number, + evaluated: number, +): QualityQueryRunner & { lastExpression?: string; lastParams?: Record } { + const r: QualityQueryRunner & { lastExpression?: string; lastParams?: Record } = { + engine: 'clickhouse', + async count(args) { + r.lastExpression = args.violationExpression; + r.lastParams = args.params; + return { violations, evaluated }; + }, + }; + return r; +} + +const RULE: QualityRuleSpec = { + rule_id: 'q_not_null_customer', + tenant_id: 't_alpha', + dataset_id: 'ds_sales', + rule: 'not_null', + field: 'customer', + severity: 'block', + params: {}, +}; + +describe('BI-P025 — the native provider EXECUTES, and says so', () => { + it('declares every rule BI-P025 names', () => { + expect([...QUALITY_RULES]).toEqual([ + 'not_null', + 'unique', + 'range', + 'referential', + 'freshness', + 'schema', + 'distribution', + 'custom', + ]); + }); + + it('passes a clean dataset', async () => { + const verdict = await new NativeQualityProvider(runner(0, 1000)).evaluate(RULE); + expect(verdict.passed).toBe(true); + expect(verdict.provider_kind).toBe('executing'); + }); + + it('FAILS an empty dataset instead of reporting zero violations', async () => { + // The false green this whole file exists for: a not-null check over zero + // rows finds zero violations and looks healthy while the pipeline is dead. + const verdict = await new NativeQualityProvider(runner(0, 0)).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('EMPTY'); + }); + + it('fails a dataset with violations', async () => { + const verdict = await new NativeQualityProvider(runner(7, 1000)).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.violation_count).toBe(7); + }); + + it('binds rule parameters instead of splicing them into the expression', async () => { + const r = runner(0, 10); + await new NativeQualityProvider(r).evaluate({ + ...RULE, + rule: 'distribution', + field: 'channel', + params: { accepted_values: ['web', "retail'; DROP TABLE x; --"] }, + }); + expect(r.lastExpression).not.toContain('DROP'); + expect(Object.values(r.lastParams ?? {})).toContain("retail'; DROP TABLE x; --"); + }); + + it('refuses a field name that is not an identifier', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + await expect(p.evaluate({ ...RULE, field: 'customer; DROP TABLE x' })).rejects.toBeInstanceOf(SiaxError); + }); + + it('refuses an incomplete rule rather than evaluating a meaningless one', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + await expect(p.evaluate({ ...RULE, rule: 'range', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'freshness', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'distribution', params: {} })).rejects.toBeInstanceOf(SiaxError); + await expect(p.evaluate({ ...RULE, rule: 'custom', params: {} })).rejects.toBeInstanceOf(SiaxError); + }); + + it('says CAPABILITY_UNAVAILABLE for rules it cannot evaluate, rather than passing them', async () => { + const p = new NativeQualityProvider(runner(0, 10)); + for (const rule of ['schema', 'referential'] as const) { + await expect(p.evaluate({ ...RULE, rule })).rejects.toBeInstanceOf(SiaxError); + expect(p.supports(rule)).toBe(false); + } + }); +}); + +describe('BI-P025 — the ingesting providers do not pretend to execute', () => { + it('dbt and GE declare themselves as ingesting', async () => { + expect(new DbtQualityProvider(new Map()).providerKind).toBe('ingesting'); + expect(new GreatExpectationsQualityProvider(new Map()).providerKind).toBe('ingesting'); + }); + + it('a MISSING dbt result is a failure, not a pass', async () => { + // Treating "no result" as "passed" is how a gate stops gating the moment + // the test stops running. + const verdict = await new DbtQualityProvider(new Map()).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('no dbt result'); + }); + + it.each(['error', 'warn', 'skipped'] as const)('dbt status %s is NOT a pass', async (status) => { + const results = new Map([[RULE.rule_id, { unique_id: 't1', status, failures: 0 }]]); + const verdict = await new DbtQualityProvider(results).evaluate(RULE); + expect(verdict.passed).toBe(false); + }); + + it('dbt pass is a pass', async () => { + const results = new Map([[RULE.rule_id, { unique_id: 't1', status: 'pass' as const, failures: 0 }]]); + expect((await new DbtQualityProvider(results).evaluate(RULE)).passed).toBe(true); + }); + + it('a missing GE suite is a failure', async () => { + expect((await new GreatExpectationsQualityProvider(new Map()).evaluate(RULE)).passed).toBe(false); + }); + + it("a GE suite's unexpected counts are carried through", async () => { + const suites = new Map([ + [ + RULE.rule_id, + { + expectation_suite_name: 'sales_suite', + success: false, + results: [ + { + expectation_config: { expectation_type: 'expect_column_values_to_not_be_null' }, + success: false, + result: { unexpected_count: 12, element_count: 1000 }, + }, + ], + }, + ], + ]); + const verdict = await new GreatExpectationsQualityProvider(suites).evaluate(RULE); + expect(verdict.passed).toBe(false); + expect(verdict.violation_count).toBe(12); + expect(verdict.evaluated_rows).toBe(1000); + }); +}); + +describe('BI-P025 DoD — a failed critical gate QUARANTINES the serving projection', () => { + function verdict(overrides: Partial = {}): QualityVerdict { + return { + rule_id: 'q1', + dataset_id: 'ds_sales', + tenant_id: 't_alpha', + rule: 'not_null', + severity: 'block', + passed: true, + violation_count: 0, + evaluated_rows: 100, + produced_by: 'native:clickhouse', + provider_kind: 'executing', + evaluated_at: new Date().toISOString(), + ...overrides, + }; + } + + it('a blocking failure quarantines, and the READ is then REFUSED', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + // Before: readable. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + + const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'block' })]); + expect(outcome.quarantined).toBe(true); + expect(outcome.passed).toBe(false); + + // After: refused. Not annotated — refused. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('a WARN failure does not quarantine, but is surfaced', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + const outcome = await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false, severity: 'warn' })]); + expect(outcome.quarantined).toBe(false); + expect(outcome.warnings).toHaveLength(1); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it("a gate over ZERO verdicts QUARANTINES — 'no rules ran' is not 'everything passed'", async () => { + // The same shape as l0re's silently-skipped tests and api0's + // `trivy --exit-code 0`: a check that produced nothing reporting green. + const gate = new QualityGate(new InMemoryQuarantineStore()); + const outcome = await gate.apply('t_alpha', 'ds_sales', []); + expect(outcome.passed).toBe(false); + expect(outcome.quarantined).toBe(true); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('a later passing run does NOT auto-release a quarantine', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: true })]); + // A flapping rule would otherwise quarantine and un-quarantine the dataset + // between two reads of the same dashboard. + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).rejects.toBeInstanceOf(SiaxError); + }); + + it('release requires a principal and a reason', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await expect(gate.release('t_alpha', 'ds_sales', '', 'x')).rejects.toBeInstanceOf(SiaxError); + await expect(gate.release('t_alpha', 'ds_sales', 'u_1', ' ')).rejects.toBeInstanceOf(SiaxError); + const released = await gate.release('t_alpha', 'ds_sales', 'u_1', 'upstream backfilled'); + expect(released.released_by).toBe('u_1'); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it("quarantine is tenant-scoped: one tenant's failure does not block another", async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await gate.apply('t_alpha', 'ds_sales', [verdict({ passed: false })]); + await expect(gate.assertProjectionReadable('t_beta', 'ds_sales')).resolves.toBeUndefined(); + }); + + it('refuses a verdict belonging to another tenant or dataset', async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + await expect( + gate.apply('t_alpha', 'ds_sales', [verdict({ tenant_id: 't_beta' })]), + ).rejects.toBeInstanceOf(SiaxError); + await expect( + gate.apply('t_alpha', 'ds_sales', [verdict({ dataset_id: 'ds_other' })]), + ).rejects.toBeInstanceOf(SiaxError); + }); +}); + +describe('BI-P026 — five states, and UNKNOWN is not FRESH', () => { + const HOUR = 3_600_000; + const NOW = Date.parse('2026-09-07T12:00:00Z'); + const policy = { expectedIntervalMs: HOUR }; + + it('declares the five states BI-P026 names', () => { + expect([...FRESHNESS_STATES]).toEqual(['FRESH', 'DELAYED', 'STALE', 'FAILED', 'UNKNOWN']); + }); + + it('only FRESH may be presented as current', () => { + expect(isPresentableAsCurrent('FRESH')).toBe(true); + for (const s of NON_CURRENT_STATES) expect(isPresentableAsCurrent(s)).toBe(false); + expect(NON_CURRENT_STATES).toContain('UNKNOWN'); + }); + + const cases: Array<[string, Parameters[0], string]> = [ + [ + 'recent load', + { lastSuccessfulLoadAt: '2026-09-07T11:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'FRESH', + ], + [ + 'behind schedule', + { lastSuccessfulLoadAt: '2026-09-07T09:50:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'DELAYED', + ], + [ + 'well past the threshold', + { lastSuccessfulLoadAt: '2026-09-06T12:00:00Z', lastRunStatus: 'succeeded', policy, now: NOW }, + 'STALE', + ], + [ + 'the last run FAILED, even though the data looks recent', + { lastSuccessfulLoadAt: '2026-09-07T11:59:00Z', lastRunStatus: 'failed', policy, now: NOW }, + 'FAILED', + ], + ['never run', { lastRunStatus: 'never_run', policy, now: NOW }, 'UNKNOWN'], + ['no load timestamp at all', { policy, now: NOW }, 'UNKNOWN'], + ['an unparseable timestamp', { lastSuccessfulLoadAt: 'not a date', policy, now: NOW }, 'UNKNOWN'], + ]; + + it.each(cases)('%s -> %s', (_label, input, expected) => { + expect(computeFreshness(input).state).toBe(expected); + }); + + it('FAILED outranks a recent timestamp — a broken pipeline must not look FRESH', () => { + const env = computeFreshness({ + lastSuccessfulLoadAt: '2026-09-07T11:59:59Z', + lastRunStatus: 'failed', + policy, + now: NOW, + }); + expect(env.state).toBe('FAILED'); + expect(env.disclosure).toContain('FAILED'); + }); + + it('every state carries a disclosure a UI can render verbatim', () => { + for (const [, input] of cases) { + const env = computeFreshness(input); + expect(env.disclosure.length).toBeGreaterThan(10); + } + }); +}); + +describe('BI-P026 DoD — stale data is never SILENTLY presented as current', () => { + const env = computeFreshness({ + lastSuccessfulLoadAt: '2026-09-01T00:00:00Z', + lastRunStatus: 'succeeded', + policy: { expectedIntervalMs: 3_600_000 }, + now: Date.parse('2026-09-07T12:00:00Z'), + }); + + it('refuses to release a result with NO freshness envelope', () => { + // Not defaulted to FRESH, not defaulted to UNKNOWN: unreleasable. "We + // forgot to attach it" and "the data is current" are indistinguishable to + // a reader, so a default would make forgetting the silent path. + expect(() => assertFreshnessDisclosed({}, 'dashboard panel p1')).toThrowError(SiaxError); + }); + + it('refuses to release STALE data that the consumer has not acknowledged', () => { + expect(env.state).toBe('STALE'); + let thrown: unknown; + try { + assertFreshnessDisclosed({ freshness: env }, 'dashboard panel p1'); + } catch (e) { + thrown = e; + } + expect(thrown).toBeInstanceOf(SiaxError); + expect((thrown as SiaxError).message).toContain('STALE'); + }); + + it('releases STALE data once the consumer has acknowledged it', () => { + expect(assertFreshnessDisclosed({ freshness: env, acknowledged_non_current: true }, 'p1').state).toBe( + 'STALE', + ); + }); + + it.each(NON_CURRENT_STATES)('refuses an unacknowledged %s result', (state) => { + expect(() => assertFreshnessDisclosed({ freshness: { ...env, state } }, 'p1')).toThrowError(SiaxError); + }); + + it('releases FRESH data without ceremony', () => { + expect(assertFreshnessDisclosed({ freshness: { ...env, state: 'FRESH' } }, 'p1').state).toBe('FRESH'); + }); + + it('refuses an envelope carrying a state that is not one of the five', () => { + expect(() => + assertFreshnessDisclosed({ freshness: { ...env, state: 'PROBABLY_FINE' as never } }, 'p1'), + ).toThrowError(SiaxError); + }); +}); diff --git a/tests/integration/data-quality-clickhouse.integration.test.ts b/tests/integration/data-quality-clickhouse.integration.test.ts new file mode 100644 index 0000000..203de1a --- /dev/null +++ b/tests/integration/data-quality-clickhouse.integration.test.ts @@ -0,0 +1,244 @@ +/** + * tests/integration/data-quality-clickhouse.integration.test.ts + * ============================================================== + * BI-P025 — the native quality provider against a live ClickHouse. + * + * The unit suite proves the provider's LOGIC (an empty dataset is a failure, a + * missing dbt result is a failure). This suite proves the part a fake runner + * cannot: that the compiled expressions are valid ClickHouse and count the + * right rows, including the empty-dataset case where a real engine returns + * zero violations over zero rows and every naive implementation reports green. + * + * It also re-proves BI-P011's forced tenant predicate on this path: a quality + * run executes on an ADMINISTRATIVE credential (it must read the base table), + * so if the predicate were missing here, one tenant's dirty rows would + * quarantine another tenant's projection. + * + * Gated on CLICKHOUSE_URL. With the variable set and the server down, this + * suite FAILS rather than skipping. + */ +import fs from 'node:fs'; +import path from 'node:path'; +import { beforeAll, describe, expect, it } from 'vitest'; +import { NativeQualityProvider, type QualityRuleSpec } from '@/src/analytics/quality/quality-provider'; +import { ClickHouseQualityRunner } from '@/src/analytics/quality/clickhouse-quality-runner'; +import { InMemoryQuarantineStore, QualityGate } from '@/src/analytics/quality/quarantine'; + +const BASE_URL = process.env.CLICKHOUSE_URL ?? ''; +const AUTH = process.env.CLICKHOUSE_AUTH ?? 'default:'; +const suite = BASE_URL ? describe : describe.skip; +const DB = 'opt_quality_test'; +const EVIDENCE_PATH = path.join(process.cwd(), 'evidence/bi-p025/native-quality-live.json'); + +async function admin(sql: string): Promise<{ ok: boolean; body: string }> { + const url = new URL(BASE_URL); + url.searchParams.set('query', sql); + const res = await fetch(url.toString(), { + method: 'POST', + headers: { Authorization: `Basic ${Buffer.from(AUTH).toString('base64')}` }, + body: '', + signal: AbortSignal.timeout(60_000), + }); + return { ok: res.ok, body: await res.text() }; +} + +const runner = new ClickHouseQualityRunner({ + baseUrl: BASE_URL || 'http://unset', + basicAuth: AUTH, + database: DB, + tables: { ds_sales: 'sales', ds_empty: 'empty_sales' }, +}); +const provider = new NativeQualityProvider(runner); +const evidence: Record = {}; + +function rule(over: Partial): QualityRuleSpec { + return { + rule_id: 'q1', + tenant_id: 't_alpha', + dataset_id: 'ds_sales', + rule: 'not_null', + field: 'customer', + severity: 'block', + params: {}, + ...over, + }; +} + +beforeAll(async () => { + if (!BASE_URL) return; + const v = await admin('SELECT version()'); + if (!v.ok) throw new Error('CLICKHOUSE_URL is set but ClickHouse is not answering'); + evidence.clickhouse_version = v.body.trim(); + evidence.recorded_at = new Date().toISOString(); + + await admin(`DROP DATABASE IF EXISTS ${DB}`); + await admin(`CREATE DATABASE ${DB}`); + await admin( + `CREATE TABLE ${DB}.sales (tenant_id String, dataset_id String, customer String, amount Float64, ` + + `channel String, event_time DateTime) ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + await admin( + `CREATE TABLE ${DB}.empty_sales (tenant_id String, dataset_id String, customer String, amount Float64, ` + + `channel String, event_time DateTime) ENGINE = MergeTree ORDER BY (tenant_id, event_time)`, + ); + + // t_alpha: 100 clean rows, then deliberate defects. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'cust_' || toString(number), number, 'web', now() ` + + `FROM numbers(100)`, + ); + // 5 rows with an empty customer (the not_null violation). + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', '', number, 'web', now() FROM numbers(5)`, + ); + // 3 rows out of range (amount > 1000). + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'big_' || toString(number), 5000, 'web', now() ` + + `FROM numbers(3)`, + ); + // 4 rows with an unaccepted channel. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'ch_' || toString(number), 1, 'carrier_pigeon', now() ` + + `FROM numbers(4)`, + ); + // 6 rows sharing two duplicate keys (3 each) — the unique violation. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_alpha', 'ds_sales', 'dup_' || toString(number % 2), number, 'web', now() ` + + `FROM numbers(6)`, + ); + + // t_beta: FILTHY. Every rule fails here. If the tenant predicate is missing, + // t_alpha's counts jump and this tenant quarantines the other one. + await admin( + `INSERT INTO ${DB}.sales SELECT 't_beta', 'ds_sales', '', 999999, 'carrier_pigeon', now() FROM numbers(500)`, + ); +}, 120_000); + +suite('BI-P025 live — the compiled expressions are valid SQL and count the right rows', () => { + it("not_null finds exactly the 5 empty-customer rows, out of t_alpha's 118", async () => { + const verdict = await provider.evaluate(rule({ rule: 'not_null', field: 'customer' })); + expect(verdict.evaluated_rows).toBe(118); + expect(verdict.violation_count).toBe(5); + expect(verdict.passed).toBe(false); + evidence.not_null = { evaluated: verdict.evaluated_rows, violations: verdict.violation_count }; + }); + + it('range finds exactly the 3 rows over the maximum', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'range', field: 'amount', params: { min: 0, max: 1000 } }), + ); + expect(verdict.violation_count).toBe(3); + evidence.range = { violations: verdict.violation_count }; + }); + + it('distribution finds exactly the 4 rows outside the accepted values', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'distribution', field: 'channel', params: { accepted_values: ['web', 'retail'] } }), + ); + expect(verdict.violation_count).toBe(4); + evidence.distribution = { violations: verdict.violation_count }; + }); + + it('unique counts ROWS sitting on a duplicated key, not the number of keys', async () => { + const verdict = await provider.evaluate(rule({ rule: 'unique', field: 'customer' })); + // 11, and the arithmetic is worth writing down because the first version + // of this test asserted 6 and the engine disagreed: + // * `dup_0` and `dup_1` appear 3 times each -> 6 rows + // * the five EMPTY customers are all the same key '' -> 5 rows + // The empty strings are duplicates of one another, so `not_null` and + // `unique` overlap on the same defect. That is the engine being right and + // the fixture arithmetic being wrong, not a rule that over-counts. + expect(verdict.violation_count).toBe(11); + evidence.unique = { + violations: verdict.violation_count, + breakdown: '6 rows on dup_0/dup_1, plus 5 rows sharing the empty-string key', + }; + }); + + it('freshness finds no violation for rows written just now', async () => { + const verdict = await provider.evaluate( + rule({ rule: 'freshness', params: { timestamp_field: 'event_time', max_age_seconds: 3600 } }), + ); + expect(verdict.violation_count).toBe(0); + expect(verdict.passed).toBe(true); + }); + + it('a hostile accepted-value is bound, not executed', async () => { + const verdict = await provider.evaluate( + rule({ + rule: 'distribution', + field: 'channel', + params: { accepted_values: ['web', 'retail', "'); DROP TABLE sales; --"] }, + }), + ); + expect(verdict.violation_count).toBe(4); + // The table is still there, with its rows. + const after = await admin(`SELECT count() FROM ${DB}.sales`); + expect(Number(after.body.trim())).toBe(618); + }); +}); + +suite('BI-P025 live — the EMPTY dataset false green, against a real engine', () => { + it('a real ClickHouse returns 0 violations over 0 rows, and the provider calls it a FAILURE', async () => { + // The counterfactual first: the raw counts genuinely look clean. + const raw = await runner.count({ + tenant_id: 't_alpha', + dataset_id: 'ds_empty', + violationExpression: "customer IS NULL OR customer = ''", + params: {}, + }); + expect(raw).toEqual({ violations: 0, evaluated: 0 }); + + // A naive implementation reports green here. This one does not. + const verdict = await provider.evaluate(rule({ dataset_id: 'ds_empty' })); + expect(verdict.passed).toBe(false); + expect(verdict.detail).toContain('EMPTY'); + evidence.empty_dataset = { + raw_counts_from_clickhouse: raw, + verdict_passed: verdict.passed, + note: 'a real engine returns zero violations over zero rows; the provider refuses to call that a pass', + }; + }); +}); + +suite("BI-P025 live — a quality run cannot read another tenant's rows", () => { + it("t_alpha's counts are unaffected by t_beta's 500 filthy rows", async () => { + const all = await admin(`SELECT count() FROM ${DB}.sales`); + expect(Number(all.body.trim())).toBe(618); // 118 alpha + 500 beta + + const alpha = await provider.evaluate( + rule({ tenant_id: 't_alpha', rule: 'not_null', field: 'customer' }), + ); + const beta = await provider.evaluate(rule({ tenant_id: 't_beta', rule: 'not_null', field: 'customer' })); + + expect(alpha.evaluated_rows).toBe(118); + expect(alpha.violation_count).toBe(5); + expect(beta.evaluated_rows).toBe(500); + expect(beta.violation_count).toBe(500); + evidence.tenant_isolation = { + base_table_rows: 618, + alpha_evaluated: alpha.evaluated_rows, + alpha_violations: alpha.violation_count, + beta_evaluated: beta.evaluated_rows, + beta_violations: beta.violation_count, + }; + }); + + it("t_beta's blocking failure quarantines t_beta only", async () => { + const gate = new QualityGate(new InMemoryQuarantineStore()); + const beta = await provider.evaluate(rule({ tenant_id: 't_beta', rule: 'not_null', field: 'customer' })); + await gate.apply('t_beta', 'ds_sales', [beta]); + await expect(gate.assertProjectionReadable('t_beta', 'ds_sales')).rejects.toThrow(/QUARANTINED/); + await expect(gate.assertProjectionReadable('t_alpha', 'ds_sales')).resolves.toBeUndefined(); + }); + + it('refuses a dataset with no registered physical table rather than guessing one', async () => { + await expect(provider.evaluate(rule({ dataset_id: 'ds_unregistered' }))).rejects.toThrow(); + }); + + it('writes the evidence file', () => { + fs.mkdirSync(path.dirname(EVIDENCE_PATH), { recursive: true }); + fs.writeFileSync(EVIDENCE_PATH, `${JSON.stringify(evidence, null, 2)}\n`); + expect(fs.existsSync(EVIDENCE_PATH)).toBe(true); + }); +});