Files
0pt/src/analytics/quality/quality-provider.ts
T

419 lines
14 KiB
TypeScript

/**
* src/analytics/quality/quality-provider.ts
* ==========================================
* BI-P025 — DATA QUALITY PROVIDER.
*
* One interface, three provider kinds, and an honest split between them:
*
* NATIVE — this module compiles the rule into a counting query and runs it
* through an injected runner. It EXECUTES.
* DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against
* the BI-P022 contract). This module INGESTS the results. It does
* not run dbt, and `providerKind` says so rather than implying
* otherwise.
* GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here
* executes it. The adapter maps a GE validation result onto the
* canonical verdict so a GE suite counts toward the same gate.
*
* Calling all three "providers" while only one executes would be the kind of
* overstatement that turns into "we have Great Expectations coverage" in a
* status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or
* `"ingesting"`, and the gate records which produced each verdict.
*
* ── BI-P025's DoD ─────────────────────────────────────────────────────────
* "Failed critical quality gate can QUARANTINE serving projection."
*
* `quarantine.ts` is that, and the important half is what happens on the READ
* side: a quarantined projection is REFUSED, not annotated. An annotation on a
* dashboard nobody reads is not a quarantine.
*/
import { SiaxError } from '../../platform/errors';
import type { QualityRuleKind, QualitySeverity } from '../domain/entities';
/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */
export const QUALITY_RULES = [
'not_null',
'unique',
'range',
'referential',
'freshness',
'schema',
'distribution',
'custom',
] as const;
export type QualityRule = (typeof QUALITY_RULES)[number];
/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */
export const RULE_KIND_ALIASES: Record<QualityRule, QualityRuleKind> = {
not_null: 'not_null',
unique: 'unique',
range: 'range',
referential: 'referential',
freshness: 'freshness',
schema: 'custom',
distribution: 'accepted_values',
custom: 'custom',
};
export interface QualityRuleSpec {
rule_id: string;
tenant_id: string;
dataset_id: string;
rule: QualityRule;
/** Column the rule applies to; absent for dataset-level rules. */
field?: string;
severity: QualitySeverity;
params: QualityRuleParams;
description?: string;
}
export interface QualityRuleParams {
/** range */
min?: number;
max?: number;
/** distribution / accepted values */
accepted_values?: string[];
/** distribution: the share of rows that must fall in accepted_values */
min_share?: number;
/** referential */
references_dataset?: string;
references_field?: string;
/** freshness: the maximum acceptable age of the newest row */
max_age_seconds?: number;
timestamp_field?: string;
/** schema: the columns and types that must exist */
expected_columns?: Array<{ name: string; type: string }>;
/** unique / not_null take no parameters */
/** custom: an operator-authored counting query, reviewed like code */
custom_violation_sql?: string;
}
export interface QualityVerdict {
rule_id: string;
dataset_id: string;
tenant_id: string;
rule: QualityRule;
severity: QualitySeverity;
passed: boolean;
violation_count: number;
evaluated_rows: number;
/** Which provider produced this, and whether it EXECUTED or merely ingested. */
produced_by: string;
provider_kind: 'executing' | 'ingesting';
evaluated_at: string;
/** Short operator-facing note. Never raw engine output. */
detail?: string;
}
export interface SiaxDataQualityProvider {
readonly provider: string;
readonly providerKind: 'executing' | 'ingesting';
/** Rules this provider can actually evaluate. */
supports(rule: QualityRule): boolean;
evaluate(spec: QualityRuleSpec): Promise<QualityVerdict>;
}
/**
* Runs a counting query and returns two numbers. The only engine coupling in
* this module, so a DuckDB or Trino runner drops in unchanged.
*/
export interface QualityQueryRunner {
readonly engine: string;
/**
* Must apply the caller's tenant scoping. The runner is handed the tenant
* explicitly rather than trusting the SQL to carry it.
*/
count(args: {
tenant_id: string;
dataset_id: string;
violationExpression: string;
/** Bound parameters. Values NEVER appear in the expression text. */
params: Record<string, string>;
}): Promise<{ violations: number; evaluated: number }>;
}
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
function assertField(field: string | undefined, rule: QualityRule): string {
if (!field || !IDENT_RE.test(field)) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
`rule "${rule}" requires a valid field name`,
);
}
return field;
}
/**
* The provider that actually runs checks.
*
* Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and
* the runner counts rows where it is true. That shape is chosen so a rule can
* never return "how many rows are fine", which is the number that looks
* healthy when a table is empty.
*/
export class NativeQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'native';
readonly providerKind = 'executing' as const;
constructor(
private readonly runner: QualityQueryRunner,
private readonly now: () => number = Date.now,
) {}
supports(rule: QualityRule): boolean {
// `schema` and `referential` need catalog/second-table access the counting
// runner does not have; they are honestly unsupported here rather than
// silently passing.
return !['schema', 'referential'].includes(rule);
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
if (!this.supports(spec.rule)) {
throw new SiaxError(
'CAPABILITY_UNAVAILABLE',
'0pt.analytics.quality',
`the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`,
);
}
const { expression, params } = this.compile(spec);
const { violations, evaluated } = await this.runner.count({
tenant_id: spec.tenant_id,
dataset_id: spec.dataset_id,
violationExpression: expression,
params,
});
// An EMPTY dataset is not a passing dataset. A `not_null` check over zero
// rows finds zero violations and would report success while the pipeline
// is silently broken — the single most common false green in data quality.
const emptyIsFailure = spec.rule !== 'custom';
const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0;
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
passed,
violation_count: violations,
evaluated_rows: evaluated,
produced_by: `${this.provider}:${this.runner.engine}`,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail:
evaluated === 0 && emptyIsFailure
? 'dataset is EMPTY — reported as a failure, not as zero violations'
: undefined,
};
}
private compile(spec: QualityRuleSpec): { expression: string; params: Record<string, string> } {
const p = spec.params;
switch (spec.rule) {
case 'not_null': {
const f = assertField(spec.field, spec.rule);
return { expression: `${f} IS NULL OR ${f} = ''`, params: {} };
}
case 'unique': {
const f = assertField(spec.field, spec.rule);
// Counted over a grouped subquery by the runner; the expression names
// the key so the runner can build it.
return { expression: `__unique__(${f})`, params: {} };
}
case 'range': {
const f = assertField(spec.field, spec.rule);
if (p.min === undefined && p.max === undefined) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a range rule needs a min or a max',
);
}
const parts: string[] = [];
const params: Record<string, string> = {};
if (p.min !== undefined) {
parts.push(`${f} < {qmin:Float64}`);
params.param_qmin = String(p.min);
}
if (p.max !== undefined) {
parts.push(`${f} > {qmax:Float64}`);
params.param_qmax = String(p.max);
}
return { expression: parts.join(' OR '), params };
}
case 'freshness': {
const f = assertField(p.timestamp_field ?? spec.field, spec.rule);
if (!p.max_age_seconds || p.max_age_seconds <= 0) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a freshness rule needs max_age_seconds',
);
}
return {
expression: `${f} < now() - toIntervalSecond({qage:UInt64})`,
params: { param_qage: String(Math.floor(p.max_age_seconds)) },
};
}
case 'distribution': {
const f = assertField(spec.field, spec.rule);
if (!p.accepted_values?.length) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a distribution rule needs accepted_values',
);
}
const params: Record<string, string> = {};
const bound = p.accepted_values.map((v, i) => {
params[`param_qv${i}`] = v;
return `{qv${i}:String}`;
});
return { expression: `${f} NOT IN (${bound.join(', ')})`, params };
}
case 'custom': {
if (!p.custom_violation_sql?.trim()) {
throw new SiaxError(
'INVALID_REQUEST',
'0pt.analytics.quality',
'a custom rule needs custom_violation_sql',
);
}
// Deliberately NOT parsed or guarded here. A custom rule is
// operator-authored, reviewed like code, and stored in the rule table
// — not caller input. It never reaches this function from a request
// body; `QualityRuleSpec` rows are written by an admin path.
return { expression: p.custom_violation_sql, params: {} };
}
default:
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`);
}
}
}
/**
* Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against
* the BI-P022 contract, and pretending otherwise here would misreport where
* the work happens.
*/
export interface DbtTestResult {
unique_id: string;
status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped';
failures: number;
/** dbt does not report rows evaluated; absent, not zero. */
rows_evaluated?: number;
message?: string;
}
export class DbtQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'dbt';
readonly providerKind = 'ingesting' as const;
constructor(
private readonly results: Map<string, DbtTestResult>,
private readonly now: () => number = Date.now,
) {}
supports(): boolean {
return true;
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
const result = this.results.get(spec.rule_id);
if (!result) {
// A missing result is UNKNOWN, and unknown is a FAILURE for a blocking
// rule. Treating "no result" as "passed" is how a quality gate stops
// gating the moment the test stops running.
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
passed: false,
violation_count: 0,
evaluated_rows: 0,
produced_by: this.provider,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail: 'no dbt result for this rule — reported as a failure, not as a pass',
};
}
return {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
// `skipped` and `error` are NOT passes. dbt's `skipped` in particular is
// the shape of l0re's silently-skipped Postgres tests.
passed: result.status === 'pass',
violation_count: result.failures,
evaluated_rows: result.rows_evaluated ?? 0,
produced_by: `dbt:${result.unique_id}`,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined,
};
}
}
/** Ingests a Great Expectations validation result. Does NOT run GE. */
export interface GreatExpectationsResult {
expectation_suite_name: string;
success: boolean;
results: Array<{
expectation_config: { expectation_type: string };
success: boolean;
result?: { unexpected_count?: number; element_count?: number };
}>;
}
export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider {
readonly provider = 'great_expectations';
readonly providerKind = 'ingesting' as const;
constructor(
private readonly suites: Map<string, GreatExpectationsResult>,
private readonly now: () => number = Date.now,
) {}
supports(): boolean {
return true;
}
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
const suite = this.suites.get(spec.rule_id);
const base = {
rule_id: spec.rule_id,
dataset_id: spec.dataset_id,
tenant_id: spec.tenant_id,
rule: spec.rule,
severity: spec.severity,
produced_by: this.provider,
provider_kind: this.providerKind,
evaluated_at: new Date(this.now()).toISOString(),
};
if (!suite) {
return {
...base,
passed: false,
violation_count: 0,
evaluated_rows: 0,
detail: 'no GE result — reported as a failure',
};
}
const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0);
const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0);
return {
...base,
passed: suite.success,
violation_count: unexpected,
evaluated_rows: elements,
produced_by: `great_expectations:${suite.expectation_suite_name}`,
detail: suite.success ? undefined : 'GE suite failed',
};
}
}