419 lines
14 KiB
TypeScript
419 lines
14 KiB
TypeScript
/**
|
|
* src/analytics/quality/quality-provider.ts
|
|
* ==========================================
|
|
* BI-P025 — DATA QUALITY PROVIDER.
|
|
*
|
|
* One interface, three provider kinds, and an honest split between them:
|
|
*
|
|
* NATIVE — this module compiles the rule into a counting query and runs it
|
|
* through an injected runner. It EXECUTES.
|
|
* DBT — dbt runs its own tests elsewhere (n0d#38 owns execution against
|
|
* the BI-P022 contract). This module INGESTS the results. It does
|
|
* not run dbt, and `providerKind` says so rather than implying
|
|
* otherwise.
|
|
* GREAT_EXPECTATIONS — same posture. GE is a Python library; nothing here
|
|
* executes it. The adapter maps a GE validation result onto the
|
|
* canonical verdict so a GE suite counts toward the same gate.
|
|
*
|
|
* Calling all three "providers" while only one executes would be the kind of
|
|
* overstatement that turns into "we have Great Expectations coverage" in a
|
|
* status report. `SiaxDataQualityProvider.providerKind` is `"executing"` or
|
|
* `"ingesting"`, and the gate records which produced each verdict.
|
|
*
|
|
* ── BI-P025's DoD ─────────────────────────────────────────────────────────
|
|
* "Failed critical quality gate can QUARANTINE serving projection."
|
|
*
|
|
* `quarantine.ts` is that, and the important half is what happens on the READ
|
|
* side: a quarantined projection is REFUSED, not annotated. An annotation on a
|
|
* dashboard nobody reads is not a quarantine.
|
|
*/
|
|
import { SiaxError } from '../../platform/errors';
|
|
import type { QualityRuleKind, QualitySeverity } from '../domain/entities';
|
|
|
|
/** BI-P025's rule list, verbatim, mapped onto BI-P002's persisted kinds. */
|
|
export const QUALITY_RULES = [
|
|
'not_null',
|
|
'unique',
|
|
'range',
|
|
'referential',
|
|
'freshness',
|
|
'schema',
|
|
'distribution',
|
|
'custom',
|
|
] as const;
|
|
export type QualityRule = (typeof QUALITY_RULES)[number];
|
|
|
|
/** BI-P002 persists `accepted_values` and `row_count`; BI-P025 names `distribution` and `range`. */
|
|
export const RULE_KIND_ALIASES: Record<QualityRule, QualityRuleKind> = {
|
|
not_null: 'not_null',
|
|
unique: 'unique',
|
|
range: 'range',
|
|
referential: 'referential',
|
|
freshness: 'freshness',
|
|
schema: 'custom',
|
|
distribution: 'accepted_values',
|
|
custom: 'custom',
|
|
};
|
|
|
|
export interface QualityRuleSpec {
|
|
rule_id: string;
|
|
tenant_id: string;
|
|
dataset_id: string;
|
|
rule: QualityRule;
|
|
/** Column the rule applies to; absent for dataset-level rules. */
|
|
field?: string;
|
|
severity: QualitySeverity;
|
|
params: QualityRuleParams;
|
|
description?: string;
|
|
}
|
|
|
|
export interface QualityRuleParams {
|
|
/** range */
|
|
min?: number;
|
|
max?: number;
|
|
/** distribution / accepted values */
|
|
accepted_values?: string[];
|
|
/** distribution: the share of rows that must fall in accepted_values */
|
|
min_share?: number;
|
|
/** referential */
|
|
references_dataset?: string;
|
|
references_field?: string;
|
|
/** freshness: the maximum acceptable age of the newest row */
|
|
max_age_seconds?: number;
|
|
timestamp_field?: string;
|
|
/** schema: the columns and types that must exist */
|
|
expected_columns?: Array<{ name: string; type: string }>;
|
|
/** unique / not_null take no parameters */
|
|
/** custom: an operator-authored counting query, reviewed like code */
|
|
custom_violation_sql?: string;
|
|
}
|
|
|
|
export interface QualityVerdict {
|
|
rule_id: string;
|
|
dataset_id: string;
|
|
tenant_id: string;
|
|
rule: QualityRule;
|
|
severity: QualitySeverity;
|
|
passed: boolean;
|
|
violation_count: number;
|
|
evaluated_rows: number;
|
|
/** Which provider produced this, and whether it EXECUTED or merely ingested. */
|
|
produced_by: string;
|
|
provider_kind: 'executing' | 'ingesting';
|
|
evaluated_at: string;
|
|
/** Short operator-facing note. Never raw engine output. */
|
|
detail?: string;
|
|
}
|
|
|
|
export interface SiaxDataQualityProvider {
|
|
readonly provider: string;
|
|
readonly providerKind: 'executing' | 'ingesting';
|
|
/** Rules this provider can actually evaluate. */
|
|
supports(rule: QualityRule): boolean;
|
|
evaluate(spec: QualityRuleSpec): Promise<QualityVerdict>;
|
|
}
|
|
|
|
/**
|
|
* Runs a counting query and returns two numbers. The only engine coupling in
|
|
* this module, so a DuckDB or Trino runner drops in unchanged.
|
|
*/
|
|
export interface QualityQueryRunner {
|
|
readonly engine: string;
|
|
/**
|
|
* Must apply the caller's tenant scoping. The runner is handed the tenant
|
|
* explicitly rather than trusting the SQL to carry it.
|
|
*/
|
|
count(args: {
|
|
tenant_id: string;
|
|
dataset_id: string;
|
|
violationExpression: string;
|
|
/** Bound parameters. Values NEVER appear in the expression text. */
|
|
params: Record<string, string>;
|
|
}): Promise<{ violations: number; evaluated: number }>;
|
|
}
|
|
|
|
const IDENT_RE = /^[A-Za-z_][A-Za-z0-9_]{0,62}$/;
|
|
|
|
function assertField(field: string | undefined, rule: QualityRule): string {
|
|
if (!field || !IDENT_RE.test(field)) {
|
|
throw new SiaxError(
|
|
'INVALID_REQUEST',
|
|
'0pt.analytics.quality',
|
|
`rule "${rule}" requires a valid field name`,
|
|
);
|
|
}
|
|
return field;
|
|
}
|
|
|
|
/**
|
|
* The provider that actually runs checks.
|
|
*
|
|
* Every rule compiles to a VIOLATION EXPRESSION — a boolean over one row — and
|
|
* the runner counts rows where it is true. That shape is chosen so a rule can
|
|
* never return "how many rows are fine", which is the number that looks
|
|
* healthy when a table is empty.
|
|
*/
|
|
export class NativeQualityProvider implements SiaxDataQualityProvider {
|
|
readonly provider = 'native';
|
|
readonly providerKind = 'executing' as const;
|
|
|
|
constructor(
|
|
private readonly runner: QualityQueryRunner,
|
|
private readonly now: () => number = Date.now,
|
|
) {}
|
|
|
|
supports(rule: QualityRule): boolean {
|
|
// `schema` and `referential` need catalog/second-table access the counting
|
|
// runner does not have; they are honestly unsupported here rather than
|
|
// silently passing.
|
|
return !['schema', 'referential'].includes(rule);
|
|
}
|
|
|
|
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
|
if (!this.supports(spec.rule)) {
|
|
throw new SiaxError(
|
|
'CAPABILITY_UNAVAILABLE',
|
|
'0pt.analytics.quality',
|
|
`the native provider cannot evaluate "${spec.rule}" — use a dbt test or a Great Expectations suite`,
|
|
);
|
|
}
|
|
const { expression, params } = this.compile(spec);
|
|
const { violations, evaluated } = await this.runner.count({
|
|
tenant_id: spec.tenant_id,
|
|
dataset_id: spec.dataset_id,
|
|
violationExpression: expression,
|
|
params,
|
|
});
|
|
|
|
// An EMPTY dataset is not a passing dataset. A `not_null` check over zero
|
|
// rows finds zero violations and would report success while the pipeline
|
|
// is silently broken — the single most common false green in data quality.
|
|
const emptyIsFailure = spec.rule !== 'custom';
|
|
const passed = evaluated === 0 && emptyIsFailure ? false : violations === 0;
|
|
|
|
return {
|
|
rule_id: spec.rule_id,
|
|
dataset_id: spec.dataset_id,
|
|
tenant_id: spec.tenant_id,
|
|
rule: spec.rule,
|
|
severity: spec.severity,
|
|
passed,
|
|
violation_count: violations,
|
|
evaluated_rows: evaluated,
|
|
produced_by: `${this.provider}:${this.runner.engine}`,
|
|
provider_kind: this.providerKind,
|
|
evaluated_at: new Date(this.now()).toISOString(),
|
|
detail:
|
|
evaluated === 0 && emptyIsFailure
|
|
? 'dataset is EMPTY — reported as a failure, not as zero violations'
|
|
: undefined,
|
|
};
|
|
}
|
|
|
|
private compile(spec: QualityRuleSpec): { expression: string; params: Record<string, string> } {
|
|
const p = spec.params;
|
|
switch (spec.rule) {
|
|
case 'not_null': {
|
|
const f = assertField(spec.field, spec.rule);
|
|
return { expression: `${f} IS NULL OR ${f} = ''`, params: {} };
|
|
}
|
|
case 'unique': {
|
|
const f = assertField(spec.field, spec.rule);
|
|
// Counted over a grouped subquery by the runner; the expression names
|
|
// the key so the runner can build it.
|
|
return { expression: `__unique__(${f})`, params: {} };
|
|
}
|
|
case 'range': {
|
|
const f = assertField(spec.field, spec.rule);
|
|
if (p.min === undefined && p.max === undefined) {
|
|
throw new SiaxError(
|
|
'INVALID_REQUEST',
|
|
'0pt.analytics.quality',
|
|
'a range rule needs a min or a max',
|
|
);
|
|
}
|
|
const parts: string[] = [];
|
|
const params: Record<string, string> = {};
|
|
if (p.min !== undefined) {
|
|
parts.push(`${f} < {qmin:Float64}`);
|
|
params.param_qmin = String(p.min);
|
|
}
|
|
if (p.max !== undefined) {
|
|
parts.push(`${f} > {qmax:Float64}`);
|
|
params.param_qmax = String(p.max);
|
|
}
|
|
return { expression: parts.join(' OR '), params };
|
|
}
|
|
case 'freshness': {
|
|
const f = assertField(p.timestamp_field ?? spec.field, spec.rule);
|
|
if (!p.max_age_seconds || p.max_age_seconds <= 0) {
|
|
throw new SiaxError(
|
|
'INVALID_REQUEST',
|
|
'0pt.analytics.quality',
|
|
'a freshness rule needs max_age_seconds',
|
|
);
|
|
}
|
|
return {
|
|
expression: `${f} < now() - toIntervalSecond({qage:UInt64})`,
|
|
params: { param_qage: String(Math.floor(p.max_age_seconds)) },
|
|
};
|
|
}
|
|
case 'distribution': {
|
|
const f = assertField(spec.field, spec.rule);
|
|
if (!p.accepted_values?.length) {
|
|
throw new SiaxError(
|
|
'INVALID_REQUEST',
|
|
'0pt.analytics.quality',
|
|
'a distribution rule needs accepted_values',
|
|
);
|
|
}
|
|
const params: Record<string, string> = {};
|
|
const bound = p.accepted_values.map((v, i) => {
|
|
params[`param_qv${i}`] = v;
|
|
return `{qv${i}:String}`;
|
|
});
|
|
return { expression: `${f} NOT IN (${bound.join(', ')})`, params };
|
|
}
|
|
case 'custom': {
|
|
if (!p.custom_violation_sql?.trim()) {
|
|
throw new SiaxError(
|
|
'INVALID_REQUEST',
|
|
'0pt.analytics.quality',
|
|
'a custom rule needs custom_violation_sql',
|
|
);
|
|
}
|
|
// Deliberately NOT parsed or guarded here. A custom rule is
|
|
// operator-authored, reviewed like code, and stored in the rule table
|
|
// — not caller input. It never reaches this function from a request
|
|
// body; `QualityRuleSpec` rows are written by an admin path.
|
|
return { expression: p.custom_violation_sql, params: {} };
|
|
}
|
|
default:
|
|
throw new SiaxError('INVALID_REQUEST', '0pt.analytics.quality', `unsupported rule "${spec.rule}"`);
|
|
}
|
|
}
|
|
}
|
|
|
|
/**
|
|
* Ingests dbt test results. Does NOT run dbt — n0d#38 owns execution against
|
|
* the BI-P022 contract, and pretending otherwise here would misreport where
|
|
* the work happens.
|
|
*/
|
|
export interface DbtTestResult {
|
|
unique_id: string;
|
|
status: 'pass' | 'fail' | 'error' | 'warn' | 'skipped';
|
|
failures: number;
|
|
/** dbt does not report rows evaluated; absent, not zero. */
|
|
rows_evaluated?: number;
|
|
message?: string;
|
|
}
|
|
|
|
export class DbtQualityProvider implements SiaxDataQualityProvider {
|
|
readonly provider = 'dbt';
|
|
readonly providerKind = 'ingesting' as const;
|
|
constructor(
|
|
private readonly results: Map<string, DbtTestResult>,
|
|
private readonly now: () => number = Date.now,
|
|
) {}
|
|
|
|
supports(): boolean {
|
|
return true;
|
|
}
|
|
|
|
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
|
const result = this.results.get(spec.rule_id);
|
|
if (!result) {
|
|
// A missing result is UNKNOWN, and unknown is a FAILURE for a blocking
|
|
// rule. Treating "no result" as "passed" is how a quality gate stops
|
|
// gating the moment the test stops running.
|
|
return {
|
|
rule_id: spec.rule_id,
|
|
dataset_id: spec.dataset_id,
|
|
tenant_id: spec.tenant_id,
|
|
rule: spec.rule,
|
|
severity: spec.severity,
|
|
passed: false,
|
|
violation_count: 0,
|
|
evaluated_rows: 0,
|
|
produced_by: this.provider,
|
|
provider_kind: this.providerKind,
|
|
evaluated_at: new Date(this.now()).toISOString(),
|
|
detail: 'no dbt result for this rule — reported as a failure, not as a pass',
|
|
};
|
|
}
|
|
return {
|
|
rule_id: spec.rule_id,
|
|
dataset_id: spec.dataset_id,
|
|
tenant_id: spec.tenant_id,
|
|
rule: spec.rule,
|
|
severity: spec.severity,
|
|
// `skipped` and `error` are NOT passes. dbt's `skipped` in particular is
|
|
// the shape of l0re's silently-skipped Postgres tests.
|
|
passed: result.status === 'pass',
|
|
violation_count: result.failures,
|
|
evaluated_rows: result.rows_evaluated ?? 0,
|
|
produced_by: `dbt:${result.unique_id}`,
|
|
provider_kind: this.providerKind,
|
|
evaluated_at: new Date(this.now()).toISOString(),
|
|
detail: result.status !== 'pass' ? `dbt status ${result.status}` : undefined,
|
|
};
|
|
}
|
|
}
|
|
|
|
/** Ingests a Great Expectations validation result. Does NOT run GE. */
|
|
export interface GreatExpectationsResult {
|
|
expectation_suite_name: string;
|
|
success: boolean;
|
|
results: Array<{
|
|
expectation_config: { expectation_type: string };
|
|
success: boolean;
|
|
result?: { unexpected_count?: number; element_count?: number };
|
|
}>;
|
|
}
|
|
|
|
export class GreatExpectationsQualityProvider implements SiaxDataQualityProvider {
|
|
readonly provider = 'great_expectations';
|
|
readonly providerKind = 'ingesting' as const;
|
|
constructor(
|
|
private readonly suites: Map<string, GreatExpectationsResult>,
|
|
private readonly now: () => number = Date.now,
|
|
) {}
|
|
|
|
supports(): boolean {
|
|
return true;
|
|
}
|
|
|
|
async evaluate(spec: QualityRuleSpec): Promise<QualityVerdict> {
|
|
const suite = this.suites.get(spec.rule_id);
|
|
const base = {
|
|
rule_id: spec.rule_id,
|
|
dataset_id: spec.dataset_id,
|
|
tenant_id: spec.tenant_id,
|
|
rule: spec.rule,
|
|
severity: spec.severity,
|
|
produced_by: this.provider,
|
|
provider_kind: this.providerKind,
|
|
evaluated_at: new Date(this.now()).toISOString(),
|
|
};
|
|
if (!suite) {
|
|
return {
|
|
...base,
|
|
passed: false,
|
|
violation_count: 0,
|
|
evaluated_rows: 0,
|
|
detail: 'no GE result — reported as a failure',
|
|
};
|
|
}
|
|
const unexpected = suite.results.reduce((a, r) => a + (r.result?.unexpected_count ?? 0), 0);
|
|
const elements = suite.results.reduce((a, r) => Math.max(a, r.result?.element_count ?? 0), 0);
|
|
return {
|
|
...base,
|
|
passed: suite.success,
|
|
violation_count: unexpected,
|
|
evaluated_rows: elements,
|
|
produced_by: `great_expectations:${suite.expectation_suite_name}`,
|
|
detail: suite.success ? undefined : 'GE suite failed',
|
|
};
|
|
}
|
|
}
|