/** * src/quality/expectation-engine.ts * ================================== * The LOCAL executor for the core Great Expectations expectation subset * (Master-DoD §10 "Great Expectations-integrationen verklig" — the * interchange format IS GE's; execution is either this local engine for the * supported subset, or delegated to the GE backend in 0pt-motor via N0D for * everything else — §11 DQ "Great Expectations executed"). * * HONESTY RULE: an unsupported expectation type NEVER silently passes. * `supports()` says what this engine can run; QualityService delegates or * fails loudly for the rest. * * Supported (semantics per GE docs, element-wise unless noted): * expect_column_values_to_not_be_null {column} * expect_column_values_to_be_unique {column} * expect_column_values_to_be_between {column, min_value?, max_value?} * expect_column_values_to_match_regex {column, regex} * expect_column_values_to_be_in_set {column, value_set} * expect_table_row_count_to_be_between {min_value?, max_value?} (table-level) * expect_column_to_exist {column} (table-level) */ import { SiaxError } from "../platform/errors"; import type { QualityExpectation } from "./quality-types"; export interface ExpectationOutcome { success: boolean; pass_ratio: number; evaluated_count: number; unexpected_count: number; unexpected_samples: unknown[]; } const SUPPORTED = new Set([ "expect_column_values_to_not_be_null", "expect_column_values_to_be_unique", "expect_column_values_to_be_between", "expect_column_values_to_match_regex", "expect_column_values_to_be_in_set", "expect_table_row_count_to_be_between", "expect_column_to_exist", ]); const MAX_SAMPLES = 20; export class ExpectationEngine { supports(expectationType: string): boolean { return SUPPORTED.has(expectationType); } supportedTypes(): string[] { return [...SUPPORTED]; } execute(expectation: QualityExpectation, rows: Record[]): ExpectationOutcome { const { expectation_type: type, kwargs } = expectation; if (!this.supports(type)) { throw new SiaxError( "CAPABILITY_UNAVAILABLE", "0pt.quality", `expectation_type "${type}" is not executable by the local engine — delegate to the GE backend` ); } const column = kwargs.column as string | undefined; switch (type) { case "expect_column_to_exist": { const exists = rows.length === 0 || rows.some((r) => column! in r); return tableOutcome(exists); } case "expect_table_row_count_to_be_between": { const min = numOrUndefined(kwargs.min_value); const max = numOrUndefined(kwargs.max_value); const n = rows.length; const ok = (min === undefined || n >= min) && (max === undefined || n <= max); return { ...tableOutcome(ok), evaluated_count: n }; } case "expect_column_values_to_not_be_null": { return elementwise(rows, column!, (v) => v !== null && v !== undefined); } case "expect_column_values_to_be_unique": { const seen = new Map(); for (const r of rows) { const key = JSON.stringify(r[column!] ?? null); seen.set(key, (seen.get(key) ?? 0) + 1); } return elementwise(rows, column!, (v) => (seen.get(JSON.stringify(v ?? null)) ?? 0) <= 1); } case "expect_column_values_to_be_between": { const min = numOrUndefined(kwargs.min_value); const max = numOrUndefined(kwargs.max_value); return elementwise(rows, column!, (v) => { if (v === null || v === undefined) return true; // GE semantics: nulls don't fail range checks const n = Number(v); if (!Number.isFinite(n)) return false; return (min === undefined || n >= min) && (max === undefined || n <= max); }); } case "expect_column_values_to_match_regex": { const re = new RegExp(String(kwargs.regex)); return elementwise(rows, column!, (v) => { if (v === null || v === undefined) return true; return re.test(String(v)); }); } case "expect_column_values_to_be_in_set": { const set = new Set((kwargs.value_set as unknown[]) ?? []); return elementwise(rows, column!, (v) => { if (v === null || v === undefined) return true; return set.has(v); }); } default: // Unreachable (guarded by supports()), kept for exhaustiveness. throw new SiaxError("CAPABILITY_UNAVAILABLE", "0pt.quality", `unhandled expectation "${type}"`); } } } function tableOutcome(ok: boolean): ExpectationOutcome { return { success: ok, pass_ratio: ok ? 1 : 0, evaluated_count: 1, unexpected_count: ok ? 0 : 1, unexpected_samples: [], }; } function numOrUndefined(v: unknown): number | undefined { if (v === null || v === undefined) return undefined; const n = Number(v); return Number.isFinite(n) ? n : undefined; } function elementwise( rows: Record[], column: string, predicate: (v: unknown) => boolean ): ExpectationOutcome { let unexpected = 0; const samples: unknown[] = []; for (const r of rows) { const v = r[column]; if (!predicate(v)) { unexpected++; if (samples.length < MAX_SAMPLES) samples.push(v ?? null); } } const evaluated = rows.length; return { success: unexpected === 0, pass_ratio: evaluated === 0 ? 1 : (evaluated - unexpected) / evaluated, evaluated_count: evaluated, unexpected_count: unexpected, unexpected_samples: samples, }; }