feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
da58176735
commit
b46807da75
@@ -0,0 +1,158 @@
|
||||
/**
|
||||
* src/quality/expectation-engine.ts
|
||||
* ==================================
|
||||
* The LOCAL executor for the core Great Expectations expectation subset
|
||||
* (Master-DoD §10 "Great Expectations-integrationen verklig" — the
|
||||
* interchange format IS GE's; execution is either this local engine for the
|
||||
* supported subset, or delegated to the GE backend in 0pt-motor via N0D for
|
||||
* everything else — §11 DQ "Great Expectations executed").
|
||||
*
|
||||
* HONESTY RULE: an unsupported expectation type NEVER silently passes.
|
||||
* `supports()` says what this engine can run; QualityService delegates or
|
||||
* fails loudly for the rest.
|
||||
*
|
||||
* Supported (semantics per GE docs, element-wise unless noted):
|
||||
* expect_column_values_to_not_be_null {column}
|
||||
* expect_column_values_to_be_unique {column}
|
||||
* expect_column_values_to_be_between {column, min_value?, max_value?}
|
||||
* expect_column_values_to_match_regex {column, regex}
|
||||
* expect_column_values_to_be_in_set {column, value_set}
|
||||
* expect_table_row_count_to_be_between {min_value?, max_value?} (table-level)
|
||||
* expect_column_to_exist {column} (table-level)
|
||||
*/
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import type { QualityExpectation } from "./quality-types";
|
||||
|
||||
export interface ExpectationOutcome {
|
||||
success: boolean;
|
||||
pass_ratio: number;
|
||||
evaluated_count: number;
|
||||
unexpected_count: number;
|
||||
unexpected_samples: unknown[];
|
||||
}
|
||||
|
||||
const SUPPORTED = new Set([
|
||||
"expect_column_values_to_not_be_null",
|
||||
"expect_column_values_to_be_unique",
|
||||
"expect_column_values_to_be_between",
|
||||
"expect_column_values_to_match_regex",
|
||||
"expect_column_values_to_be_in_set",
|
||||
"expect_table_row_count_to_be_between",
|
||||
"expect_column_to_exist",
|
||||
]);
|
||||
|
||||
const MAX_SAMPLES = 20;
|
||||
|
||||
export class ExpectationEngine {
|
||||
supports(expectationType: string): boolean {
|
||||
return SUPPORTED.has(expectationType);
|
||||
}
|
||||
|
||||
supportedTypes(): string[] {
|
||||
return [...SUPPORTED];
|
||||
}
|
||||
|
||||
execute(expectation: QualityExpectation, rows: Record<string, unknown>[]): ExpectationOutcome {
|
||||
const { expectation_type: type, kwargs } = expectation;
|
||||
if (!this.supports(type)) {
|
||||
throw new SiaxError(
|
||||
"CAPABILITY_UNAVAILABLE",
|
||||
"0pt.quality",
|
||||
`expectation_type "${type}" is not executable by the local engine — delegate to the GE backend`
|
||||
);
|
||||
}
|
||||
const column = kwargs.column as string | undefined;
|
||||
|
||||
switch (type) {
|
||||
case "expect_column_to_exist": {
|
||||
const exists = rows.length === 0 || rows.some((r) => column! in r);
|
||||
return tableOutcome(exists);
|
||||
}
|
||||
case "expect_table_row_count_to_be_between": {
|
||||
const min = numOrUndefined(kwargs.min_value);
|
||||
const max = numOrUndefined(kwargs.max_value);
|
||||
const n = rows.length;
|
||||
const ok = (min === undefined || n >= min) && (max === undefined || n <= max);
|
||||
return { ...tableOutcome(ok), evaluated_count: n };
|
||||
}
|
||||
case "expect_column_values_to_not_be_null": {
|
||||
return elementwise(rows, column!, (v) => v !== null && v !== undefined);
|
||||
}
|
||||
case "expect_column_values_to_be_unique": {
|
||||
const seen = new Map<string, number>();
|
||||
for (const r of rows) {
|
||||
const key = JSON.stringify(r[column!] ?? null);
|
||||
seen.set(key, (seen.get(key) ?? 0) + 1);
|
||||
}
|
||||
return elementwise(rows, column!, (v) => (seen.get(JSON.stringify(v ?? null)) ?? 0) <= 1);
|
||||
}
|
||||
case "expect_column_values_to_be_between": {
|
||||
const min = numOrUndefined(kwargs.min_value);
|
||||
const max = numOrUndefined(kwargs.max_value);
|
||||
return elementwise(rows, column!, (v) => {
|
||||
if (v === null || v === undefined) return true; // GE semantics: nulls don't fail range checks
|
||||
const n = Number(v);
|
||||
if (!Number.isFinite(n)) return false;
|
||||
return (min === undefined || n >= min) && (max === undefined || n <= max);
|
||||
});
|
||||
}
|
||||
case "expect_column_values_to_match_regex": {
|
||||
const re = new RegExp(String(kwargs.regex));
|
||||
return elementwise(rows, column!, (v) => {
|
||||
if (v === null || v === undefined) return true;
|
||||
return re.test(String(v));
|
||||
});
|
||||
}
|
||||
case "expect_column_values_to_be_in_set": {
|
||||
const set = new Set((kwargs.value_set as unknown[]) ?? []);
|
||||
return elementwise(rows, column!, (v) => {
|
||||
if (v === null || v === undefined) return true;
|
||||
return set.has(v);
|
||||
});
|
||||
}
|
||||
default:
|
||||
// Unreachable (guarded by supports()), kept for exhaustiveness.
|
||||
throw new SiaxError("CAPABILITY_UNAVAILABLE", "0pt.quality", `unhandled expectation "${type}"`);
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
function tableOutcome(ok: boolean): ExpectationOutcome {
|
||||
return {
|
||||
success: ok,
|
||||
pass_ratio: ok ? 1 : 0,
|
||||
evaluated_count: 1,
|
||||
unexpected_count: ok ? 0 : 1,
|
||||
unexpected_samples: [],
|
||||
};
|
||||
}
|
||||
|
||||
function numOrUndefined(v: unknown): number | undefined {
|
||||
if (v === null || v === undefined) return undefined;
|
||||
const n = Number(v);
|
||||
return Number.isFinite(n) ? n : undefined;
|
||||
}
|
||||
|
||||
function elementwise(
|
||||
rows: Record<string, unknown>[],
|
||||
column: string,
|
||||
predicate: (v: unknown) => boolean
|
||||
): ExpectationOutcome {
|
||||
let unexpected = 0;
|
||||
const samples: unknown[] = [];
|
||||
for (const r of rows) {
|
||||
const v = r[column];
|
||||
if (!predicate(v)) {
|
||||
unexpected++;
|
||||
if (samples.length < MAX_SAMPLES) samples.push(v ?? null);
|
||||
}
|
||||
}
|
||||
const evaluated = rows.length;
|
||||
return {
|
||||
success: unexpected === 0,
|
||||
pass_ratio: evaluated === 0 ? 1 : (evaluated - unexpected) / evaluated,
|
||||
evaluated_count: evaluated,
|
||||
unexpected_count: unexpected,
|
||||
unexpected_samples: samples,
|
||||
};
|
||||
}
|
||||
@@ -0,0 +1,393 @@
|
||||
/**
|
||||
* src/quality/quality-service.ts
|
||||
* ===============================
|
||||
* 0pt's Data Quality capability (Master-DoD §10 Data quality). Owns:
|
||||
*
|
||||
* - the RULE REGISTRY (versioned; mutation bumps `version`, old versions
|
||||
* recorded as superseded — §10 rule versioning; products supply rules
|
||||
* via `supplied_by`),
|
||||
* - RUNS (execute all active rules for a dataset against supplied rows via
|
||||
* the local GE-subset engine, or DELEGATE to the GE backend in 0pt-motor
|
||||
* through an N0D job when any rule needs it),
|
||||
* - RESULTS / ISSUES (failed rules open issues for human review),
|
||||
* - SCORES + TREND (severity-weighted score per run; trend = score series).
|
||||
*
|
||||
* The GE delegation contract (§11 DQ): job_type "0pt.quality.ge_run" with the
|
||||
* GE suite JSON as input; the executing engine posts results back through
|
||||
* 0pt's API. This service creates the job and marks the run "delegated" —
|
||||
* it NEVER fabricates results for expectations it could not execute.
|
||||
*
|
||||
* Lineage: every finished run emits an OpenLineage COMPLETE/FAIL event
|
||||
* (inputs: the dataset; outputs: the run's issues/results identity).
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
||||
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
|
||||
import { LineageEmitter } from "../lineage/lineage-emitter";
|
||||
import { ExpectationEngine } from "./expectation-engine";
|
||||
import type {
|
||||
CreateQualityRuleInput,
|
||||
DataQualityIssue,
|
||||
DataQualityResult,
|
||||
DataQualityRule,
|
||||
DataQualityRun,
|
||||
QualityExpectation,
|
||||
QualityTrendPoint,
|
||||
} from "./quality-types";
|
||||
|
||||
const SEVERITY_WEIGHT: Record<DataQualityRule["severity"], number> = {
|
||||
info: 0.25,
|
||||
warning: 0.5,
|
||||
error: 1,
|
||||
critical: 2,
|
||||
};
|
||||
|
||||
export class QualityService {
|
||||
/** rule_id -> current rule. Superseded versions in ruleHistory. */
|
||||
private readonly rules = new Map<string, DataQualityRule>();
|
||||
private readonly ruleHistory = new Map<string, DataQualityRule[]>();
|
||||
private readonly runs = new Map<string, DataQualityRun>();
|
||||
private readonly issues = new Map<string, DataQualityIssue>();
|
||||
private readonly engine = new ExpectationEngine();
|
||||
private readonly aud0: Aud0Client;
|
||||
private readonly n0d: N0dClient;
|
||||
private readonly lineage: LineageEmitter;
|
||||
|
||||
constructor(opts?: { aud0BaseUrl?: string; n0dBaseUrl?: string }) {
|
||||
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
||||
this.n0d = createN0dClient(opts?.n0dBaseUrl);
|
||||
this.lineage = new LineageEmitter(this.aud0);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Rule registry (versioned)
|
||||
// -----------------------------------------------------------------------
|
||||
async createRule(input: CreateQualityRuleInput): Promise<DataQualityRule> {
|
||||
if (this.rules.has(input.rule_id)) {
|
||||
throw new SiaxError("CONFLICT", "0pt.quality", `rule ${input.rule_id} already exists — use updateRule`);
|
||||
}
|
||||
this.validateExpectationShape(input.expectation);
|
||||
const rule: DataQualityRule = {
|
||||
rule_id: input.rule_id,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
dataset_id: input.dataset_id,
|
||||
name: input.name,
|
||||
expectation: input.expectation,
|
||||
severity: input.severity ?? "error",
|
||||
version: 1,
|
||||
status: "active",
|
||||
supplied_by: input.supplied_by ?? "0pt",
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
this.rules.set(rule.rule_id, rule);
|
||||
await this.emitRuleEvent("0pt.quality_rule_created", rule);
|
||||
return rule;
|
||||
}
|
||||
|
||||
async updateRule(
|
||||
ruleId: string,
|
||||
tenantId: string,
|
||||
patch: Partial<Pick<DataQualityRule, "name" | "expectation" | "severity" | "status">>
|
||||
): Promise<DataQualityRule> {
|
||||
const current = this.getRule(ruleId, tenantId);
|
||||
if (patch.expectation) this.validateExpectationShape(patch.expectation);
|
||||
const history = this.ruleHistory.get(ruleId) ?? [];
|
||||
history.push({ ...current, status: "superseded" });
|
||||
this.ruleHistory.set(ruleId, history);
|
||||
const updated: DataQualityRule = {
|
||||
...current,
|
||||
...patch,
|
||||
version: current.version + 1,
|
||||
updated_at: new Date().toISOString(),
|
||||
};
|
||||
this.rules.set(ruleId, updated);
|
||||
await this.emitRuleEvent("0pt.quality_rule_updated", updated);
|
||||
return updated;
|
||||
}
|
||||
|
||||
getRule(ruleId: string, tenantId: string): DataQualityRule {
|
||||
const rule = this.rules.get(ruleId);
|
||||
// Cross-tenant probe → NOT_FOUND (no existence oracle).
|
||||
if (!rule || rule.tenant_id !== tenantId) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.quality", `rule ${ruleId} not found`);
|
||||
}
|
||||
return rule;
|
||||
}
|
||||
|
||||
getRuleVersions(ruleId: string, tenantId: string): DataQualityRule[] {
|
||||
const current = this.getRule(ruleId, tenantId);
|
||||
return [...(this.ruleHistory.get(ruleId) ?? []), current];
|
||||
}
|
||||
|
||||
listRules(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityRule["status"] }): DataQualityRule[] {
|
||||
if (!filter.tenant_id) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "tenant_id is required");
|
||||
}
|
||||
return [...this.rules.values()].filter(
|
||||
(r) =>
|
||||
r.tenant_id === filter.tenant_id &&
|
||||
(filter.dataset_id === undefined || r.dataset_id === filter.dataset_id) &&
|
||||
(filter.status === undefined || r.status === filter.status)
|
||||
);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Runs
|
||||
// -----------------------------------------------------------------------
|
||||
/**
|
||||
* Execute all active rules for a dataset against the supplied rows.
|
||||
* When every expectation is locally executable, the run completes
|
||||
* synchronously. When ANY rule needs the GE backend, the WHOLE run is
|
||||
* delegated (one engine per run keeps results comparable) — status
|
||||
* "delegated" + an N0D job id; results arrive via recordDelegatedResults.
|
||||
*/
|
||||
async runQuality(input: {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
dataset_id: string;
|
||||
dataset_version?: number;
|
||||
rows?: Record<string, unknown>[];
|
||||
correlation_id?: string;
|
||||
}): Promise<DataQualityRun> {
|
||||
const rules = this.listRules({
|
||||
tenant_id: input.tenant_id,
|
||||
dataset_id: input.dataset_id,
|
||||
status: "active",
|
||||
});
|
||||
if (rules.length === 0) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.quality", `no active rules for dataset ${input.dataset_id}`);
|
||||
}
|
||||
const runId = `dqrun_${crypto.randomUUID()}`;
|
||||
const startedAt = new Date().toISOString();
|
||||
|
||||
const allLocal = rules.every((r) => this.engine.supports(r.expectation.expectation_type));
|
||||
if (!allLocal || input.rows === undefined) {
|
||||
// DELEGATE to the GE backend (0pt-motor) via N0D — contract only here;
|
||||
// the run stays "delegated" until results are posted back. NEVER faked.
|
||||
const job = await this.n0d.createJob({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
job_type: "0pt.quality.ge_run",
|
||||
capability_id: "0pt.quality.run.v1",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: "svc_0pt",
|
||||
output_artifact_type: "quality_result",
|
||||
requires_backbone: false,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "standard",
|
||||
risk_level: "L2",
|
||||
correlation_id: input.correlation_id,
|
||||
idempotency_key: `dq:${input.dataset_id}:${input.dataset_version ?? "latest"}:${runId}`,
|
||||
});
|
||||
const run: DataQualityRun = {
|
||||
run_id: runId,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
dataset_id: input.dataset_id,
|
||||
dataset_version: input.dataset_version,
|
||||
status: "delegated",
|
||||
results: [],
|
||||
started_at: startedAt,
|
||||
correlation_id: input.correlation_id,
|
||||
delegated_job_id: job.job_id,
|
||||
};
|
||||
this.runs.set(runId, run);
|
||||
return run;
|
||||
}
|
||||
|
||||
const results: DataQualityResult[] = [];
|
||||
for (const rule of rules) {
|
||||
const outcome = this.engine.execute(rule.expectation, input.rows);
|
||||
results.push({
|
||||
rule_id: rule.rule_id,
|
||||
rule_version: rule.version,
|
||||
executed_by: "0pt-local",
|
||||
...outcome,
|
||||
});
|
||||
}
|
||||
const run = await this.finishRun(
|
||||
{
|
||||
run_id: runId,
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
dataset_id: input.dataset_id,
|
||||
dataset_version: input.dataset_version,
|
||||
status: "running",
|
||||
results,
|
||||
started_at: startedAt,
|
||||
correlation_id: input.correlation_id,
|
||||
},
|
||||
rules
|
||||
);
|
||||
return run;
|
||||
}
|
||||
|
||||
/** GE-backend callback path: attach delegated results and finish the run. */
|
||||
async recordDelegatedResults(
|
||||
runId: string,
|
||||
tenantId: string,
|
||||
results: DataQualityResult[]
|
||||
): Promise<DataQualityRun> {
|
||||
const run = this.getRun(runId, tenantId);
|
||||
if (run.status !== "delegated") {
|
||||
throw new SiaxError("CONFLICT", "0pt.quality", `run ${runId} is not awaiting delegated results`);
|
||||
}
|
||||
const rules = this.listRules({ tenant_id: tenantId, dataset_id: run.dataset_id });
|
||||
return this.finishRun({ ...run, results }, rules);
|
||||
}
|
||||
|
||||
getRun(runId: string, tenantId: string): DataQualityRun {
|
||||
const run = this.runs.get(runId);
|
||||
if (!run || run.tenant_id !== tenantId) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.quality", `run ${runId} not found`);
|
||||
}
|
||||
return run;
|
||||
}
|
||||
|
||||
listRuns(filter: { tenant_id: string; dataset_id?: string }): DataQualityRun[] {
|
||||
return [...this.runs.values()].filter(
|
||||
(r) => r.tenant_id === filter.tenant_id && (filter.dataset_id === undefined || r.dataset_id === filter.dataset_id)
|
||||
);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Issues / score / trend
|
||||
// -----------------------------------------------------------------------
|
||||
listIssues(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityIssue["status"] }): DataQualityIssue[] {
|
||||
return [...this.issues.values()].filter(
|
||||
(i) =>
|
||||
i.tenant_id === filter.tenant_id &&
|
||||
(filter.dataset_id === undefined || i.dataset_id === filter.dataset_id) &&
|
||||
(filter.status === undefined || i.status === filter.status)
|
||||
);
|
||||
}
|
||||
|
||||
async resolveIssue(issueId: string, tenantId: string, status: "resolved" | "wont_fix" | "acknowledged"): Promise<DataQualityIssue> {
|
||||
const issue = this.issues.get(issueId);
|
||||
if (!issue || issue.tenant_id !== tenantId) {
|
||||
throw new SiaxError("NOT_FOUND", "0pt.quality", `issue ${issueId} not found`);
|
||||
}
|
||||
issue.status = status;
|
||||
return issue;
|
||||
}
|
||||
|
||||
/** §10 trend: chronological score series for a dataset. */
|
||||
getTrend(tenantId: string, datasetId: string): QualityTrendPoint[] {
|
||||
return [...this.runs.values()]
|
||||
.filter((r) => r.tenant_id === tenantId && r.dataset_id === datasetId && r.score !== undefined && r.finished_at)
|
||||
.sort((a, b) => (a.finished_at! < b.finished_at! ? -1 : 1))
|
||||
.map((r) => ({ run_id: r.run_id, dataset_id: datasetId, score: r.score!, finished_at: r.finished_at! }));
|
||||
}
|
||||
|
||||
/** Severity-weighted score 0..100 over a run's results. */
|
||||
computeScore(results: DataQualityResult[], rules: DataQualityRule[]): number {
|
||||
if (results.length === 0) return 0;
|
||||
const byId = new Map(rules.map((r) => [r.rule_id, r]));
|
||||
let weightSum = 0;
|
||||
let weighted = 0;
|
||||
for (const res of results) {
|
||||
const weight = SEVERITY_WEIGHT[byId.get(res.rule_id)?.severity ?? "error"];
|
||||
weightSum += weight;
|
||||
weighted += weight * res.pass_ratio;
|
||||
}
|
||||
return Math.round((weighted / weightSum) * 10000) / 100;
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// internals
|
||||
// -----------------------------------------------------------------------
|
||||
private async finishRun(run: DataQualityRun, rules: DataQualityRule[]): Promise<DataQualityRun> {
|
||||
const failed = run.results.filter((r) => !r.success);
|
||||
const byId = new Map(rules.map((r) => [r.rule_id, r]));
|
||||
for (const res of failed) {
|
||||
const rule = byId.get(res.rule_id);
|
||||
const issue: DataQualityIssue = {
|
||||
issue_id: `dqiss_${crypto.randomUUID()}`,
|
||||
run_id: run.run_id,
|
||||
rule_id: res.rule_id,
|
||||
rule_version: res.rule_version,
|
||||
tenant_id: run.tenant_id,
|
||||
dataset_id: run.dataset_id,
|
||||
severity: rule?.severity ?? "error",
|
||||
status: "open",
|
||||
summary: `${rule?.name ?? res.rule_id}: ${res.unexpected_count}/${res.evaluated_count} unexpected`,
|
||||
created_at: new Date().toISOString(),
|
||||
};
|
||||
this.issues.set(issue.issue_id, issue);
|
||||
}
|
||||
|
||||
const finished: DataQualityRun = {
|
||||
...run,
|
||||
status: failed.length === 0 ? "succeeded" : "failed",
|
||||
score: this.computeScore(run.results, rules),
|
||||
finished_at: new Date().toISOString(),
|
||||
};
|
||||
this.runs.set(finished.run_id, finished);
|
||||
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_0pt_quality_run_${finished.run_id}`,
|
||||
event_type: "0pt.quality_run_finished",
|
||||
tenant_id: finished.tenant_id,
|
||||
project_id: finished.project_id,
|
||||
actor_id: "svc_0pt",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.quality.run.v1",
|
||||
resource_type: "quality_run",
|
||||
resource_id: finished.run_id,
|
||||
risk_level: "L2",
|
||||
result: finished.status === "succeeded" ? "success" : "failure",
|
||||
payload: { score: finished.score, failed_rules: failed.length, total_rules: finished.results.length },
|
||||
created_at: finished.finished_at!,
|
||||
correlation_id: finished.correlation_id,
|
||||
input_refs: [`0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`],
|
||||
output_refs: [`0pt:quality_run:${finished.run_id}`],
|
||||
});
|
||||
await this.lineage.emit({
|
||||
tenant_id: finished.tenant_id,
|
||||
project_id: finished.project_id,
|
||||
capability_id: "0pt.quality.run.v1",
|
||||
run_id: finished.run_id,
|
||||
event_type: finished.status === "succeeded" ? "COMPLETE" : "FAIL",
|
||||
inputs: [
|
||||
{
|
||||
name: `dataset/${finished.dataset_id}`,
|
||||
ref: `0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`,
|
||||
},
|
||||
],
|
||||
outputs: [{ name: `quality_run/${finished.run_id}`, ref: `0pt:quality_run:${finished.run_id}` }],
|
||||
correlation_id: finished.correlation_id,
|
||||
run_facets: { siaxQuality: { score: finished.score, failed_rules: failed.length } },
|
||||
});
|
||||
|
||||
return finished;
|
||||
}
|
||||
|
||||
private validateExpectationShape(expectation: QualityExpectation): void {
|
||||
if (!expectation.expectation_type?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation_type is required");
|
||||
}
|
||||
if (expectation.kwargs === null || typeof expectation.kwargs !== "object") {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation.kwargs must be an object");
|
||||
}
|
||||
}
|
||||
|
||||
private async emitRuleEvent(eventType: string, rule: DataQualityRule): Promise<void> {
|
||||
await this.aud0.ingestEvent({
|
||||
event_id: `evt_${eventType}_${rule.rule_id}_v${rule.version}`,
|
||||
event_type: eventType,
|
||||
tenant_id: rule.tenant_id,
|
||||
project_id: rule.project_id,
|
||||
actor_id: "svc_0pt",
|
||||
app_id: "0pt",
|
||||
capability_id: "0pt.quality.rule.v1",
|
||||
resource_type: "quality_rule",
|
||||
resource_id: rule.rule_id,
|
||||
risk_level: "L1",
|
||||
payload: { version: rule.version, severity: rule.severity, supplied_by: rule.supplied_by },
|
||||
created_at: new Date().toISOString(),
|
||||
});
|
||||
}
|
||||
}
|
||||
@@ -0,0 +1,114 @@
|
||||
/**
|
||||
* src/quality/quality-types.ts
|
||||
* =============================
|
||||
* Types for 0pt's Data Quality capability (Master-DoD §10 Data quality:
|
||||
* rule registry, runs, results, issues, scores, trend, rule versioning,
|
||||
* product-supplied rules).
|
||||
*
|
||||
* IMPORTANT HISTORY: this repo once carried FABRICATED claims that a
|
||||
* quality-check service existed (see CLAUDE.md + docs/P0_DATASET_QUALITY_FLOW.md,
|
||||
* which is the explicit target spec). THIS module is the first real
|
||||
* implementation — anything it cannot do is stated, not implied.
|
||||
*
|
||||
* Expectations are serialized in Great Expectations' canonical JSON shape
|
||||
* (`expectation_type` + `kwargs`) so a rule authored here can be executed
|
||||
* verbatim by a GE backend (0pt-motor's Python engine, Master-DoD §11 DQ)
|
||||
* AND by the local TypeScript engine for the supported core subset. The GE
|
||||
* suite is the interchange format; the local engine is honest about which
|
||||
* expectation types it supports (unsupported ⇒ must delegate, never fake).
|
||||
*/
|
||||
|
||||
/** GE-compatible expectation (interchange shape). */
|
||||
export interface QualityExpectation {
|
||||
/** Great Expectations expectation type, e.g. "expect_column_values_to_not_be_null". */
|
||||
expectation_type: string;
|
||||
kwargs: Record<string, unknown>;
|
||||
}
|
||||
|
||||
export type QualitySeverity = "info" | "warning" | "error" | "critical";
|
||||
|
||||
export interface DataQualityRule {
|
||||
rule_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
/** Dataset the rule targets; product-supplied rules set their own dataset ids. */
|
||||
dataset_id: string;
|
||||
name: string;
|
||||
expectation: QualityExpectation;
|
||||
severity: QualitySeverity;
|
||||
/** Monotonic rule version — bumped on every mutation (§10 rule versioning). */
|
||||
version: number;
|
||||
status: "active" | "disabled" | "superseded";
|
||||
/** Which product supplied the rule ("0pt" | "vehiq" | ...). */
|
||||
supplied_by: string;
|
||||
created_at: string;
|
||||
updated_at?: string;
|
||||
}
|
||||
|
||||
export interface CreateQualityRuleInput {
|
||||
rule_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
dataset_id: string;
|
||||
name: string;
|
||||
expectation: QualityExpectation;
|
||||
severity?: QualitySeverity;
|
||||
supplied_by?: string;
|
||||
}
|
||||
|
||||
/** One executed expectation within a run. */
|
||||
export interface DataQualityResult {
|
||||
rule_id: string;
|
||||
rule_version: number;
|
||||
success: boolean;
|
||||
/** Fraction of evaluated rows/elements that passed (1 for table-level checks that pass). */
|
||||
pass_ratio: number;
|
||||
evaluated_count: number;
|
||||
unexpected_count: number;
|
||||
/** Up to 20 sample unexpected values (never full rows — PII minimization). */
|
||||
unexpected_samples: unknown[];
|
||||
/** Engine that executed the expectation. */
|
||||
executed_by: "0pt-local" | "great_expectations";
|
||||
}
|
||||
|
||||
/** An issue opened for a failed rule (human-review handoff). */
|
||||
export interface DataQualityIssue {
|
||||
issue_id: string;
|
||||
run_id: string;
|
||||
rule_id: string;
|
||||
rule_version: number;
|
||||
tenant_id: string;
|
||||
dataset_id: string;
|
||||
severity: QualitySeverity;
|
||||
status: "open" | "acknowledged" | "resolved" | "wont_fix";
|
||||
summary: string;
|
||||
created_at: string;
|
||||
}
|
||||
|
||||
export type QualityRunStatus = "running" | "succeeded" | "failed" | "delegated";
|
||||
|
||||
export interface DataQualityRun {
|
||||
run_id: string;
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
dataset_id: string;
|
||||
/** Dataset version under test, when versioned. */
|
||||
dataset_version?: number;
|
||||
status: QualityRunStatus;
|
||||
results: DataQualityResult[];
|
||||
/** Overall score 0..100 (severity-weighted pass ratio). */
|
||||
score?: number;
|
||||
started_at: string;
|
||||
finished_at?: string;
|
||||
correlation_id?: string;
|
||||
/** N0D job id when execution was delegated to the GE backend. */
|
||||
delegated_job_id?: string;
|
||||
}
|
||||
|
||||
/** One point of the §10 trend series. */
|
||||
export interface QualityTrendPoint {
|
||||
run_id: string;
|
||||
dataset_id: string;
|
||||
score: number;
|
||||
finished_at: string;
|
||||
}
|
||||
Reference in New Issue
Block a user