Files
0pt/src/quality/quality-service.ts
T
admin 2d34cb1629
CI (SIAX Cloud) / security (pull_request) Successful in 16s
CI (SIAX Cloud) / security (push) Successful in 16s
Security Scan (SIAX Cloud) / security (push) Successful in 17s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 23s
CI (SIAX Cloud) / integration (push) Failing after 27s
CI (SIAX Cloud) / integration (pull_request) Failing after 31s
CI (SIAX Cloud) / quality (pull_request) Successful in 42s
CI (SIAX Cloud) / quality (push) Successful in 42s
feat(P-001, P-002): 0pt→N0D quality delegation + internal OpenAPI
2026-09-08 22:38:47 +02:00

386 lines
15 KiB
TypeScript

/**
* src/quality/quality-service.ts
* ===============================
* 0pt's Data Quality capability (Master-DoD §10 Data quality). Owns:
*
* - the RULE REGISTRY (versioned; mutation bumps `version`, old versions
* recorded as superseded — §10 rule versioning; products supply rules
* via `supplied_by`),
* - RUNS (execute all active rules for a dataset against supplied rows via
* the local GE-subset engine, or DELEGATE to the GE backend in 0pt-motor
* through an N0D job when any rule needs it),
* - RESULTS / ISSUES (failed rules open issues for human review),
* - SCORES + TREND (severity-weighted score per run; trend = score series).
*
* The GE delegation contract (§11 DQ): job_type "0pt.quality.ge_run" with the
* GE suite JSON as input; the executing engine posts results back through
* 0pt's API. This service creates the job and marks the run "delegated" —
* it NEVER fabricates results for expectations it could not execute.
*
* Lineage: every finished run emits an OpenLineage COMPLETE/FAIL event
* (inputs: the dataset; outputs: the run's issues/results identity).
*/
import crypto from "crypto";
import { SiaxError } from "../platform/errors";
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
import { LineageEmitter } from "../lineage/lineage-emitter";
import { ExpectationEngine } from "./expectation-engine";
import type {
CreateQualityRuleInput,
DataQualityIssue,
DataQualityResult,
DataQualityRule,
DataQualityRun,
QualityExpectation,
QualityTrendPoint,
} from "./quality-types";
const SEVERITY_WEIGHT: Record<DataQualityRule["severity"], number> = {
info: 0.25,
warning: 0.5,
error: 1,
critical: 2,
};
export class QualityService {
/** rule_id -> current rule. Superseded versions in ruleHistory. */
private readonly rules = new Map<string, DataQualityRule>();
private readonly ruleHistory = new Map<string, DataQualityRule[]>();
private readonly runs = new Map<string, DataQualityRun>();
private readonly issues = new Map<string, DataQualityIssue>();
private readonly engine = new ExpectationEngine();
private readonly aud0: Aud0Client;
private readonly n0d: N0dClient;
private readonly lineage: LineageEmitter;
constructor(opts?: { aud0BaseUrl?: string; n0dBaseUrl?: string }) {
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
this.n0d = createN0dClient(opts?.n0dBaseUrl);
this.lineage = new LineageEmitter(this.aud0);
}
// -----------------------------------------------------------------------
// Rule registry (versioned)
// -----------------------------------------------------------------------
async createRule(input: CreateQualityRuleInput): Promise<DataQualityRule> {
if (this.rules.has(input.rule_id)) {
throw new SiaxError("CONFLICT", "0pt.quality", `rule ${input.rule_id} already exists — use updateRule`);
}
this.validateExpectationShape(input.expectation);
const rule: DataQualityRule = {
rule_id: input.rule_id,
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
name: input.name,
expectation: input.expectation,
severity: input.severity ?? "error",
version: 1,
status: "active",
supplied_by: input.supplied_by ?? "0pt",
created_at: new Date().toISOString(),
};
this.rules.set(rule.rule_id, rule);
await this.emitRuleEvent("0pt.quality_rule_created", rule);
return rule;
}
async updateRule(
ruleId: string,
tenantId: string,
patch: Partial<Pick<DataQualityRule, "name" | "expectation" | "severity" | "status">>
): Promise<DataQualityRule> {
const current = this.getRule(ruleId, tenantId);
if (patch.expectation) this.validateExpectationShape(patch.expectation);
const history = this.ruleHistory.get(ruleId) ?? [];
history.push({ ...current, status: "superseded" });
this.ruleHistory.set(ruleId, history);
const updated: DataQualityRule = {
...current,
...patch,
version: current.version + 1,
updated_at: new Date().toISOString(),
};
this.rules.set(ruleId, updated);
await this.emitRuleEvent("0pt.quality_rule_updated", updated);
return updated;
}
getRule(ruleId: string, tenantId: string): DataQualityRule {
const rule = this.rules.get(ruleId);
// Cross-tenant probe → NOT_FOUND (no existence oracle).
if (!rule || rule.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.quality", `rule ${ruleId} not found`);
}
return rule;
}
getRuleVersions(ruleId: string, tenantId: string): DataQualityRule[] {
const current = this.getRule(ruleId, tenantId);
return [...(this.ruleHistory.get(ruleId) ?? []), current];
}
listRules(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityRule["status"] }): DataQualityRule[] {
if (!filter.tenant_id) {
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "tenant_id is required");
}
return [...this.rules.values()].filter(
(r) =>
r.tenant_id === filter.tenant_id &&
(filter.dataset_id === undefined || r.dataset_id === filter.dataset_id) &&
(filter.status === undefined || r.status === filter.status)
);
}
// -----------------------------------------------------------------------
// Runs
// -----------------------------------------------------------------------
/**
* Execute all active rules for a dataset against the supplied rows.
* When every expectation is locally executable, the run completes
* synchronously. When ANY rule needs the GE backend, the WHOLE run is
* delegated (one engine per run keeps results comparable) — status
* "delegated" + an N0D job id; results arrive via recordDelegatedResults.
*/
async runQuality(input: {
tenant_id: string;
project_id: string;
dataset_id: string;
dataset_version?: number;
rows?: Record<string, unknown>[];
correlation_id?: string;
}): Promise<DataQualityRun> {
const rules = this.listRules({
tenant_id: input.tenant_id,
dataset_id: input.dataset_id,
status: "active",
});
if (rules.length === 0) {
throw new SiaxError("NOT_FOUND", "0pt.quality", `no active rules for dataset ${input.dataset_id}`);
}
const runId = `dqrun_${crypto.randomUUID()}`;
const startedAt = new Date().toISOString();
const allLocal = rules.every((r) => this.engine.supports(r.expectation.expectation_type));
if (!allLocal || input.rows === undefined) {
// DELEGATE to the GE backend (0pt-motor) via N0D — owner-seam:
// n0dRunDataQuality (POST /v1/n0d/quality/run), NOT the generic
// createJob. N0D OWNS quality execution; 0pt only requests it.
const job = await this.n0d.runDataQuality({
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
quality_rule_set_id: input.dataset_id,
idempotency_key: `dq:${input.dataset_id}:${input.dataset_version ?? "latest"}:${runId}`,
});
const run: DataQualityRun = {
run_id: runId,
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
dataset_version: input.dataset_version,
status: "delegated",
results: [],
started_at: startedAt,
correlation_id: input.correlation_id,
delegated_job_id: job.job_id,
};
this.runs.set(runId, run);
return run;
}
const results: DataQualityResult[] = [];
for (const rule of rules) {
const outcome = this.engine.execute(rule.expectation, input.rows);
results.push({
rule_id: rule.rule_id,
rule_version: rule.version,
executed_by: "0pt-local",
...outcome,
});
}
const run = await this.finishRun(
{
run_id: runId,
tenant_id: input.tenant_id,
project_id: input.project_id,
dataset_id: input.dataset_id,
dataset_version: input.dataset_version,
status: "running",
results,
started_at: startedAt,
correlation_id: input.correlation_id,
},
rules
);
return run;
}
/** GE-backend callback path: attach delegated results and finish the run. */
async recordDelegatedResults(
runId: string,
tenantId: string,
results: DataQualityResult[]
): Promise<DataQualityRun> {
const run = this.getRun(runId, tenantId);
if (run.status !== "delegated") {
throw new SiaxError("CONFLICT", "0pt.quality", `run ${runId} is not awaiting delegated results`);
}
const rules = this.listRules({ tenant_id: tenantId, dataset_id: run.dataset_id });
return this.finishRun({ ...run, results }, rules);
}
getRun(runId: string, tenantId: string): DataQualityRun {
const run = this.runs.get(runId);
if (!run || run.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.quality", `run ${runId} not found`);
}
return run;
}
listRuns(filter: { tenant_id: string; dataset_id?: string }): DataQualityRun[] {
return [...this.runs.values()].filter(
(r) => r.tenant_id === filter.tenant_id && (filter.dataset_id === undefined || r.dataset_id === filter.dataset_id)
);
}
// -----------------------------------------------------------------------
// Issues / score / trend
// -----------------------------------------------------------------------
listIssues(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityIssue["status"] }): DataQualityIssue[] {
return [...this.issues.values()].filter(
(i) =>
i.tenant_id === filter.tenant_id &&
(filter.dataset_id === undefined || i.dataset_id === filter.dataset_id) &&
(filter.status === undefined || i.status === filter.status)
);
}
async resolveIssue(issueId: string, tenantId: string, status: "resolved" | "wont_fix" | "acknowledged"): Promise<DataQualityIssue> {
const issue = this.issues.get(issueId);
if (!issue || issue.tenant_id !== tenantId) {
throw new SiaxError("NOT_FOUND", "0pt.quality", `issue ${issueId} not found`);
}
issue.status = status;
return issue;
}
/** §10 trend: chronological score series for a dataset. */
getTrend(tenantId: string, datasetId: string): QualityTrendPoint[] {
return [...this.runs.values()]
.filter((r) => r.tenant_id === tenantId && r.dataset_id === datasetId && r.score !== undefined && r.finished_at)
.sort((a, b) => (a.finished_at! < b.finished_at! ? -1 : 1))
.map((r) => ({ run_id: r.run_id, dataset_id: datasetId, score: r.score!, finished_at: r.finished_at! }));
}
/** Severity-weighted score 0..100 over a run's results. */
computeScore(results: DataQualityResult[], rules: DataQualityRule[]): number {
if (results.length === 0) return 0;
const byId = new Map(rules.map((r) => [r.rule_id, r]));
let weightSum = 0;
let weighted = 0;
for (const res of results) {
const weight = SEVERITY_WEIGHT[byId.get(res.rule_id)?.severity ?? "error"];
weightSum += weight;
weighted += weight * res.pass_ratio;
}
return Math.round((weighted / weightSum) * 10000) / 100;
}
// -----------------------------------------------------------------------
// internals
// -----------------------------------------------------------------------
private async finishRun(run: DataQualityRun, rules: DataQualityRule[]): Promise<DataQualityRun> {
const failed = run.results.filter((r) => !r.success);
const byId = new Map(rules.map((r) => [r.rule_id, r]));
for (const res of failed) {
const rule = byId.get(res.rule_id);
const issue: DataQualityIssue = {
issue_id: `dqiss_${crypto.randomUUID()}`,
run_id: run.run_id,
rule_id: res.rule_id,
rule_version: res.rule_version,
tenant_id: run.tenant_id,
dataset_id: run.dataset_id,
severity: rule?.severity ?? "error",
status: "open",
summary: `${rule?.name ?? res.rule_id}: ${res.unexpected_count}/${res.evaluated_count} unexpected`,
created_at: new Date().toISOString(),
};
this.issues.set(issue.issue_id, issue);
}
const finished: DataQualityRun = {
...run,
status: failed.length === 0 ? "succeeded" : "failed",
score: this.computeScore(run.results, rules),
finished_at: new Date().toISOString(),
};
this.runs.set(finished.run_id, finished);
await this.aud0.ingestEvent({
event_id: `evt_0pt_quality_run_${finished.run_id}`,
event_type: "0pt.quality_run_finished",
tenant_id: finished.tenant_id,
project_id: finished.project_id,
actor_id: "svc_0pt",
app_id: "0pt",
capability_id: "0pt.quality.run.v1",
resource_type: "quality_run",
resource_id: finished.run_id,
risk_level: "L2",
result: finished.status === "succeeded" ? "success" : "failure",
payload: { score: finished.score, failed_rules: failed.length, total_rules: finished.results.length },
created_at: finished.finished_at!,
correlation_id: finished.correlation_id,
input_refs: [`0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`],
output_refs: [`0pt:quality_run:${finished.run_id}`],
});
await this.lineage.emit({
tenant_id: finished.tenant_id,
project_id: finished.project_id,
capability_id: "0pt.quality.run.v1",
run_id: finished.run_id,
event_type: finished.status === "succeeded" ? "COMPLETE" : "FAIL",
inputs: [
{
name: `dataset/${finished.dataset_id}`,
ref: `0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`,
},
],
outputs: [{ name: `quality_run/${finished.run_id}`, ref: `0pt:quality_run:${finished.run_id}` }],
correlation_id: finished.correlation_id,
run_facets: { siaxQuality: { score: finished.score, failed_rules: failed.length } },
});
return finished;
}
private validateExpectationShape(expectation: QualityExpectation): void {
if (!expectation.expectation_type?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation_type is required");
}
if (expectation.kwargs === null || typeof expectation.kwargs !== "object") {
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation.kwargs must be an object");
}
}
private async emitRuleEvent(eventType: string, rule: DataQualityRule): Promise<void> {
await this.aud0.ingestEvent({
event_id: `evt_${eventType}_${rule.rule_id}_v${rule.version}`,
event_type: eventType,
tenant_id: rule.tenant_id,
project_id: rule.project_id,
actor_id: "svc_0pt",
app_id: "0pt",
capability_id: "0pt.quality.rule.v1",
resource_type: "quality_rule",
resource_id: rule.rule_id,
risk_level: "L1",
payload: { version: rule.version, severity: rule.severity, supplied_by: rule.supplied_by },
created_at: new Date().toISOString(),
});
}
}