CI (SIAX Cloud) / security (pull_request) Successful in 16s
CI (SIAX Cloud) / security (push) Successful in 16s
Security Scan (SIAX Cloud) / security (push) Successful in 17s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 23s
CI (SIAX Cloud) / integration (push) Failing after 27s
CI (SIAX Cloud) / integration (pull_request) Failing after 31s
CI (SIAX Cloud) / quality (pull_request) Successful in 42s
CI (SIAX Cloud) / quality (push) Successful in 42s
386 lines
15 KiB
TypeScript
386 lines
15 KiB
TypeScript
/**
|
|
* src/quality/quality-service.ts
|
|
* ===============================
|
|
* 0pt's Data Quality capability (Master-DoD §10 Data quality). Owns:
|
|
*
|
|
* - the RULE REGISTRY (versioned; mutation bumps `version`, old versions
|
|
* recorded as superseded — §10 rule versioning; products supply rules
|
|
* via `supplied_by`),
|
|
* - RUNS (execute all active rules for a dataset against supplied rows via
|
|
* the local GE-subset engine, or DELEGATE to the GE backend in 0pt-motor
|
|
* through an N0D job when any rule needs it),
|
|
* - RESULTS / ISSUES (failed rules open issues for human review),
|
|
* - SCORES + TREND (severity-weighted score per run; trend = score series).
|
|
*
|
|
* The GE delegation contract (§11 DQ): job_type "0pt.quality.ge_run" with the
|
|
* GE suite JSON as input; the executing engine posts results back through
|
|
* 0pt's API. This service creates the job and marks the run "delegated" —
|
|
* it NEVER fabricates results for expectations it could not execute.
|
|
*
|
|
* Lineage: every finished run emits an OpenLineage COMPLETE/FAIL event
|
|
* (inputs: the dataset; outputs: the run's issues/results identity).
|
|
*/
|
|
import crypto from "crypto";
|
|
import { SiaxError } from "../platform/errors";
|
|
import { createAud0Client, type Aud0Client } from "../clients/aud0-client";
|
|
import { createN0dClient, type N0dClient } from "../clients/n0d-client";
|
|
import { LineageEmitter } from "../lineage/lineage-emitter";
|
|
import { ExpectationEngine } from "./expectation-engine";
|
|
import type {
|
|
CreateQualityRuleInput,
|
|
DataQualityIssue,
|
|
DataQualityResult,
|
|
DataQualityRule,
|
|
DataQualityRun,
|
|
QualityExpectation,
|
|
QualityTrendPoint,
|
|
} from "./quality-types";
|
|
|
|
const SEVERITY_WEIGHT: Record<DataQualityRule["severity"], number> = {
|
|
info: 0.25,
|
|
warning: 0.5,
|
|
error: 1,
|
|
critical: 2,
|
|
};
|
|
|
|
export class QualityService {
|
|
/** rule_id -> current rule. Superseded versions in ruleHistory. */
|
|
private readonly rules = new Map<string, DataQualityRule>();
|
|
private readonly ruleHistory = new Map<string, DataQualityRule[]>();
|
|
private readonly runs = new Map<string, DataQualityRun>();
|
|
private readonly issues = new Map<string, DataQualityIssue>();
|
|
private readonly engine = new ExpectationEngine();
|
|
private readonly aud0: Aud0Client;
|
|
private readonly n0d: N0dClient;
|
|
private readonly lineage: LineageEmitter;
|
|
|
|
constructor(opts?: { aud0BaseUrl?: string; n0dBaseUrl?: string }) {
|
|
this.aud0 = createAud0Client(opts?.aud0BaseUrl);
|
|
this.n0d = createN0dClient(opts?.n0dBaseUrl);
|
|
this.lineage = new LineageEmitter(this.aud0);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Rule registry (versioned)
|
|
// -----------------------------------------------------------------------
|
|
async createRule(input: CreateQualityRuleInput): Promise<DataQualityRule> {
|
|
if (this.rules.has(input.rule_id)) {
|
|
throw new SiaxError("CONFLICT", "0pt.quality", `rule ${input.rule_id} already exists — use updateRule`);
|
|
}
|
|
this.validateExpectationShape(input.expectation);
|
|
const rule: DataQualityRule = {
|
|
rule_id: input.rule_id,
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
dataset_id: input.dataset_id,
|
|
name: input.name,
|
|
expectation: input.expectation,
|
|
severity: input.severity ?? "error",
|
|
version: 1,
|
|
status: "active",
|
|
supplied_by: input.supplied_by ?? "0pt",
|
|
created_at: new Date().toISOString(),
|
|
};
|
|
this.rules.set(rule.rule_id, rule);
|
|
await this.emitRuleEvent("0pt.quality_rule_created", rule);
|
|
return rule;
|
|
}
|
|
|
|
async updateRule(
|
|
ruleId: string,
|
|
tenantId: string,
|
|
patch: Partial<Pick<DataQualityRule, "name" | "expectation" | "severity" | "status">>
|
|
): Promise<DataQualityRule> {
|
|
const current = this.getRule(ruleId, tenantId);
|
|
if (patch.expectation) this.validateExpectationShape(patch.expectation);
|
|
const history = this.ruleHistory.get(ruleId) ?? [];
|
|
history.push({ ...current, status: "superseded" });
|
|
this.ruleHistory.set(ruleId, history);
|
|
const updated: DataQualityRule = {
|
|
...current,
|
|
...patch,
|
|
version: current.version + 1,
|
|
updated_at: new Date().toISOString(),
|
|
};
|
|
this.rules.set(ruleId, updated);
|
|
await this.emitRuleEvent("0pt.quality_rule_updated", updated);
|
|
return updated;
|
|
}
|
|
|
|
getRule(ruleId: string, tenantId: string): DataQualityRule {
|
|
const rule = this.rules.get(ruleId);
|
|
// Cross-tenant probe → NOT_FOUND (no existence oracle).
|
|
if (!rule || rule.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.quality", `rule ${ruleId} not found`);
|
|
}
|
|
return rule;
|
|
}
|
|
|
|
getRuleVersions(ruleId: string, tenantId: string): DataQualityRule[] {
|
|
const current = this.getRule(ruleId, tenantId);
|
|
return [...(this.ruleHistory.get(ruleId) ?? []), current];
|
|
}
|
|
|
|
listRules(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityRule["status"] }): DataQualityRule[] {
|
|
if (!filter.tenant_id) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "tenant_id is required");
|
|
}
|
|
return [...this.rules.values()].filter(
|
|
(r) =>
|
|
r.tenant_id === filter.tenant_id &&
|
|
(filter.dataset_id === undefined || r.dataset_id === filter.dataset_id) &&
|
|
(filter.status === undefined || r.status === filter.status)
|
|
);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Runs
|
|
// -----------------------------------------------------------------------
|
|
/**
|
|
* Execute all active rules for a dataset against the supplied rows.
|
|
* When every expectation is locally executable, the run completes
|
|
* synchronously. When ANY rule needs the GE backend, the WHOLE run is
|
|
* delegated (one engine per run keeps results comparable) — status
|
|
* "delegated" + an N0D job id; results arrive via recordDelegatedResults.
|
|
*/
|
|
async runQuality(input: {
|
|
tenant_id: string;
|
|
project_id: string;
|
|
dataset_id: string;
|
|
dataset_version?: number;
|
|
rows?: Record<string, unknown>[];
|
|
correlation_id?: string;
|
|
}): Promise<DataQualityRun> {
|
|
const rules = this.listRules({
|
|
tenant_id: input.tenant_id,
|
|
dataset_id: input.dataset_id,
|
|
status: "active",
|
|
});
|
|
if (rules.length === 0) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.quality", `no active rules for dataset ${input.dataset_id}`);
|
|
}
|
|
const runId = `dqrun_${crypto.randomUUID()}`;
|
|
const startedAt = new Date().toISOString();
|
|
|
|
const allLocal = rules.every((r) => this.engine.supports(r.expectation.expectation_type));
|
|
if (!allLocal || input.rows === undefined) {
|
|
// DELEGATE to the GE backend (0pt-motor) via N0D — owner-seam:
|
|
// n0dRunDataQuality (POST /v1/n0d/quality/run), NOT the generic
|
|
// createJob. N0D OWNS quality execution; 0pt only requests it.
|
|
const job = await this.n0d.runDataQuality({
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
dataset_id: input.dataset_id,
|
|
quality_rule_set_id: input.dataset_id,
|
|
idempotency_key: `dq:${input.dataset_id}:${input.dataset_version ?? "latest"}:${runId}`,
|
|
});
|
|
const run: DataQualityRun = {
|
|
run_id: runId,
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
dataset_id: input.dataset_id,
|
|
dataset_version: input.dataset_version,
|
|
status: "delegated",
|
|
results: [],
|
|
started_at: startedAt,
|
|
correlation_id: input.correlation_id,
|
|
delegated_job_id: job.job_id,
|
|
};
|
|
this.runs.set(runId, run);
|
|
return run;
|
|
}
|
|
|
|
const results: DataQualityResult[] = [];
|
|
for (const rule of rules) {
|
|
const outcome = this.engine.execute(rule.expectation, input.rows);
|
|
results.push({
|
|
rule_id: rule.rule_id,
|
|
rule_version: rule.version,
|
|
executed_by: "0pt-local",
|
|
...outcome,
|
|
});
|
|
}
|
|
const run = await this.finishRun(
|
|
{
|
|
run_id: runId,
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
dataset_id: input.dataset_id,
|
|
dataset_version: input.dataset_version,
|
|
status: "running",
|
|
results,
|
|
started_at: startedAt,
|
|
correlation_id: input.correlation_id,
|
|
},
|
|
rules
|
|
);
|
|
return run;
|
|
}
|
|
|
|
/** GE-backend callback path: attach delegated results and finish the run. */
|
|
async recordDelegatedResults(
|
|
runId: string,
|
|
tenantId: string,
|
|
results: DataQualityResult[]
|
|
): Promise<DataQualityRun> {
|
|
const run = this.getRun(runId, tenantId);
|
|
if (run.status !== "delegated") {
|
|
throw new SiaxError("CONFLICT", "0pt.quality", `run ${runId} is not awaiting delegated results`);
|
|
}
|
|
const rules = this.listRules({ tenant_id: tenantId, dataset_id: run.dataset_id });
|
|
return this.finishRun({ ...run, results }, rules);
|
|
}
|
|
|
|
getRun(runId: string, tenantId: string): DataQualityRun {
|
|
const run = this.runs.get(runId);
|
|
if (!run || run.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.quality", `run ${runId} not found`);
|
|
}
|
|
return run;
|
|
}
|
|
|
|
listRuns(filter: { tenant_id: string; dataset_id?: string }): DataQualityRun[] {
|
|
return [...this.runs.values()].filter(
|
|
(r) => r.tenant_id === filter.tenant_id && (filter.dataset_id === undefined || r.dataset_id === filter.dataset_id)
|
|
);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Issues / score / trend
|
|
// -----------------------------------------------------------------------
|
|
listIssues(filter: { tenant_id: string; dataset_id?: string; status?: DataQualityIssue["status"] }): DataQualityIssue[] {
|
|
return [...this.issues.values()].filter(
|
|
(i) =>
|
|
i.tenant_id === filter.tenant_id &&
|
|
(filter.dataset_id === undefined || i.dataset_id === filter.dataset_id) &&
|
|
(filter.status === undefined || i.status === filter.status)
|
|
);
|
|
}
|
|
|
|
async resolveIssue(issueId: string, tenantId: string, status: "resolved" | "wont_fix" | "acknowledged"): Promise<DataQualityIssue> {
|
|
const issue = this.issues.get(issueId);
|
|
if (!issue || issue.tenant_id !== tenantId) {
|
|
throw new SiaxError("NOT_FOUND", "0pt.quality", `issue ${issueId} not found`);
|
|
}
|
|
issue.status = status;
|
|
return issue;
|
|
}
|
|
|
|
/** §10 trend: chronological score series for a dataset. */
|
|
getTrend(tenantId: string, datasetId: string): QualityTrendPoint[] {
|
|
return [...this.runs.values()]
|
|
.filter((r) => r.tenant_id === tenantId && r.dataset_id === datasetId && r.score !== undefined && r.finished_at)
|
|
.sort((a, b) => (a.finished_at! < b.finished_at! ? -1 : 1))
|
|
.map((r) => ({ run_id: r.run_id, dataset_id: datasetId, score: r.score!, finished_at: r.finished_at! }));
|
|
}
|
|
|
|
/** Severity-weighted score 0..100 over a run's results. */
|
|
computeScore(results: DataQualityResult[], rules: DataQualityRule[]): number {
|
|
if (results.length === 0) return 0;
|
|
const byId = new Map(rules.map((r) => [r.rule_id, r]));
|
|
let weightSum = 0;
|
|
let weighted = 0;
|
|
for (const res of results) {
|
|
const weight = SEVERITY_WEIGHT[byId.get(res.rule_id)?.severity ?? "error"];
|
|
weightSum += weight;
|
|
weighted += weight * res.pass_ratio;
|
|
}
|
|
return Math.round((weighted / weightSum) * 10000) / 100;
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// internals
|
|
// -----------------------------------------------------------------------
|
|
private async finishRun(run: DataQualityRun, rules: DataQualityRule[]): Promise<DataQualityRun> {
|
|
const failed = run.results.filter((r) => !r.success);
|
|
const byId = new Map(rules.map((r) => [r.rule_id, r]));
|
|
for (const res of failed) {
|
|
const rule = byId.get(res.rule_id);
|
|
const issue: DataQualityIssue = {
|
|
issue_id: `dqiss_${crypto.randomUUID()}`,
|
|
run_id: run.run_id,
|
|
rule_id: res.rule_id,
|
|
rule_version: res.rule_version,
|
|
tenant_id: run.tenant_id,
|
|
dataset_id: run.dataset_id,
|
|
severity: rule?.severity ?? "error",
|
|
status: "open",
|
|
summary: `${rule?.name ?? res.rule_id}: ${res.unexpected_count}/${res.evaluated_count} unexpected`,
|
|
created_at: new Date().toISOString(),
|
|
};
|
|
this.issues.set(issue.issue_id, issue);
|
|
}
|
|
|
|
const finished: DataQualityRun = {
|
|
...run,
|
|
status: failed.length === 0 ? "succeeded" : "failed",
|
|
score: this.computeScore(run.results, rules),
|
|
finished_at: new Date().toISOString(),
|
|
};
|
|
this.runs.set(finished.run_id, finished);
|
|
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_0pt_quality_run_${finished.run_id}`,
|
|
event_type: "0pt.quality_run_finished",
|
|
tenant_id: finished.tenant_id,
|
|
project_id: finished.project_id,
|
|
actor_id: "svc_0pt",
|
|
app_id: "0pt",
|
|
capability_id: "0pt.quality.run.v1",
|
|
resource_type: "quality_run",
|
|
resource_id: finished.run_id,
|
|
risk_level: "L2",
|
|
result: finished.status === "succeeded" ? "success" : "failure",
|
|
payload: { score: finished.score, failed_rules: failed.length, total_rules: finished.results.length },
|
|
created_at: finished.finished_at!,
|
|
correlation_id: finished.correlation_id,
|
|
input_refs: [`0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`],
|
|
output_refs: [`0pt:quality_run:${finished.run_id}`],
|
|
});
|
|
await this.lineage.emit({
|
|
tenant_id: finished.tenant_id,
|
|
project_id: finished.project_id,
|
|
capability_id: "0pt.quality.run.v1",
|
|
run_id: finished.run_id,
|
|
event_type: finished.status === "succeeded" ? "COMPLETE" : "FAIL",
|
|
inputs: [
|
|
{
|
|
name: `dataset/${finished.dataset_id}`,
|
|
ref: `0pt:dataset:${finished.dataset_id}${finished.dataset_version ? `@${finished.dataset_version}` : ""}`,
|
|
},
|
|
],
|
|
outputs: [{ name: `quality_run/${finished.run_id}`, ref: `0pt:quality_run:${finished.run_id}` }],
|
|
correlation_id: finished.correlation_id,
|
|
run_facets: { siaxQuality: { score: finished.score, failed_rules: failed.length } },
|
|
});
|
|
|
|
return finished;
|
|
}
|
|
|
|
private validateExpectationShape(expectation: QualityExpectation): void {
|
|
if (!expectation.expectation_type?.trim()) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation_type is required");
|
|
}
|
|
if (expectation.kwargs === null || typeof expectation.kwargs !== "object") {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.quality", "expectation.kwargs must be an object");
|
|
}
|
|
}
|
|
|
|
private async emitRuleEvent(eventType: string, rule: DataQualityRule): Promise<void> {
|
|
await this.aud0.ingestEvent({
|
|
event_id: `evt_${eventType}_${rule.rule_id}_v${rule.version}`,
|
|
event_type: eventType,
|
|
tenant_id: rule.tenant_id,
|
|
project_id: rule.project_id,
|
|
actor_id: "svc_0pt",
|
|
app_id: "0pt",
|
|
capability_id: "0pt.quality.rule.v1",
|
|
resource_type: "quality_rule",
|
|
resource_id: rule.rule_id,
|
|
risk_level: "L1",
|
|
payload: { version: rule.version, severity: rule.severity, supplied_by: rule.supplied_by },
|
|
created_at: new Date().toISOString(),
|
|
});
|
|
}
|
|
}
|