feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER

- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
Simon Axelsson
2026-08-31 16:49:32 +02:00
co-authored by Claude Fable 5
parent da58176735
commit b46807da75
19 changed files with 4038 additions and 0 deletions
+310
View File
@@ -0,0 +1,310 @@
/**
* src/er/entity-resolution-service.ts
* ====================================
* Generic Entity Resolution capability (Master-DoD §10 Entity Resolution:
* candidate API, generic probability, blocking, explanation, product
* hard-rule hook, human review handoff).
*
* Division of labour, stated plainly:
* - THIS service is the GENERIC candidate API: deterministic blocking +
* a transparent similarity scorer (token Jaccard + normalized Levenshtein)
* producing an explainable per-pair score. It is the reference
* implementation and the contract.
* - HEAVY/statistical ER (Splink probabilistic linkage, §11 ER) is
* 0pt-motor's job — `submitBatchResolution` creates the N0D job
* ("0pt.er.batch_resolution") and never fakes its output.
* - PRODUCTS own domain judgement via the HARD-RULE HOOK: e.g. VehIQ
* registers VIN-equality rules that FORCE match/non-match regardless of
* the generic score (Master-DoD §24 "VIN hard rules ... negative
* identity evidence"). Hard rules always win over probabilities.
* - Pairs in the configurable "review band" are handed off to humans
* (status pending_review) — never auto-merged.
*
* GENERIC means GENERIC: no person-matching logic, no Ratsit/PII pipelines —
* records are opaque field maps supplied by the calling product under its
* own legal basis; this module implements comparison mechanics only.
*/
import crypto from "crypto";
import { SiaxError } from "../platform/errors";
import type { N0dClient, N0dJob } from "../clients/n0d-client";
/** An opaque record supplied by the calling product. */
export interface ErRecord {
record_id: string;
fields: Record<string, string | number | null | undefined>;
}
export type HardRuleDecision = "force_match" | "force_non_match" | "abstain";
/** Product-supplied hard rule (the §10 hook). */
export interface HardRule {
rule_id: string;
decide(a: ErRecord, b: ErRecord): HardRuleDecision;
}
export interface ErCandidateRequest {
tenant_id: string;
project_id: string;
records: ErRecord[];
/** Fields whose normalized values form the blocking key. */
blocking_fields: string[];
/** Fields compared for similarity (weight defaults to 1). */
compare_fields: { field: string; weight?: number }[];
/** Score ≥ auto_match_threshold → match (unless a hard rule vetoes). Default 0.9. */
auto_match_threshold?: number;
/** Score ≥ review_threshold (< auto) → pending_review. Default 0.6. */
review_threshold?: number;
correlation_id?: string;
}
export type ErCandidateStatus = "match" | "pending_review" | "non_match";
export interface ErFieldExplanation {
field: string;
a: string | null;
b: string | null;
similarity: number;
weight: number;
}
export interface ErCandidatePair {
candidate_id: string;
record_a: string;
record_b: string;
/** Generic similarity score 0..1. */
score: number;
status: ErCandidateStatus;
/** Which mechanism decided: hard rule id, or "score". */
decided_by: string;
/** Per-field breakdown — the §10 explanation. */
explanation: ErFieldExplanation[];
blocking_key: string;
}
export interface ErCandidateResult {
request_id: string;
tenant_id: string;
pairs: ErCandidatePair[];
/** Pairs handed to human review (subset of pairs). */
review_queue: ErCandidatePair[];
blocked_comparisons: number;
total_possible_comparisons: number;
}
export class EntityResolutionService {
private readonly hardRules: HardRule[] = [];
constructor(private readonly opts: { n0dClient?: N0dClient } = {}) {}
/** The product hard-rule hook (§10). Rules run in registration order. */
registerHardRule(rule: HardRule): void {
this.hardRules.push(rule);
}
// -----------------------------------------------------------------------
// Candidate API (synchronous, generic)
// -----------------------------------------------------------------------
generateCandidates(req: ErCandidateRequest): ErCandidateResult {
if (!req.tenant_id?.trim()) {
throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
}
if (req.records.length < 2) {
throw new SiaxError("INVALID_REQUEST", "0pt.er", "at least two records are required");
}
if (req.blocking_fields.length === 0) {
throw new SiaxError(
"INVALID_REQUEST",
"0pt.er",
"blocking_fields must not be empty — unblocked all-pairs comparison is refused (quadratic + §46)"
);
}
const autoThreshold = req.auto_match_threshold ?? 0.9;
const reviewThreshold = req.review_threshold ?? 0.6;
if (reviewThreshold >= autoThreshold) {
throw new SiaxError("INVALID_REQUEST", "0pt.er", "review_threshold must be < auto_match_threshold");
}
// Blocking: bucket records by normalized blocking key.
const blocks = new Map<string, ErRecord[]>();
for (const rec of req.records) {
const key = req.blocking_fields.map((f) => normalize(String(rec.fields[f] ?? ""))).join("|");
const bucket = blocks.get(key) ?? [];
bucket.push(rec);
blocks.set(key, bucket);
}
const pairs: ErCandidatePair[] = [];
let blockedComparisons = 0;
for (const [key, bucket] of blocks) {
if (key.replace(/\|/g, "") === "") continue; // empty blocking key never blocks together
for (let i = 0; i < bucket.length; i++) {
for (let j = i + 1; j < bucket.length; j++) {
blockedComparisons++;
pairs.push(this.comparePair(bucket[i], bucket[j], req, autoThreshold, reviewThreshold, key));
}
}
}
const n = req.records.length;
return {
request_id: `errq_${crypto.randomUUID()}`,
tenant_id: req.tenant_id,
pairs,
review_queue: pairs.filter((p) => p.status === "pending_review"),
blocked_comparisons: blockedComparisons,
total_possible_comparisons: (n * (n - 1)) / 2,
};
}
// -----------------------------------------------------------------------
// Batch/statistical path — 0pt-motor via N0D (contract only, never faked)
// -----------------------------------------------------------------------
async submitBatchResolution(input: {
tenant_id: string;
project_id: string;
/** ST0RE artifact ref of the record dataset. */
records_artifact_ref: string;
correlation_id?: string;
}): Promise<N0dJob> {
if (!this.opts.n0dClient) {
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "n0d", "no N0D client configured for batch ER");
}
return this.opts.n0dClient.createJob({
tenant_id: input.tenant_id,
project_id: input.project_id,
job_type: "0pt.er.batch_resolution",
capability_id: "0pt.er.batch.v1",
requested_by_app_id: "0pt",
requested_by_actor_id: "svc_0pt",
input_artifact_ids: [input.records_artifact_ref],
output_artifact_type: "er_candidate_set",
requires_backbone: false,
requires_gpu: false,
required_node_roles: ["compute_general"],
resource_profile: "large",
risk_level: "L3",
correlation_id: input.correlation_id,
});
}
// -----------------------------------------------------------------------
private comparePair(
a: ErRecord,
b: ErRecord,
req: ErCandidateRequest,
autoThreshold: number,
reviewThreshold: number,
blockingKey: string
): ErCandidatePair {
// Hard rules first — they always win (§10/§24).
for (const rule of this.hardRules) {
const decision = rule.decide(a, b);
if (decision !== "abstain") {
return {
candidate_id: `erc_${crypto.randomUUID()}`,
record_a: a.record_id,
record_b: b.record_id,
score: decision === "force_match" ? 1 : 0,
status: decision === "force_match" ? "match" : "non_match",
decided_by: rule.rule_id,
explanation: this.explain(a, b, req),
blocking_key: blockingKey,
};
}
}
const explanation = this.explain(a, b, req);
const weightSum = explanation.reduce((s, e) => s + e.weight, 0);
const score = weightSum === 0 ? 0 : explanation.reduce((s, e) => s + e.similarity * e.weight, 0) / weightSum;
const rounded = Math.round(score * 1000) / 1000;
const status: ErCandidateStatus =
rounded >= autoThreshold ? "match" : rounded >= reviewThreshold ? "pending_review" : "non_match";
return {
candidate_id: `erc_${crypto.randomUUID()}`,
record_a: a.record_id,
record_b: b.record_id,
score: rounded,
status,
decided_by: "score",
explanation,
blocking_key: blockingKey,
};
}
private explain(a: ErRecord, b: ErRecord, req: ErCandidateRequest): ErFieldExplanation[] {
return req.compare_fields.map(({ field, weight }) => {
const av = a.fields[field] === null || a.fields[field] === undefined ? null : String(a.fields[field]);
const bv = b.fields[field] === null || b.fields[field] === undefined ? null : String(b.fields[field]);
return {
field,
a: av,
b: bv,
similarity: similarity(av, bv),
weight: weight ?? 1,
};
});
}
}
// ---------------------------------------------------------------------------
// Transparent similarity primitives (deterministic; the "generic probability")
// ---------------------------------------------------------------------------
/**
* Case-fold, DIACRITIC-FOLD (ö→o, é→e — deliberate: "Göteborg" and
* "Goteborg" must land in the same block), strip punctuation, collapse
* whitespace. ø has no NFKD decomposition and is mapped explicitly.
*/
export function normalize(s: string): string {
return s
.toLowerCase()
.replace(/ø/g, "o")
.normalize("NFKD")
.replace(/[̀-ͯ]/g, "")
.replace(/[^a-z0-9 ]/g, " ")
.replace(/\s+/g, " ")
.trim();
}
/** Combined similarity: max(token Jaccard, 1 - normalized Levenshtein). */
export function similarity(a: string | null, b: string | null): number {
if (a === null || b === null) return 0;
const na = normalize(a);
const nb = normalize(b);
if (na === "" && nb === "") return 0;
if (na === nb) return 1;
return Math.max(tokenJaccard(na, nb), 1 - normalizedLevenshtein(na, nb));
}
function tokenJaccard(a: string, b: string): number {
const ta = new Set(a.split(" "));
const tb = new Set(b.split(" "));
const intersection = [...ta].filter((t) => tb.has(t)).length;
const union = new Set([...ta, ...tb]).size;
return union === 0 ? 0 : intersection / union;
}
function normalizedLevenshtein(a: string, b: string): number {
const maxLen = Math.max(a.length, b.length);
if (maxLen === 0) return 0;
return levenshtein(a, b) / maxLen;
}
function levenshtein(a: string, b: string): number {
const m = a.length;
const n = b.length;
if (m === 0) return n;
if (n === 0) return m;
let prev = new Array<number>(n + 1);
let curr = new Array<number>(n + 1);
for (let j = 0; j <= n; j++) prev[j] = j;
for (let i = 1; i <= m; i++) {
curr[0] = i;
for (let j = 1; j <= n; j++) {
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
}
[prev, curr] = [curr, prev];
}
return prev[n];
}