feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
This commit is contained in:
co-authored by
Claude Fable 5
parent
da58176735
commit
b46807da75
@@ -0,0 +1,310 @@
|
||||
/**
|
||||
* src/er/entity-resolution-service.ts
|
||||
* ====================================
|
||||
* Generic Entity Resolution capability (Master-DoD §10 Entity Resolution:
|
||||
* candidate API, generic probability, blocking, explanation, product
|
||||
* hard-rule hook, human review handoff).
|
||||
*
|
||||
* Division of labour, stated plainly:
|
||||
* - THIS service is the GENERIC candidate API: deterministic blocking +
|
||||
* a transparent similarity scorer (token Jaccard + normalized Levenshtein)
|
||||
* producing an explainable per-pair score. It is the reference
|
||||
* implementation and the contract.
|
||||
* - HEAVY/statistical ER (Splink probabilistic linkage, §11 ER) is
|
||||
* 0pt-motor's job — `submitBatchResolution` creates the N0D job
|
||||
* ("0pt.er.batch_resolution") and never fakes its output.
|
||||
* - PRODUCTS own domain judgement via the HARD-RULE HOOK: e.g. VehIQ
|
||||
* registers VIN-equality rules that FORCE match/non-match regardless of
|
||||
* the generic score (Master-DoD §24 "VIN hard rules ... negative
|
||||
* identity evidence"). Hard rules always win over probabilities.
|
||||
* - Pairs in the configurable "review band" are handed off to humans
|
||||
* (status pending_review) — never auto-merged.
|
||||
*
|
||||
* GENERIC means GENERIC: no person-matching logic, no Ratsit/PII pipelines —
|
||||
* records are opaque field maps supplied by the calling product under its
|
||||
* own legal basis; this module implements comparison mechanics only.
|
||||
*/
|
||||
import crypto from "crypto";
|
||||
import { SiaxError } from "../platform/errors";
|
||||
import type { N0dClient, N0dJob } from "../clients/n0d-client";
|
||||
|
||||
/** An opaque record supplied by the calling product. */
|
||||
export interface ErRecord {
|
||||
record_id: string;
|
||||
fields: Record<string, string | number | null | undefined>;
|
||||
}
|
||||
|
||||
export type HardRuleDecision = "force_match" | "force_non_match" | "abstain";
|
||||
|
||||
/** Product-supplied hard rule (the §10 hook). */
|
||||
export interface HardRule {
|
||||
rule_id: string;
|
||||
decide(a: ErRecord, b: ErRecord): HardRuleDecision;
|
||||
}
|
||||
|
||||
export interface ErCandidateRequest {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
records: ErRecord[];
|
||||
/** Fields whose normalized values form the blocking key. */
|
||||
blocking_fields: string[];
|
||||
/** Fields compared for similarity (weight defaults to 1). */
|
||||
compare_fields: { field: string; weight?: number }[];
|
||||
/** Score ≥ auto_match_threshold → match (unless a hard rule vetoes). Default 0.9. */
|
||||
auto_match_threshold?: number;
|
||||
/** Score ≥ review_threshold (< auto) → pending_review. Default 0.6. */
|
||||
review_threshold?: number;
|
||||
correlation_id?: string;
|
||||
}
|
||||
|
||||
export type ErCandidateStatus = "match" | "pending_review" | "non_match";
|
||||
|
||||
export interface ErFieldExplanation {
|
||||
field: string;
|
||||
a: string | null;
|
||||
b: string | null;
|
||||
similarity: number;
|
||||
weight: number;
|
||||
}
|
||||
|
||||
export interface ErCandidatePair {
|
||||
candidate_id: string;
|
||||
record_a: string;
|
||||
record_b: string;
|
||||
/** Generic similarity score 0..1. */
|
||||
score: number;
|
||||
status: ErCandidateStatus;
|
||||
/** Which mechanism decided: hard rule id, or "score". */
|
||||
decided_by: string;
|
||||
/** Per-field breakdown — the §10 explanation. */
|
||||
explanation: ErFieldExplanation[];
|
||||
blocking_key: string;
|
||||
}
|
||||
|
||||
export interface ErCandidateResult {
|
||||
request_id: string;
|
||||
tenant_id: string;
|
||||
pairs: ErCandidatePair[];
|
||||
/** Pairs handed to human review (subset of pairs). */
|
||||
review_queue: ErCandidatePair[];
|
||||
blocked_comparisons: number;
|
||||
total_possible_comparisons: number;
|
||||
}
|
||||
|
||||
export class EntityResolutionService {
|
||||
private readonly hardRules: HardRule[] = [];
|
||||
|
||||
constructor(private readonly opts: { n0dClient?: N0dClient } = {}) {}
|
||||
|
||||
/** The product hard-rule hook (§10). Rules run in registration order. */
|
||||
registerHardRule(rule: HardRule): void {
|
||||
this.hardRules.push(rule);
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Candidate API (synchronous, generic)
|
||||
// -----------------------------------------------------------------------
|
||||
generateCandidates(req: ErCandidateRequest): ErCandidateResult {
|
||||
if (!req.tenant_id?.trim()) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
|
||||
}
|
||||
if (req.records.length < 2) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "at least two records are required");
|
||||
}
|
||||
if (req.blocking_fields.length === 0) {
|
||||
throw new SiaxError(
|
||||
"INVALID_REQUEST",
|
||||
"0pt.er",
|
||||
"blocking_fields must not be empty — unblocked all-pairs comparison is refused (quadratic + §46)"
|
||||
);
|
||||
}
|
||||
const autoThreshold = req.auto_match_threshold ?? 0.9;
|
||||
const reviewThreshold = req.review_threshold ?? 0.6;
|
||||
if (reviewThreshold >= autoThreshold) {
|
||||
throw new SiaxError("INVALID_REQUEST", "0pt.er", "review_threshold must be < auto_match_threshold");
|
||||
}
|
||||
|
||||
// Blocking: bucket records by normalized blocking key.
|
||||
const blocks = new Map<string, ErRecord[]>();
|
||||
for (const rec of req.records) {
|
||||
const key = req.blocking_fields.map((f) => normalize(String(rec.fields[f] ?? ""))).join("|");
|
||||
const bucket = blocks.get(key) ?? [];
|
||||
bucket.push(rec);
|
||||
blocks.set(key, bucket);
|
||||
}
|
||||
|
||||
const pairs: ErCandidatePair[] = [];
|
||||
let blockedComparisons = 0;
|
||||
for (const [key, bucket] of blocks) {
|
||||
if (key.replace(/\|/g, "") === "") continue; // empty blocking key never blocks together
|
||||
for (let i = 0; i < bucket.length; i++) {
|
||||
for (let j = i + 1; j < bucket.length; j++) {
|
||||
blockedComparisons++;
|
||||
pairs.push(this.comparePair(bucket[i], bucket[j], req, autoThreshold, reviewThreshold, key));
|
||||
}
|
||||
}
|
||||
}
|
||||
|
||||
const n = req.records.length;
|
||||
return {
|
||||
request_id: `errq_${crypto.randomUUID()}`,
|
||||
tenant_id: req.tenant_id,
|
||||
pairs,
|
||||
review_queue: pairs.filter((p) => p.status === "pending_review"),
|
||||
blocked_comparisons: blockedComparisons,
|
||||
total_possible_comparisons: (n * (n - 1)) / 2,
|
||||
};
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
// Batch/statistical path — 0pt-motor via N0D (contract only, never faked)
|
||||
// -----------------------------------------------------------------------
|
||||
async submitBatchResolution(input: {
|
||||
tenant_id: string;
|
||||
project_id: string;
|
||||
/** ST0RE artifact ref of the record dataset. */
|
||||
records_artifact_ref: string;
|
||||
correlation_id?: string;
|
||||
}): Promise<N0dJob> {
|
||||
if (!this.opts.n0dClient) {
|
||||
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "n0d", "no N0D client configured for batch ER");
|
||||
}
|
||||
return this.opts.n0dClient.createJob({
|
||||
tenant_id: input.tenant_id,
|
||||
project_id: input.project_id,
|
||||
job_type: "0pt.er.batch_resolution",
|
||||
capability_id: "0pt.er.batch.v1",
|
||||
requested_by_app_id: "0pt",
|
||||
requested_by_actor_id: "svc_0pt",
|
||||
input_artifact_ids: [input.records_artifact_ref],
|
||||
output_artifact_type: "er_candidate_set",
|
||||
requires_backbone: false,
|
||||
requires_gpu: false,
|
||||
required_node_roles: ["compute_general"],
|
||||
resource_profile: "large",
|
||||
risk_level: "L3",
|
||||
correlation_id: input.correlation_id,
|
||||
});
|
||||
}
|
||||
|
||||
// -----------------------------------------------------------------------
|
||||
private comparePair(
|
||||
a: ErRecord,
|
||||
b: ErRecord,
|
||||
req: ErCandidateRequest,
|
||||
autoThreshold: number,
|
||||
reviewThreshold: number,
|
||||
blockingKey: string
|
||||
): ErCandidatePair {
|
||||
// Hard rules first — they always win (§10/§24).
|
||||
for (const rule of this.hardRules) {
|
||||
const decision = rule.decide(a, b);
|
||||
if (decision !== "abstain") {
|
||||
return {
|
||||
candidate_id: `erc_${crypto.randomUUID()}`,
|
||||
record_a: a.record_id,
|
||||
record_b: b.record_id,
|
||||
score: decision === "force_match" ? 1 : 0,
|
||||
status: decision === "force_match" ? "match" : "non_match",
|
||||
decided_by: rule.rule_id,
|
||||
explanation: this.explain(a, b, req),
|
||||
blocking_key: blockingKey,
|
||||
};
|
||||
}
|
||||
}
|
||||
|
||||
const explanation = this.explain(a, b, req);
|
||||
const weightSum = explanation.reduce((s, e) => s + e.weight, 0);
|
||||
const score = weightSum === 0 ? 0 : explanation.reduce((s, e) => s + e.similarity * e.weight, 0) / weightSum;
|
||||
const rounded = Math.round(score * 1000) / 1000;
|
||||
|
||||
const status: ErCandidateStatus =
|
||||
rounded >= autoThreshold ? "match" : rounded >= reviewThreshold ? "pending_review" : "non_match";
|
||||
return {
|
||||
candidate_id: `erc_${crypto.randomUUID()}`,
|
||||
record_a: a.record_id,
|
||||
record_b: b.record_id,
|
||||
score: rounded,
|
||||
status,
|
||||
decided_by: "score",
|
||||
explanation,
|
||||
blocking_key: blockingKey,
|
||||
};
|
||||
}
|
||||
|
||||
private explain(a: ErRecord, b: ErRecord, req: ErCandidateRequest): ErFieldExplanation[] {
|
||||
return req.compare_fields.map(({ field, weight }) => {
|
||||
const av = a.fields[field] === null || a.fields[field] === undefined ? null : String(a.fields[field]);
|
||||
const bv = b.fields[field] === null || b.fields[field] === undefined ? null : String(b.fields[field]);
|
||||
return {
|
||||
field,
|
||||
a: av,
|
||||
b: bv,
|
||||
similarity: similarity(av, bv),
|
||||
weight: weight ?? 1,
|
||||
};
|
||||
});
|
||||
}
|
||||
}
|
||||
|
||||
// ---------------------------------------------------------------------------
|
||||
// Transparent similarity primitives (deterministic; the "generic probability")
|
||||
// ---------------------------------------------------------------------------
|
||||
/**
|
||||
* Case-fold, DIACRITIC-FOLD (ö→o, é→e — deliberate: "Göteborg" and
|
||||
* "Goteborg" must land in the same block), strip punctuation, collapse
|
||||
* whitespace. ø has no NFKD decomposition and is mapped explicitly.
|
||||
*/
|
||||
export function normalize(s: string): string {
|
||||
return s
|
||||
.toLowerCase()
|
||||
.replace(/ø/g, "o")
|
||||
.normalize("NFKD")
|
||||
.replace(/[̀-ͯ]/g, "")
|
||||
.replace(/[^a-z0-9 ]/g, " ")
|
||||
.replace(/\s+/g, " ")
|
||||
.trim();
|
||||
}
|
||||
|
||||
/** Combined similarity: max(token Jaccard, 1 - normalized Levenshtein). */
|
||||
export function similarity(a: string | null, b: string | null): number {
|
||||
if (a === null || b === null) return 0;
|
||||
const na = normalize(a);
|
||||
const nb = normalize(b);
|
||||
if (na === "" && nb === "") return 0;
|
||||
if (na === nb) return 1;
|
||||
return Math.max(tokenJaccard(na, nb), 1 - normalizedLevenshtein(na, nb));
|
||||
}
|
||||
|
||||
function tokenJaccard(a: string, b: string): number {
|
||||
const ta = new Set(a.split(" "));
|
||||
const tb = new Set(b.split(" "));
|
||||
const intersection = [...ta].filter((t) => tb.has(t)).length;
|
||||
const union = new Set([...ta, ...tb]).size;
|
||||
return union === 0 ? 0 : intersection / union;
|
||||
}
|
||||
|
||||
function normalizedLevenshtein(a: string, b: string): number {
|
||||
const maxLen = Math.max(a.length, b.length);
|
||||
if (maxLen === 0) return 0;
|
||||
return levenshtein(a, b) / maxLen;
|
||||
}
|
||||
|
||||
function levenshtein(a: string, b: string): number {
|
||||
const m = a.length;
|
||||
const n = b.length;
|
||||
if (m === 0) return n;
|
||||
if (n === 0) return m;
|
||||
let prev = new Array<number>(n + 1);
|
||||
let curr = new Array<number>(n + 1);
|
||||
for (let j = 0; j <= n; j++) prev[j] = j;
|
||||
for (let i = 1; i <= m; i++) {
|
||||
curr[0] = i;
|
||||
for (let j = 1; j <= n; j++) {
|
||||
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
||||
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
|
||||
}
|
||||
[prev, curr] = [curr, prev];
|
||||
}
|
||||
return prev[n];
|
||||
}
|
||||
Reference in New Issue
Block a user