- Source Registry (§10): rights/license/purpose metadata, provenance,
freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
(no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
MANDATORY tenant + purpose/rights filtering, versioned index definitions,
bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
engine for the GE core subset (never silently passes unsupported types),
runs/results/issues/score/trend, whole-run delegation to the GE backend
via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
st0re:artifact refs only, idempotent createVersion, export registers the
ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
scoring + per-field explanations, product hard-rule hook (force both
directions), review-band handoff, Splink batch via N0D contract.
93 new unit/contract tests; suite 248 passed / 1 skipped.
Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
311 lines
11 KiB
TypeScript
311 lines
11 KiB
TypeScript
/**
|
|
* src/er/entity-resolution-service.ts
|
|
* ====================================
|
|
* Generic Entity Resolution capability (Master-DoD §10 Entity Resolution:
|
|
* candidate API, generic probability, blocking, explanation, product
|
|
* hard-rule hook, human review handoff).
|
|
*
|
|
* Division of labour, stated plainly:
|
|
* - THIS service is the GENERIC candidate API: deterministic blocking +
|
|
* a transparent similarity scorer (token Jaccard + normalized Levenshtein)
|
|
* producing an explainable per-pair score. It is the reference
|
|
* implementation and the contract.
|
|
* - HEAVY/statistical ER (Splink probabilistic linkage, §11 ER) is
|
|
* 0pt-motor's job — `submitBatchResolution` creates the N0D job
|
|
* ("0pt.er.batch_resolution") and never fakes its output.
|
|
* - PRODUCTS own domain judgement via the HARD-RULE HOOK: e.g. VehIQ
|
|
* registers VIN-equality rules that FORCE match/non-match regardless of
|
|
* the generic score (Master-DoD §24 "VIN hard rules ... negative
|
|
* identity evidence"). Hard rules always win over probabilities.
|
|
* - Pairs in the configurable "review band" are handed off to humans
|
|
* (status pending_review) — never auto-merged.
|
|
*
|
|
* GENERIC means GENERIC: no person-matching logic, no Ratsit/PII pipelines —
|
|
* records are opaque field maps supplied by the calling product under its
|
|
* own legal basis; this module implements comparison mechanics only.
|
|
*/
|
|
import crypto from "crypto";
|
|
import { SiaxError } from "../platform/errors";
|
|
import type { N0dClient, N0dJob } from "../clients/n0d-client";
|
|
|
|
/** An opaque record supplied by the calling product. */
|
|
export interface ErRecord {
|
|
record_id: string;
|
|
fields: Record<string, string | number | null | undefined>;
|
|
}
|
|
|
|
export type HardRuleDecision = "force_match" | "force_non_match" | "abstain";
|
|
|
|
/** Product-supplied hard rule (the §10 hook). */
|
|
export interface HardRule {
|
|
rule_id: string;
|
|
decide(a: ErRecord, b: ErRecord): HardRuleDecision;
|
|
}
|
|
|
|
export interface ErCandidateRequest {
|
|
tenant_id: string;
|
|
project_id: string;
|
|
records: ErRecord[];
|
|
/** Fields whose normalized values form the blocking key. */
|
|
blocking_fields: string[];
|
|
/** Fields compared for similarity (weight defaults to 1). */
|
|
compare_fields: { field: string; weight?: number }[];
|
|
/** Score ≥ auto_match_threshold → match (unless a hard rule vetoes). Default 0.9. */
|
|
auto_match_threshold?: number;
|
|
/** Score ≥ review_threshold (< auto) → pending_review. Default 0.6. */
|
|
review_threshold?: number;
|
|
correlation_id?: string;
|
|
}
|
|
|
|
export type ErCandidateStatus = "match" | "pending_review" | "non_match";
|
|
|
|
export interface ErFieldExplanation {
|
|
field: string;
|
|
a: string | null;
|
|
b: string | null;
|
|
similarity: number;
|
|
weight: number;
|
|
}
|
|
|
|
export interface ErCandidatePair {
|
|
candidate_id: string;
|
|
record_a: string;
|
|
record_b: string;
|
|
/** Generic similarity score 0..1. */
|
|
score: number;
|
|
status: ErCandidateStatus;
|
|
/** Which mechanism decided: hard rule id, or "score". */
|
|
decided_by: string;
|
|
/** Per-field breakdown — the §10 explanation. */
|
|
explanation: ErFieldExplanation[];
|
|
blocking_key: string;
|
|
}
|
|
|
|
export interface ErCandidateResult {
|
|
request_id: string;
|
|
tenant_id: string;
|
|
pairs: ErCandidatePair[];
|
|
/** Pairs handed to human review (subset of pairs). */
|
|
review_queue: ErCandidatePair[];
|
|
blocked_comparisons: number;
|
|
total_possible_comparisons: number;
|
|
}
|
|
|
|
export class EntityResolutionService {
|
|
private readonly hardRules: HardRule[] = [];
|
|
|
|
constructor(private readonly opts: { n0dClient?: N0dClient } = {}) {}
|
|
|
|
/** The product hard-rule hook (§10). Rules run in registration order. */
|
|
registerHardRule(rule: HardRule): void {
|
|
this.hardRules.push(rule);
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Candidate API (synchronous, generic)
|
|
// -----------------------------------------------------------------------
|
|
generateCandidates(req: ErCandidateRequest): ErCandidateResult {
|
|
if (!req.tenant_id?.trim()) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.er", "tenant_id is required");
|
|
}
|
|
if (req.records.length < 2) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.er", "at least two records are required");
|
|
}
|
|
if (req.blocking_fields.length === 0) {
|
|
throw new SiaxError(
|
|
"INVALID_REQUEST",
|
|
"0pt.er",
|
|
"blocking_fields must not be empty — unblocked all-pairs comparison is refused (quadratic + §46)"
|
|
);
|
|
}
|
|
const autoThreshold = req.auto_match_threshold ?? 0.9;
|
|
const reviewThreshold = req.review_threshold ?? 0.6;
|
|
if (reviewThreshold >= autoThreshold) {
|
|
throw new SiaxError("INVALID_REQUEST", "0pt.er", "review_threshold must be < auto_match_threshold");
|
|
}
|
|
|
|
// Blocking: bucket records by normalized blocking key.
|
|
const blocks = new Map<string, ErRecord[]>();
|
|
for (const rec of req.records) {
|
|
const key = req.blocking_fields.map((f) => normalize(String(rec.fields[f] ?? ""))).join("|");
|
|
const bucket = blocks.get(key) ?? [];
|
|
bucket.push(rec);
|
|
blocks.set(key, bucket);
|
|
}
|
|
|
|
const pairs: ErCandidatePair[] = [];
|
|
let blockedComparisons = 0;
|
|
for (const [key, bucket] of blocks) {
|
|
if (key.replace(/\|/g, "") === "") continue; // empty blocking key never blocks together
|
|
for (let i = 0; i < bucket.length; i++) {
|
|
for (let j = i + 1; j < bucket.length; j++) {
|
|
blockedComparisons++;
|
|
pairs.push(this.comparePair(bucket[i], bucket[j], req, autoThreshold, reviewThreshold, key));
|
|
}
|
|
}
|
|
}
|
|
|
|
const n = req.records.length;
|
|
return {
|
|
request_id: `errq_${crypto.randomUUID()}`,
|
|
tenant_id: req.tenant_id,
|
|
pairs,
|
|
review_queue: pairs.filter((p) => p.status === "pending_review"),
|
|
blocked_comparisons: blockedComparisons,
|
|
total_possible_comparisons: (n * (n - 1)) / 2,
|
|
};
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
// Batch/statistical path — 0pt-motor via N0D (contract only, never faked)
|
|
// -----------------------------------------------------------------------
|
|
async submitBatchResolution(input: {
|
|
tenant_id: string;
|
|
project_id: string;
|
|
/** ST0RE artifact ref of the record dataset. */
|
|
records_artifact_ref: string;
|
|
correlation_id?: string;
|
|
}): Promise<N0dJob> {
|
|
if (!this.opts.n0dClient) {
|
|
throw new SiaxError("DEPENDENCY_UNAVAILABLE", "n0d", "no N0D client configured for batch ER");
|
|
}
|
|
return this.opts.n0dClient.createJob({
|
|
tenant_id: input.tenant_id,
|
|
project_id: input.project_id,
|
|
job_type: "0pt.er.batch_resolution",
|
|
capability_id: "0pt.er.batch.v1",
|
|
requested_by_app_id: "0pt",
|
|
requested_by_actor_id: "svc_0pt",
|
|
input_artifact_ids: [input.records_artifact_ref],
|
|
output_artifact_type: "er_candidate_set",
|
|
requires_backbone: false,
|
|
requires_gpu: false,
|
|
required_node_roles: ["compute_general"],
|
|
resource_profile: "large",
|
|
risk_level: "L3",
|
|
correlation_id: input.correlation_id,
|
|
});
|
|
}
|
|
|
|
// -----------------------------------------------------------------------
|
|
private comparePair(
|
|
a: ErRecord,
|
|
b: ErRecord,
|
|
req: ErCandidateRequest,
|
|
autoThreshold: number,
|
|
reviewThreshold: number,
|
|
blockingKey: string
|
|
): ErCandidatePair {
|
|
// Hard rules first — they always win (§10/§24).
|
|
for (const rule of this.hardRules) {
|
|
const decision = rule.decide(a, b);
|
|
if (decision !== "abstain") {
|
|
return {
|
|
candidate_id: `erc_${crypto.randomUUID()}`,
|
|
record_a: a.record_id,
|
|
record_b: b.record_id,
|
|
score: decision === "force_match" ? 1 : 0,
|
|
status: decision === "force_match" ? "match" : "non_match",
|
|
decided_by: rule.rule_id,
|
|
explanation: this.explain(a, b, req),
|
|
blocking_key: blockingKey,
|
|
};
|
|
}
|
|
}
|
|
|
|
const explanation = this.explain(a, b, req);
|
|
const weightSum = explanation.reduce((s, e) => s + e.weight, 0);
|
|
const score = weightSum === 0 ? 0 : explanation.reduce((s, e) => s + e.similarity * e.weight, 0) / weightSum;
|
|
const rounded = Math.round(score * 1000) / 1000;
|
|
|
|
const status: ErCandidateStatus =
|
|
rounded >= autoThreshold ? "match" : rounded >= reviewThreshold ? "pending_review" : "non_match";
|
|
return {
|
|
candidate_id: `erc_${crypto.randomUUID()}`,
|
|
record_a: a.record_id,
|
|
record_b: b.record_id,
|
|
score: rounded,
|
|
status,
|
|
decided_by: "score",
|
|
explanation,
|
|
blocking_key: blockingKey,
|
|
};
|
|
}
|
|
|
|
private explain(a: ErRecord, b: ErRecord, req: ErCandidateRequest): ErFieldExplanation[] {
|
|
return req.compare_fields.map(({ field, weight }) => {
|
|
const av = a.fields[field] === null || a.fields[field] === undefined ? null : String(a.fields[field]);
|
|
const bv = b.fields[field] === null || b.fields[field] === undefined ? null : String(b.fields[field]);
|
|
return {
|
|
field,
|
|
a: av,
|
|
b: bv,
|
|
similarity: similarity(av, bv),
|
|
weight: weight ?? 1,
|
|
};
|
|
});
|
|
}
|
|
}
|
|
|
|
// ---------------------------------------------------------------------------
|
|
// Transparent similarity primitives (deterministic; the "generic probability")
|
|
// ---------------------------------------------------------------------------
|
|
/**
|
|
* Case-fold, DIACRITIC-FOLD (ö→o, é→e — deliberate: "Göteborg" and
|
|
* "Goteborg" must land in the same block), strip punctuation, collapse
|
|
* whitespace. ø has no NFKD decomposition and is mapped explicitly.
|
|
*/
|
|
export function normalize(s: string): string {
|
|
return s
|
|
.toLowerCase()
|
|
.replace(/ø/g, "o")
|
|
.normalize("NFKD")
|
|
.replace(/[̀-ͯ]/g, "")
|
|
.replace(/[^a-z0-9 ]/g, " ")
|
|
.replace(/\s+/g, " ")
|
|
.trim();
|
|
}
|
|
|
|
/** Combined similarity: max(token Jaccard, 1 - normalized Levenshtein). */
|
|
export function similarity(a: string | null, b: string | null): number {
|
|
if (a === null || b === null) return 0;
|
|
const na = normalize(a);
|
|
const nb = normalize(b);
|
|
if (na === "" && nb === "") return 0;
|
|
if (na === nb) return 1;
|
|
return Math.max(tokenJaccard(na, nb), 1 - normalizedLevenshtein(na, nb));
|
|
}
|
|
|
|
function tokenJaccard(a: string, b: string): number {
|
|
const ta = new Set(a.split(" "));
|
|
const tb = new Set(b.split(" "));
|
|
const intersection = [...ta].filter((t) => tb.has(t)).length;
|
|
const union = new Set([...ta, ...tb]).size;
|
|
return union === 0 ? 0 : intersection / union;
|
|
}
|
|
|
|
function normalizedLevenshtein(a: string, b: string): number {
|
|
const maxLen = Math.max(a.length, b.length);
|
|
if (maxLen === 0) return 0;
|
|
return levenshtein(a, b) / maxLen;
|
|
}
|
|
|
|
function levenshtein(a: string, b: string): number {
|
|
const m = a.length;
|
|
const n = b.length;
|
|
if (m === 0) return n;
|
|
if (n === 0) return m;
|
|
let prev = new Array<number>(n + 1);
|
|
let curr = new Array<number>(n + 1);
|
|
for (let j = 0; j <= n; j++) prev[j] = j;
|
|
for (let i = 1; i <= m; i++) {
|
|
curr[0] = i;
|
|
for (let j = 1; j <= n; j++) {
|
|
const cost = a[i - 1] === b[j - 1] ? 0 : 1;
|
|
curr[j] = Math.min(prev[j] + 1, curr[j - 1] + 1, prev[j - 1] + cost);
|
|
}
|
|
[prev, curr] = [curr, prev];
|
|
}
|
|
return prev[n];
|
|
}
|