Files
0pt/tests/p0/entity-resolution.test.ts
T
Simon AxelssonandClaude Fable 5 b46807da75 feat(data-os): P0 capability layer — source registry, search, quality, dataset versions, metadata, analytics, features, ER
- Source Registry (§10): rights/license/purpose metadata, provenance,
  freshness SLA, health checks, describe/query; purpose gate FAILS CLOSED
  (no rights record => deny); cross-tenant probes => NOT_FOUND.
- Search (§10/§42): OpenSearch REST adapter (no SDK) + SearchService with
  MANDATORY tenant + purpose/rights filtering, versioned index definitions,
  bulk, generation-per-rebuild (opt-{name}--g{n}) with atomic alias cutover,
  rollback, failed-rebuild generation discard. OpenSearch never canonical.
- Quality (§10): versioned rule registry (GE-JSON interchange), local
  engine for the GE core subset (never silently passes unsupported types),
  runs/results/issues/score/trend, whole-run delegation to the GE backend
  via N0D (0pt.quality.ge_run) - never fabricated results.
- Dataset (§10/§5/§53): immutable numbered versions holding canonical
  st0re:artifact refs only, idempotent createVersion, export registers the
  ST0RE artifact + AUD0 evidence + OpenLineage.
- Metadata (§10): OpenMetadata PUT /api/v1/tables projection adapter +
  projectAll rebuild path; projection never canonical.
- Analytics (§10): ClickHouse HTTP adapter, closed metric-query DSL with
  FORCED tenant predicate + readonly/max_execution_time/max_result_rows,
  TTL retention. Zero connection to b0ts/quarantined pipelines.
- Features (§10): versioned feature-view registry, real Feast
  get-online-features adapter, materialize/get-historical as N0D delegation.
- ER (§10): generic candidate API with deterministic blocking + transparent
  scoring + per-field explanations, product hard-rule hook (force both
  directions), review-band handoff, Splink batch via N0D contract.

93 new unit/contract tests; suite 248 passed / 1 skipped.

Co-Authored-By: Claude Fable 5 <noreply@anthropic.com>
2026-08-31 16:49:32 +02:00

138 lines
5.9 KiB
TypeScript

/**
* EntityResolutionService unit tests (Master-DoD §10 Entity Resolution):
* blocking, transparent scoring, explanation, product hard-rule hook,
* human-review handoff, batch delegation contract.
*/
import { describe, it, expect } from "vitest";
import {
EntityResolutionService,
normalize,
similarity,
type ErRecord,
} from "@/src/er/entity-resolution-service";
const RECORDS: ErRecord[] = [
{ record_id: "a1", fields: { name: "Volvo Personvagnar AB", city: "göteborg", vin: "YV1XZ1234" } },
{ record_id: "a2", fields: { name: "Volvo Personvagnar Aktiebolag", city: "goteborg", vin: "YV1XZ1234" } },
{ record_id: "b1", fields: { name: "Ericsson AB", city: "stockholm", vin: "ERIC999" } },
{ record_id: "b2", fields: { name: "Eriksson Bygg AB", city: "stockholm", vin: "BYGG111" } },
];
function request(overrides: Record<string, unknown> = {}) {
return {
tenant_id: "t_a",
project_id: "p_1",
records: RECORDS,
blocking_fields: ["city"],
compare_fields: [{ field: "name", weight: 2 }, { field: "city" }],
...overrides,
};
}
describe("similarity primitives", () => {
it("normalize folds diacritics + punctuation (Göteborg ≡ goteborg — same block); similarity is 1 for equal, 0 for null", () => {
expect(normalize("Göteborg!")).toBe("goteborg");
expect(normalize("Göteborg")).toBe(normalize("goteborg"));
expect(normalize("Ørsted")).toBe("orsted");
expect(similarity("Volvo AB", "volvo ab")).toBe(1);
expect(similarity(null, "x")).toBe(0);
expect(similarity("abc", "xyz")).toBeLessThan(0.5);
});
});
describe("EntityResolutionService — candidate API", () => {
it("blocking limits comparisons to same-block pairs and reports the reduction", () => {
const er = new EntityResolutionService();
const res = er.generateCandidates(request());
// 4 records = 6 possible pairs; blocking on city → (a1,a2) + (b1,b2) = 2.
expect(res.total_possible_comparisons).toBe(6);
expect(res.blocked_comparisons).toBe(2);
expect(res.pairs).toHaveLength(2);
});
it("scores near-duplicates high (match) and different companies lower, with per-field explanations", () => {
const er = new EntityResolutionService();
const res = er.generateCandidates(request());
const volvo = res.pairs.find((p) => p.record_a === "a1")!;
const eric = res.pairs.find((p) => p.record_a === "b1")!;
expect(volvo.score).toBeGreaterThan(eric.score);
expect(volvo.decided_by).toBe("score");
// Explanation is per-field with weights (§10 explanation).
expect(volvo.explanation.map((e) => e.field)).toEqual(["name", "city"]);
expect(volvo.explanation[0].weight).toBe(2);
expect(volvo.explanation[1].similarity).toBe(1);
});
it("hands mid-band pairs to human review, never auto-merging them", () => {
const er = new EntityResolutionService();
const res = er.generateCandidates(request({ auto_match_threshold: 0.95, review_threshold: 0.3 }));
const eric = res.pairs.find((p) => p.record_a === "b1")!;
expect(eric.status).toBe("pending_review");
expect(res.review_queue.map((p) => p.candidate_id)).toContain(eric.candidate_id);
});
it("product hard rules override the generic score in BOTH directions (§10 hook)", () => {
const er = new EntityResolutionService();
er.registerHardRule({
rule_id: "vehiq.vin_equality",
decide: (a, b) => {
if (!a.fields.vin || !b.fields.vin) return "abstain";
return a.fields.vin === b.fields.vin ? "force_match" : "force_non_match";
},
});
const res = er.generateCandidates(request());
const volvo = res.pairs.find((p) => p.record_a === "a1")!;
const eric = res.pairs.find((p) => p.record_a === "b1")!;
expect(volvo.status).toBe("match");
expect(volvo.decided_by).toBe("vehiq.vin_equality");
expect(volvo.score).toBe(1);
// Same city block but different VIN → forced non-match despite name similarity.
expect(eric.status).toBe("non_match");
expect(eric.decided_by).toBe("vehiq.vin_equality");
});
it("refuses unblocked all-pairs runs and invalid thresholds", () => {
const er = new EntityResolutionService();
expect(() => er.generateCandidates(request({ blocking_fields: [] }))).toThrowError(/blocking_fields/);
expect(() =>
er.generateCandidates(request({ auto_match_threshold: 0.5, review_threshold: 0.6 }))
).toThrowError(/review_threshold/);
expect(() => er.generateCandidates(request({ records: [RECORDS[0]] }))).toThrowError(/two records/);
expect(() => er.generateCandidates(request({ tenant_id: "" }))).toThrowError(/tenant_id/);
});
it("records with empty blocking keys are never blocked together", () => {
const er = new EntityResolutionService();
const res = er.generateCandidates(
request({
records: [
{ record_id: "x1", fields: { name: "A", city: null } },
{ record_id: "x2", fields: { name: "A", city: null } },
],
})
);
expect(res.pairs).toHaveLength(0);
});
});
describe("EntityResolutionService — batch delegation (0pt-motor via N0D)", () => {
it("submits the Splink batch job with the records artifact as input (contract only, never faked)", async () => {
const { createN0dClient } = await import("@/src/clients/n0d-client");
const er = new EntityResolutionService({ n0dClient: createN0dClient() }); // test runtime → mock
const job = await er.submitBatchResolution({
tenant_id: "t_a",
project_id: "p_1",
records_artifact_ref: "st0re:artifact:art_records",
});
expect(job.job_id).toContain("mock_job_");
expect(job.job_type).toBe("0pt.er.batch_resolution");
});
it("without an N0D client the batch path fails hard (DEPENDENCY_UNAVAILABLE)", async () => {
const er = new EntityResolutionService();
await expect(
er.submitBatchResolution({ tenant_id: "t_a", project_id: "p_1", records_artifact_ref: "st0re:artifact:x" })
).rejects.toMatchObject({ code: "DEPENDENCY_UNAVAILABLE" });
});
});