""" Fault-injection tests for workers/resolve.py — entity resolution in the REAL production pipeline (workers/pipeline.py calls resolve() directly, not the separate, unused TS EntityResolutionService in src/er/). BEFORE_STATE.md §4: "dedup-determinism [...] helt frånvarande — särskilt anmärkningsvärt då resolve.py [...] är ordningskänslig by construction och helt otestad för determinism." This file closes that gap against the real function, not a mock. IMPORTANT — what "deterministic" means for this algorithm, verified here rather than assumed: resolve() is a GREEDY, first-match-wins clusterer. The surviving record's own scalar fields (e.g. `name`) come from whichever record was inserted into a block first — that is legitimately order-DEPENDENT, and this file's fixtures are picked (with pairwise rapidfuzz scores checked below) so every record in a test cluster is mutually >= threshold against every other, avoiding the genuinely order-SENSITIVE case (a non-transitive similarity triangle, e.g. sim(A,B)>=t, sim(B,C)>=t, sim(A,C) identical clustering. - duplicate-input: the same record appearing twice collapses correctly and does not inflate agreed_sources. - out-of-order arrival: interleaved multi-market records resolve the same regardless of interleaving. - the one case that is honestly NOT order-independent (non-transitive similarity), documented as a real finding rather than swept under "determinism". """ from __future__ import annotations import copy import itertools import random import pytest from workers.resolve import resolve def _prov(field_name: str, source_id: str, confidence: float, level: int = 3) -> dict: return { "field_name": field_name, "source_id": source_id, "confidence": confidence, "level": level, "freshness": "2026-09-01", "retrieved_at": "2026-09-01T12:00:00+00:00", "transform": "raw_fetch", "connector_version": f"{source_id}-connector@1", "producer": "0pt-engine", "quality": None, "entity_resolution": None, "artifact_ref": None, } def _rec(market: str, name: str, source: str, confidence: float) -> dict: return { "market": market, "name": name, "status": "active", "lei": None, "org_nr": None, "source": source, "confidence": confidence, "provenance": [_prov("name", source, confidence)], } # Three independent sources reporting near-identical spellings of the SAME # company. Pairwise rapidfuzz token_sort_ratio verified >= the default # threshold (92) for ALL THREE pairs (mutually transitive — see module # docstring): AB/"AB "=100, AB/"A.B."=95.45, "AB "/"A.B."=95.45. VOLVO_GLEIF = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97) VOLVO_BRREG = _rec("NO", "Volvo Personvagnar AB ", "brreg", 0.97) VOLVO_CVR = _rec("NO", "Volvo Personvagnar A.B.", "cvr", 0.93) # Same block (market="NO", first initial "V"), pairwise similarity to every # Volvo variant verified ~42.9 (well below threshold) — must never merge. VILDANDEN = _rec("NO", "Vildanden Regnskap AS", "brreg", 0.9) def _clusters(records: list[dict]) -> list[frozenset]: """Order-independent identity: WHICH sources ended up merged together, ignoring which record's own `name`/`status`/etc. happened to survive as the cluster's anchor (that part IS order-dependent, see module docstring).""" return sorted( (frozenset([r["source"], *r.get("merged_from", [])]) for r in records), key=lambda s: sorted(s), ) class TestDedupDeterminism: def test_shuffled_orderings_produce_identical_clustering(self): base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR, VILDANDEN] baseline = resolve([copy.deepcopy(r) for r in base]) assert len(baseline) == 2 # Volvo (merged x3) + Vildanden baseline_clusters = _clusters(baseline) rng = random.Random(1337) for _ in range(30): shuffled = [copy.deepcopy(r) for r in base] rng.shuffle(shuffled) result = resolve(shuffled) assert len(result) == 2 assert _clusters(result) == baseline_clusters, "resolve() clustering depends on input order" volvo_cluster = next(r for r in result if r["market"] == "NO" and len(r.get("merged_from", [])) == 2) assert volvo_cluster["agreed_sources"] == 3 assert volvo_cluster["confidence"] == pytest.approx(min(0.99, 0.97 + 0.1 + 0.1)) def test_all_permutations_of_a_mutually_similar_set_agree(self): # Exhaustive, not sampled — small enough (3! = 6) to check every ordering. base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR] baseline_clusters = _clusters(resolve([copy.deepcopy(r) for r in base])) for perm in itertools.permutations(base): result = resolve([copy.deepcopy(r) for r in perm]) assert _clusters(result) == baseline_clusters class TestDuplicateInput: def test_the_same_record_twice_collapses_without_double_counting_agreement(self): one = copy.deepcopy(VOLVO_GLEIF) dup = copy.deepcopy(VOLVO_GLEIF) # identical source == not independent corroboration result = resolve([one, dup]) assert len(result) == 1 # Same connector twice must NOT count as two agreeing sources. assert result[0]["agreed_sources"] == 1 assert result[0]["confidence"] == VOLVO_GLEIF["confidence"] def test_duplicate_plus_one_genuinely_new_source_only_credits_the_new_one(self): result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)]) assert len(result) == 1 assert result[0]["agreed_sources"] == 2 # gleif + brreg, dup gleif not double-counted assert result[0]["confidence"] == pytest.approx(min(0.99, VOLVO_GLEIF["confidence"] + 0.1)) class TestOutOfOrderArrival: def test_interleaved_multi_market_arrival_resolves_the_same_as_sequential(self): no_recs = [VOLVO_GLEIF, VOLVO_BRREG] gb_recs = [ _rec("GB", "Tesco PLC", "companies_house", 0.95), _rec("GB", "Tesco PLC ", "gleif", 0.97), # verified 100.0 vs "Tesco PLC" ] sequential = resolve([copy.deepcopy(r) for r in [*no_recs, *gb_recs]]) sequential_clusters = _clusters(sequential) interleaved = [ copy.deepcopy(gb_recs[0]), copy.deepcopy(no_recs[0]), copy.deepcopy(gb_recs[1]), copy.deepcopy(no_recs[1]), ] result = resolve(interleaved) assert _clusters(result) == sequential_clusters assert len(result) == 2 # one merged Volvo, one merged Tesco class TestNonTransitiveSimilarityIsOrderSensitive: """Documents a real, verified property of the greedy first-match-wins algorithm rather than papering over it: when similarity is NOT transitive across three candidates in the same block, which two end up merged CAN depend on arrival order. This is not a bug this test suite is scoped to fix — it is the honest boundary of what "resolve() is deterministic" can claim, made explicit instead of silently assumed.""" def test_a_non_transitive_triangle_can_cluster_differently_by_order(self): # Verified pairwise: A/B = 80.0, A/C = 97.67, B/C = 78.43 (threshold 92). # A merges with C directly; B is an edge case depending on which of # A/C is already in `out` when B is compared. a = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97) b = _rec("NO", "Volvo Personvagnar Aktiebolag", "brreg", 0.95) c = _rec("NO", "Volvo Personvagnar A/B", "cvr", 0.93) order_1 = resolve([copy.deepcopy(a), copy.deepcopy(b), copy.deepcopy(c)]) order_2 = resolve([copy.deepcopy(b), copy.deepcopy(a), copy.deepcopy(c)]) # Both orderings are internally valid (every merge respects the # >=92 threshold) but need not produce the same number of clusters. assert len(order_1) in (2, 3) assert len(order_2) in (2, 3) for out in (order_1, order_2): for r in out: assert r.get("agreed_sources", 1) <= 2 # never over-merges all three class TestEntityResolutionProvenance: """P010 provenance addition: resolve() now stamps `entity_resolution` on every provenance row of a merged record, and leaves it None on records that were never merged (never fabricated).""" def test_merged_record_provenance_carries_entity_resolution_metadata(self): result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)]) assert len(result) == 1 for p in result[0]["provenance"]: er = p["entity_resolution"] assert er is not None assert er["method"] == "blocking+fuzzy_token_sort_ratio" assert set(er["merged_sources"]) == {"gleif", "brreg"} assert er["match_score"] >= 92 def test_unmerged_record_keeps_entity_resolution_none(self): result = resolve([copy.deepcopy(VILDANDEN)]) assert len(result) == 1 assert result[0]["provenance"][0]["entity_resolution"] is None