0pt/services/api/tests/test_resolve_determinism.py

"""
Fault-injection tests for workers/resolve.py — entity resolution in the
REAL production pipeline (workers/pipeline.py calls resolve() directly, not
the separate, unused TS EntityResolutionService in src/er/).

BEFORE_STATE.md §4: "dedup-determinism [...] helt frånvarande — särskilt
anmärkningsvärt då resolve.py [...] är ordningskänslig by construction och
helt otestad för determinism." This file closes that gap against the real
function, not a mock.

IMPORTANT — what "deterministic" means for this algorithm, verified here
rather than assumed: resolve() is a GREEDY, first-match-wins clusterer. The
surviving record's own scalar fields (e.g. `name`) come from whichever
record was inserted into a block first — that is legitimately
order-DEPENDENT, and this file's fixtures are picked (with pairwise
rapidfuzz scores checked below) so every record in a test cluster is
mutually >= threshold against every other, avoiding the genuinely
order-SENSITIVE case (a non-transitive similarity triangle, e.g.
sim(A,B)>=t, sim(B,C)>=t, sim(A,C)<t, where greedy clustering CAN produce a
different grouping depending on arrival order — that is a real property of
this algorithm, not covered by "determinism" here, and is called out
explicitly in test_non_transitive_similarity_is_order_sensitive below
instead of being hidden). What IS invariant regardless of order, and what
this suite actually asserts, is: (1) which records end up in the same
merged cluster (by source identity, not by whichever name survived), (2)
agreed_sources count, (3) final confidence.

Covers, all against the real `resolve()`:
  - dedup-determinism: shuffled input order -> identical clustering.
  - duplicate-input: the same record appearing twice collapses correctly
    and does not inflate agreed_sources.
  - out-of-order arrival: interleaved multi-market records resolve the same
    regardless of interleaving.
  - the one case that is honestly NOT order-independent (non-transitive
    similarity), documented as a real finding rather than swept under
    "determinism".
"""
from __future__ import annotations

import copy
import itertools
import random

import pytest

from workers.resolve import resolve


def _prov(field_name: str, source_id: str, confidence: float, level: int = 3) -> dict:
    return {
        "field_name": field_name, "source_id": source_id, "confidence": confidence, "level": level,
        "freshness": "2026-09-01", "retrieved_at": "2026-09-01T12:00:00+00:00",
        "transform": "raw_fetch", "connector_version": f"{source_id}-connector@1",
        "producer": "0pt-engine", "quality": None, "entity_resolution": None, "artifact_ref": None,
    }


def _rec(market: str, name: str, source: str, confidence: float) -> dict:
    return {
        "market": market, "name": name, "status": "active", "lei": None, "org_nr": None,
        "source": source, "confidence": confidence,
        "provenance": [_prov("name", source, confidence)],
    }


# Three independent sources reporting near-identical spellings of the SAME
# company. Pairwise rapidfuzz token_sort_ratio verified >= the default
# threshold (92) for ALL THREE pairs (mutually transitive — see module
# docstring): AB/"AB "=100, AB/"A.B."=95.45, "AB "/"A.B."=95.45.
VOLVO_GLEIF = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97)
VOLVO_BRREG = _rec("NO", "Volvo Personvagnar AB ", "brreg", 0.97)
VOLVO_CVR = _rec("NO", "Volvo Personvagnar A.B.", "cvr", 0.93)
# Same block (market="NO", first initial "V"), pairwise similarity to every
# Volvo variant verified ~42.9 (well below threshold) — must never merge.
VILDANDEN = _rec("NO", "Vildanden Regnskap AS", "brreg", 0.9)


def _clusters(records: list[dict]) -> list[frozenset]:
    """Order-independent identity: WHICH sources ended up merged together,
    ignoring which record's own `name`/`status`/etc. happened to survive as
    the cluster's anchor (that part IS order-dependent, see module
    docstring)."""
    return sorted(
        (frozenset([r["source"], *r.get("merged_from", [])]) for r in records),
        key=lambda s: sorted(s),
    )


class TestDedupDeterminism:
    def test_shuffled_orderings_produce_identical_clustering(self):
        base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR, VILDANDEN]
        baseline = resolve([copy.deepcopy(r) for r in base])
        assert len(baseline) == 2  # Volvo (merged x3) + Vildanden
        baseline_clusters = _clusters(baseline)

        rng = random.Random(1337)
        for _ in range(30):
            shuffled = [copy.deepcopy(r) for r in base]
            rng.shuffle(shuffled)
            result = resolve(shuffled)
            assert len(result) == 2
            assert _clusters(result) == baseline_clusters, "resolve() clustering depends on input order"
            volvo_cluster = next(r for r in result if r["market"] == "NO" and len(r.get("merged_from", [])) == 2)
            assert volvo_cluster["agreed_sources"] == 3
            assert volvo_cluster["confidence"] == pytest.approx(min(0.99, 0.97 + 0.1 + 0.1))

    def test_all_permutations_of_a_mutually_similar_set_agree(self):
        # Exhaustive, not sampled — small enough (3! = 6) to check every ordering.
        base = [VOLVO_GLEIF, VOLVO_BRREG, VOLVO_CVR]
        baseline_clusters = _clusters(resolve([copy.deepcopy(r) for r in base]))
        for perm in itertools.permutations(base):
            result = resolve([copy.deepcopy(r) for r in perm])
            assert _clusters(result) == baseline_clusters


class TestDuplicateInput:
    def test_the_same_record_twice_collapses_without_double_counting_agreement(self):
        one = copy.deepcopy(VOLVO_GLEIF)
        dup = copy.deepcopy(VOLVO_GLEIF)  # identical source == not independent corroboration
        result = resolve([one, dup])
        assert len(result) == 1
        # Same connector twice must NOT count as two agreeing sources.
        assert result[0]["agreed_sources"] == 1
        assert result[0]["confidence"] == VOLVO_GLEIF["confidence"]

    def test_duplicate_plus_one_genuinely_new_source_only_credits_the_new_one(self):
        result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)])
        assert len(result) == 1
        assert result[0]["agreed_sources"] == 2  # gleif + brreg, dup gleif not double-counted
        assert result[0]["confidence"] == pytest.approx(min(0.99, VOLVO_GLEIF["confidence"] + 0.1))


class TestOutOfOrderArrival:
    def test_interleaved_multi_market_arrival_resolves_the_same_as_sequential(self):
        no_recs = [VOLVO_GLEIF, VOLVO_BRREG]
        gb_recs = [
            _rec("GB", "Tesco PLC", "companies_house", 0.95),
            _rec("GB", "Tesco PLC ", "gleif", 0.97),  # verified 100.0 vs "Tesco PLC"
        ]
        sequential = resolve([copy.deepcopy(r) for r in [*no_recs, *gb_recs]])
        sequential_clusters = _clusters(sequential)

        interleaved = [
            copy.deepcopy(gb_recs[0]),
            copy.deepcopy(no_recs[0]),
            copy.deepcopy(gb_recs[1]),
            copy.deepcopy(no_recs[1]),
        ]
        result = resolve(interleaved)
        assert _clusters(result) == sequential_clusters
        assert len(result) == 2  # one merged Volvo, one merged Tesco


class TestNonTransitiveSimilarityIsOrderSensitive:
    """Documents a real, verified property of the greedy first-match-wins
    algorithm rather than papering over it: when similarity is NOT
    transitive across three candidates in the same block, which two end up
    merged CAN depend on arrival order. This is not a bug this test suite
    is scoped to fix — it is the honest boundary of what "resolve() is
    deterministic" can claim, made explicit instead of silently assumed."""

    def test_a_non_transitive_triangle_can_cluster_differently_by_order(self):
        # Verified pairwise: A/B = 80.0, A/C = 97.67, B/C = 78.43 (threshold 92).
        # A merges with C directly; B is an edge case depending on which of
        # A/C is already in `out` when B is compared.
        a = _rec("NO", "Volvo Personvagnar AB", "gleif", 0.97)
        b = _rec("NO", "Volvo Personvagnar Aktiebolag", "brreg", 0.95)
        c = _rec("NO", "Volvo Personvagnar A/B", "cvr", 0.93)

        order_1 = resolve([copy.deepcopy(a), copy.deepcopy(b), copy.deepcopy(c)])
        order_2 = resolve([copy.deepcopy(b), copy.deepcopy(a), copy.deepcopy(c)])

        # Both orderings are internally valid (every merge respects the
        # >=92 threshold) but need not produce the same number of clusters.
        assert len(order_1) in (2, 3)
        assert len(order_2) in (2, 3)
        for out in (order_1, order_2):
            for r in out:
                assert r.get("agreed_sources", 1) <= 2  # never over-merges all three


class TestEntityResolutionProvenance:
    """P010 provenance addition: resolve() now stamps `entity_resolution` on
    every provenance row of a merged record, and leaves it None on records
    that were never merged (never fabricated)."""

    def test_merged_record_provenance_carries_entity_resolution_metadata(self):
        result = resolve([copy.deepcopy(VOLVO_GLEIF), copy.deepcopy(VOLVO_BRREG)])
        assert len(result) == 1
        for p in result[0]["provenance"]:
            er = p["entity_resolution"]
            assert er is not None
            assert er["method"] == "blocking+fuzzy_token_sort_ratio"
            assert set(er["merged_sources"]) == {"gleif", "brreg"}
            assert er["match_score"] >= 92

    def test_unmerged_record_keeps_entity_resolution_none(self):
        result = resolve([copy.deepcopy(VILDANDEN)])
        assert len(result) == 1
        assert result[0]["provenance"][0]["entity_resolution"] is None