fix(recovery): restore source removed by production documentation merge
This commit is contained in:
@@ -0,0 +1,21 @@
|
||||
# 0pt data engine — API + worker image (runs on a VPS, NOT Vercel; long jobs).
|
||||
FROM python:3.12-slim
|
||||
|
||||
# Commit-kedjan (samma mönster som 0pt-motor/scripts/build-images.sh): samma
|
||||
# SHA i image-taggen (sätts av byggaren), OCI-label och GET /health.
|
||||
ARG GIT_SHA=unknown
|
||||
LABEL org.opencontainers.image.revision=${GIT_SHA}
|
||||
ENV GIT_SHA=${GIT_SHA}
|
||||
|
||||
WORKDIR /app
|
||||
ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
|
||||
|
||||
COPY requirements.txt .
|
||||
RUN pip install -r requirements.txt
|
||||
|
||||
COPY . .
|
||||
|
||||
EXPOSE 8000
|
||||
# API: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000
|
||||
# Worker: python -m workers.run
|
||||
CMD ["uvicorn", "orchestrator.main:app", "--host", "0.0.0.0", "--port", "8000"]
|
||||
@@ -0,0 +1,34 @@
|
||||
# 0pt data engine (`services/api`)
|
||||
|
||||
The legal data motor (docs/06). FastAPI orchestrator + worker pipeline that turn
|
||||
**compliant** sources into delivered, provenance-tagged data. Runs on a VPS via
|
||||
[`../../infra/docker-compose.yml`](../../infra/docker-compose.yml) — **not on Vercel** (long jobs).
|
||||
|
||||
## Sourcing boundary (CLAUDE.md §1 — non-negotiable)
|
||||
|
||||
Per field × market, in strict order: **official API/bulk → authorized public web
|
||||
(robots + rate limits) → licensed partner data → otherwise level 0 "ej tillgänglig"**.
|
||||
We never invent values.
|
||||
|
||||
**Deliberately excluded (never add here):** WAF/bot-detection bypass, FlareSolverr,
|
||||
stealth/undetected browsers, proxy/fingerprint rotation to evade bans, scraping behind
|
||||
login walls, automated fake-account creation. This is the honest-accuracy moat.
|
||||
|
||||
## Layout
|
||||
- `sources/connectors.py` — compliant connectors (GLEIF, Brønnøysund, Companies House live; PRH/CVR/Bolagsverket gated on verified terms → level 0 until wired).
|
||||
- `sources/sources.yaml` — source registry (access/licence/cadence; VERIFY terms).
|
||||
- `orchestrator/main.py` — REST API (health, sources, estimates, orders, jobs) per docs/04.
|
||||
- `workers/adaptive_router.py` — picks the cheapest **compliant** method per market (api > bulk > authorized_web > licensed). No bypass tiers.
|
||||
- `workers/pipeline.py` — sourcing → extraction → structuring (dedup) → enrichment/fusion → QA (drops rows below the order's confidence target). Writes provenance per field.
|
||||
|
||||
## Run locally
|
||||
```bash
|
||||
pip install -r requirements.txt
|
||||
uvicorn orchestrator.main:app --reload # API on :8000
|
||||
python -m workers.run # run one demo job
|
||||
```
|
||||
|
||||
## Reconciliation note
|
||||
Supersedes the earlier `nordic-eu-data-platform` backend: the compliant connectors +
|
||||
pipeline are migrated here; the FlareSolverr/stealth/proxy/fake-account paths and the
|
||||
`test_bypass_chain`/`test_proxy_pool` examples are intentionally **not** carried over.
|
||||
@@ -0,0 +1 @@
|
||||
# 0pt data engine — orchestrator package (FastAPI app).
|
||||
@@ -0,0 +1,265 @@
|
||||
"""
|
||||
0pt data engine — FastAPI orchestrator (docs/04). Compliant sourcing only.
|
||||
Runs on a VPS/worker host, NOT on Vercel (long jobs). Amounts in öre.
|
||||
Reconciles the Next.js /api routes with the Python data engine: the Next layer
|
||||
is the customer-facing edge; this service runs the actual pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import hmac
|
||||
import os
|
||||
import uuid
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
import yaml
|
||||
from fastapi import FastAPI, HTTPException
|
||||
from fastapi.middleware.cors import CORSMiddleware
|
||||
from fastapi.responses import JSONResponse
|
||||
from pydantic import BaseModel
|
||||
|
||||
from sources.connectors import opensanctions_screen, rdw_vehicle
|
||||
from workers.pipeline import run_job
|
||||
|
||||
app = FastAPI(title="0pt data engine", version="0.1")
|
||||
app.add_middleware(
|
||||
CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
|
||||
)
|
||||
|
||||
_SOURCES = yaml.safe_load((Path(__file__).resolve().parent.parent / "sources" / "sources.yaml").read_text(encoding="utf-8"))["sources"]
|
||||
|
||||
|
||||
def _coerce_int(v: Any, default: int) -> int:
|
||||
"""int(v) raises on decimal-formatted strings/floats; coerce via float() first
|
||||
and fall back to `default` on any unparseable, client-controlled input instead
|
||||
of a 500."""
|
||||
try:
|
||||
return int(float(v))
|
||||
except (TypeError, ValueError):
|
||||
return default
|
||||
|
||||
# ── estimate model (mirrors apps/web lib/data.computeEstimate) ───────────────
|
||||
FIELD_RATE = 240
|
||||
ENTITY_MULT = {"Bolag": 1.0, "Beslutsfattare": 1.4, "Fordon": 1.2}
|
||||
FIELD_WEIGHT = {"identity": 1.0, "lei": 0.8, "financials": 1.6, "ubo": 1.4, "contact": 1.8, "intent": 1.5}
|
||||
ENRICH_WEIGHT = {"dedup": 0.1, "crm": 0.2, "sanctions": 0.15, "nace": 0.1, "tech": 0.15}
|
||||
|
||||
|
||||
class EstimateIn(BaseModel):
|
||||
markets: list[str] = []
|
||||
entity_type: str = "Bolag"
|
||||
fields: list[str] = []
|
||||
enrichment: list[str] = []
|
||||
volume: int = 1000
|
||||
frequency: str = "Engångs"
|
||||
confidence_target: int = 92
|
||||
|
||||
|
||||
def compute_estimate(b: EstimateIn) -> dict[str, Any]:
|
||||
markets = max(1, len(b.markets))
|
||||
fields = set(b.fields) | {"identity"}
|
||||
fw = sum(FIELD_WEIGHT.get(f, 0.5) for f in fields) or 1
|
||||
em = ENTITY_MULT.get(b.entity_type, 1.0)
|
||||
cm = 1 + max(0, b.confidence_target - 90) * 0.03
|
||||
ew = sum(ENRICH_WEIGHT.get(e, 0) for e in b.enrichment)
|
||||
base = fw * markets * FIELD_RATE * (b.volume / 1000) * em * cm
|
||||
price = round(base * (1 + ew) / 5) * 5
|
||||
deposit = round(price / 2)
|
||||
days = min(14, max(1, round(1 + markets * 0.4 + b.volume / 6000 + len(fields) * 0.25)))
|
||||
avg_level = sum(3 if f in ("identity", "lei") else 2 if f in ("financials", "ubo") else 1 for f in fields) / len(fields)
|
||||
ceiling = round(82 + avg_level * 5)
|
||||
est_conf = min(b.confidence_target, ceiling)
|
||||
return {
|
||||
"total_ore": price * 100, "deposit_ore": deposit * 100, "balance_ore": (price - deposit) * 100,
|
||||
"est_days": days, "est_confidence": est_conf / 100, "recurring": b.frequency != "Engångs",
|
||||
}
|
||||
|
||||
|
||||
# ── in-memory store (swapped for asyncpg when DATABASE_URL is set) ───────────
|
||||
_ORDERS: dict[str, dict[str, Any]] = {}
|
||||
_JOBS: dict[str, dict[str, Any]] = {}
|
||||
# P014: one asyncio.Lock per order_id, held for the duration of a /run call.
|
||||
# Process-local only (matches the honest in-memory-store limitation this file
|
||||
# already documents), but closes the actual race a bare `if oid in _JOBS`
|
||||
# check would leave open: two /run requests for the same order_id arriving
|
||||
# back to back both await run_job() before either has stored a result, so a
|
||||
# read-then-write check alone would let both through. See `run()` below.
|
||||
_JOB_LOCKS: dict[str, asyncio.Lock] = {}
|
||||
|
||||
|
||||
def _lock_for(oid: str) -> asyncio.Lock:
|
||||
lock = _JOB_LOCKS.get(oid)
|
||||
if lock is None:
|
||||
lock = asyncio.Lock()
|
||||
_JOB_LOCKS[oid] = lock
|
||||
return lock
|
||||
|
||||
|
||||
class OrderIn(BaseModel):
|
||||
flow: str = "custom"
|
||||
config: dict[str, Any] = {}
|
||||
|
||||
|
||||
# ── internal-token gate (fail-closed) ────────────────────────────────────────
|
||||
# The edge sends `X-Internal-Token` on every engineFetch (lib/engine.ts), but
|
||||
# until 2026-09-15 this service never VERIFIED it — a deployed engine accepted
|
||||
# anonymous traffic from anyone who could reach the port. The gate below
|
||||
# enforces it when INTERNAL_TOKEN is set: every route except /health (the
|
||||
# liveness probe) must carry the exact token or gets 401. When INTERNAL_TOKEN
|
||||
# is unset the gate is inert (local dev / docker compose without auth wiring);
|
||||
# the production deployment ALWAYS sets it (see infra/docker-compose.yml).
|
||||
_INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "").strip()
|
||||
_PUBLIC_PATHS = {"/health", "/docs", "/openapi.json", "/redoc"}
|
||||
|
||||
|
||||
class _InternalTokenGate:
|
||||
"""Raw-ASGI middleware: constant-time X-Internal-Token check, 401 on
|
||||
missing/wrong token. Written as a plain ASGI class (not
|
||||
@app.middleware('http')) so behavior does not depend on the
|
||||
BaseHTTPMiddleware dispatch quirks of the installed starlette."""
|
||||
|
||||
def __init__(self, app) -> None:
|
||||
self.app = app
|
||||
|
||||
async def __call__(self, scope, receive, send):
|
||||
if scope["type"] == "http" and _INTERNAL_TOKEN and scope["path"] not in _PUBLIC_PATHS:
|
||||
headers = {}
|
||||
for k, v in scope.get("headers") or []:
|
||||
headers.setdefault(k.decode("latin-1").lower(), v.decode("latin-1"))
|
||||
supplied = headers.get("x-internal-token", "")
|
||||
if not hmac.compare_digest(supplied, _INTERNAL_TOKEN):
|
||||
resp = JSONResponse(
|
||||
status_code=401,
|
||||
content={"error": {"code": "unauthorized", "message": "missing or invalid X-Internal-Token"}},
|
||||
)
|
||||
await resp(scope, receive, send)
|
||||
return
|
||||
await self.app(scope, receive, send)
|
||||
|
||||
|
||||
app.add_middleware(_InternalTokenGate)
|
||||
|
||||
|
||||
@app.get("/health")
|
||||
async def health():
|
||||
return {
|
||||
"ok": True,
|
||||
"service": "0pt-data-engine",
|
||||
"db": bool(os.getenv("DATABASE_URL")),
|
||||
"sources": len(_SOURCES),
|
||||
# Commit-kedjan: samma SHA som image-taggen/OCI-labeln (Dockerfile ARG
|
||||
# GIT_SHA). 'unknown' gör ett oprovenienserat image synligt istället
|
||||
# för tyst falskt — samma policy som 0pt-motor/scripts/verify-image.sh.
|
||||
"revision": os.getenv("GIT_SHA", "unknown"),
|
||||
}
|
||||
|
||||
|
||||
@app.get("/sources")
|
||||
async def sources():
|
||||
return [{k: s.get(k) for k in ("source_id", "name", "market", "access", "licence", "cadence")} for s in _SOURCES]
|
||||
|
||||
|
||||
@app.get("/sources/{market}")
|
||||
async def sources_for(market: str):
|
||||
return [s for s in _SOURCES if s.get("market") in (market, "global")]
|
||||
|
||||
|
||||
@app.get("/vehicles/{plate}")
|
||||
async def vehicle(plate: str):
|
||||
try:
|
||||
v = await rdw_vehicle(plate)
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=503, detail="vehicle source unavailable") from exc
|
||||
return {**(v or {"vehicle": None}), "source_status": "ok"}
|
||||
|
||||
|
||||
@app.get("/screen")
|
||||
async def screen(q: str):
|
||||
try:
|
||||
result = await opensanctions_screen(q)
|
||||
return {**result, "source_status": "ok"}
|
||||
except Exception as exc:
|
||||
raise HTTPException(status_code=503, detail="screening source unavailable") from exc
|
||||
|
||||
|
||||
@app.post("/estimates")
|
||||
async def estimates(body: EstimateIn):
|
||||
return compute_estimate(body)
|
||||
|
||||
|
||||
@app.post("/orders", status_code=201)
|
||||
async def create_order(body: OrderIn):
|
||||
est = compute_estimate(EstimateIn(
|
||||
markets=body.config.get("markets", []),
|
||||
entity_type=body.config.get("type", "Bolag"),
|
||||
fields=body.config.get("fieldIds", []),
|
||||
enrichment=body.config.get("enrichmentIds", []),
|
||||
volume=_coerce_int(body.config.get("volume", 1000), 1000),
|
||||
frequency=body.config.get("frequency", "Engångs"),
|
||||
confidence_target=_coerce_int(body.config.get("confidence", 92), 92),
|
||||
))
|
||||
oid = str(uuid.uuid4())
|
||||
_ORDERS[oid] = {"order_id": oid, "flow": body.flow, "status": "draft", "config": body.config, **est}
|
||||
return {"order_id": oid, "status": "draft", "deposit_ore": est["deposit_ore"], "total_ore": est["total_ore"]}
|
||||
|
||||
|
||||
@app.get("/orders")
|
||||
async def list_orders():
|
||||
return list(_ORDERS.values())
|
||||
|
||||
|
||||
@app.get("/orders/{oid}")
|
||||
async def get_order(oid: str):
|
||||
o = _ORDERS.get(oid)
|
||||
if not o:
|
||||
raise HTTPException(404, "order saknas")
|
||||
return {**o, "job": _JOBS.get(oid)}
|
||||
|
||||
|
||||
class StatusIn(BaseModel):
|
||||
status: str
|
||||
|
||||
|
||||
@app.patch("/orders/{oid}")
|
||||
async def set_status(oid: str, body: StatusIn):
|
||||
o = _ORDERS.get(oid)
|
||||
if not o:
|
||||
raise HTTPException(404, "order saknas")
|
||||
o["status"] = body.status
|
||||
return o
|
||||
|
||||
|
||||
@app.post("/orders/{oid}/run")
|
||||
async def run(oid: str):
|
||||
"""
|
||||
P014: idempotent delivery. An order should be fulfilled/delivered ONCE.
|
||||
Before this fix, calling this endpoint twice for the same order_id (a
|
||||
retried request, a duplicate trigger, an operator double-click) re-ran
|
||||
the full compliant sourcing pipeline a second time and silently
|
||||
overwrote `_JOBS[oid]` with a second, independently-produced result --
|
||||
a genuinely different delivery (a second live fetch against upstream
|
||||
registries, a second QA pass with its own confidence draws), completely
|
||||
indistinguishable from the first at the API surface. This is the actual
|
||||
"duplicate delivery" gap P014's recon flagged as NOT MET: nothing
|
||||
downstream of order creation was idempotent at all. A job already
|
||||
recorded for this order_id is now returned unchanged; `run_job` is never
|
||||
invoked twice for the same order in this process's lifetime. The lock
|
||||
(see `_lock_for` above) closes the race between two requests arriving
|
||||
concurrently, not just sequential retries.
|
||||
"""
|
||||
o = _ORDERS.get(oid)
|
||||
if not o:
|
||||
raise HTTPException(404, "order saknas")
|
||||
async with _lock_for(oid):
|
||||
if oid in _JOBS:
|
||||
return _JOBS[oid]
|
||||
o["status"] = "in_production"
|
||||
_JOBS[oid] = await run_job(oid, o["config"])
|
||||
o["status"] = "review"
|
||||
return _JOBS[oid]
|
||||
|
||||
|
||||
@app.get("/orders/{oid}/status")
|
||||
async def job_status(oid: str):
|
||||
return _JOBS.get(oid) or {"stage": "queued", "pct": 0, "state": "queued"}
|
||||
@@ -0,0 +1,11 @@
|
||||
[pytest]
|
||||
# services/api's modules import each other as top-level packages
|
||||
# (`from workers.pipeline import run_job`, `from sources.connectors import ...`)
|
||||
# relative to this directory — orchestrator/main.py does the same. Running
|
||||
# pytest from services/api (or with this file as rootdir) needs that on
|
||||
# sys.path; pytest 8's rootdir-relative testpaths + this ini being IN
|
||||
# services/api handles that via pytest's automatic rootdir insertion for
|
||||
# rootdir-relative imports when there is no conftest.py package structure.
|
||||
testpaths = tests
|
||||
asyncio_mode = auto
|
||||
pythonpath = .
|
||||
@@ -0,0 +1,7 @@
|
||||
# Test-only dependencies for services/api. NOT installed in the production
|
||||
# image (Dockerfile only installs requirements.txt) — see pytest.ini for how
|
||||
# the test suite is run locally / in CI.
|
||||
-r requirements.txt
|
||||
pytest==8.3.4
|
||||
pytest-asyncio==0.25.2
|
||||
respx==0.22.0
|
||||
@@ -0,0 +1,10 @@
|
||||
# 0pt data engine — compliant stack only (docs/07). No bypass/stealth/proxy libs.
|
||||
fastapi==0.115.6
|
||||
uvicorn[standard]==0.34.0
|
||||
pydantic==2.10.4
|
||||
httpx==0.28.1
|
||||
asyncpg==0.30.0
|
||||
python-dotenv==1.0.1
|
||||
pyyaml==6.0.2
|
||||
rapidfuzz==3.11.0
|
||||
tenacity==9.0.0
|
||||
@@ -0,0 +1 @@
|
||||
# 0pt data engine — sources package (connectors + registry).
|
||||
@@ -0,0 +1,319 @@
|
||||
"""
|
||||
0pt data engine — COMPLIANT source connectors.
|
||||
|
||||
Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market:
|
||||
1. official API / bulk 2. authorized public web (robots + rate limits)
|
||||
3. licensed partner data 4. otherwise -> level 0 "ej tillgänglig / licens".
|
||||
|
||||
EXPLICITLY NOT PRESENT (and must never be added here):
|
||||
WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint
|
||||
rotation to evade bans, scraping behind login walls, fake-account creation.
|
||||
|
||||
Every connector returns a normalized company dict + per-field provenance rows
|
||||
so nothing ships unsourced. `Provenance` (below) carries the full field set
|
||||
BEFORE_STATE.md §3 found missing from the actual dataflow — this repo's
|
||||
5-field bridge_provenance was: source, confidence, level, and a DATE-only
|
||||
freshness; the concept "bridge_provenance" as a named thing did not exist
|
||||
anywhere in the repo at all. This adds, per field: retrieval TIMESTAMP
|
||||
(distinct from the legacy date-only `freshness`), the transform applied,
|
||||
connector/schema version, the producing system, and two slots
|
||||
(`quality`/`entity_resolution`) that downstream stages (quality bridge,
|
||||
resolve.py) fill in as the record moves through the pipeline — they start
|
||||
`None` here and are never backfilled with a guess.
|
||||
Terms change — VERIFY each source's current access/licence before relying.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from datetime import datetime, timezone
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
# A polite client that identifies itself and obeys timeouts/rate limits.
|
||||
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
|
||||
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)
|
||||
|
||||
# Bumped when a connector's parsing/field-mapping logic changes in a way that
|
||||
# could shift output shape — recorded per-field as `Provenance.connector_version`
|
||||
# so a downstream consumer can tell which parsing logic produced a value.
|
||||
CONNECTOR_VERSIONS: dict[str, str] = {
|
||||
"gleif": "gleif-connector@1",
|
||||
"brreg": "brreg-connector@1",
|
||||
"companies_house": "companies-house-connector@1",
|
||||
"prh": "prh-connector@1",
|
||||
"cvr": "cvr-connector@1",
|
||||
"rdw": "rdw-connector@1",
|
||||
}
|
||||
|
||||
PRODUCER = "0pt-engine"
|
||||
|
||||
# P014: statuses a compliant source uses to signal "stop asking me this way"
|
||||
# (rate-limited, forbidden, or a legal/geo block) rather than "this record
|
||||
# doesn't exist". Collapsing these into an ordinary None (as every connector
|
||||
# did before this pass) is not itself a compliance violation -- nothing here
|
||||
# ever escalates to a bypass -- but it silently discarded a real signal:
|
||||
# adaptive_router.py had no way to know a source had just told it to back
|
||||
# off, so nothing could avoid hammering a rate-limited endpoint on the very
|
||||
# next record. See SourceBlockedError and adaptive_router.record_blocked.
|
||||
BLOCKED_STATUSES = frozenset({401, 403, 429, 451})
|
||||
|
||||
|
||||
class SourceBlockedError(Exception):
|
||||
"""
|
||||
Raised by a connector when the source responds with one of
|
||||
BLOCKED_STATUSES. This is informational, never an instruction to retry
|
||||
harder or escalate -- callers (pipeline.py's _fetch loop) catch it only
|
||||
to record the block via adaptive_router.record_blocked and move on to
|
||||
the next compliant source in the plan, exactly as they already would for
|
||||
a plain None. Compliance stance (CLAUDE.md §1) is unchanged: a block is
|
||||
always a stop signal for THAT source, never a cue to bypass it.
|
||||
"""
|
||||
|
||||
def __init__(self, source_id: str, status_code: int):
|
||||
self.source_id = source_id
|
||||
self.status_code = status_code
|
||||
super().__init__(f"{source_id} signaled a block (HTTP {status_code})")
|
||||
|
||||
|
||||
def _raise_if_blocked(source_id: str, response: httpx.Response) -> None:
|
||||
if response.status_code in BLOCKED_STATUSES:
|
||||
raise SourceBlockedError(source_id, response.status_code)
|
||||
|
||||
|
||||
def _client() -> httpx.AsyncClient:
|
||||
return httpx.AsyncClient(
|
||||
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
|
||||
timeout=_TIMEOUT,
|
||||
follow_redirects=True,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class Provenance:
|
||||
field_name: str
|
||||
source_id: str
|
||||
confidence: float
|
||||
level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
|
||||
freshness: str | None = None # legacy: retrieval DATE only (kept for back-compat)
|
||||
retrieved_at: str | None = None # full ISO-8601 UTC timestamp of the actual fetch
|
||||
transform: str | None = None # e.g. "raw_fetch", "normalized", "entity_resolution_merge"
|
||||
connector_version: str | None = None # e.g. "brreg-connector@1" — see CONNECTOR_VERSIONS
|
||||
producer: str = PRODUCER # which system produced this field's value
|
||||
quality: dict[str, Any] | None = None # filled by the quality bridge, if/when it runs (never guessed)
|
||||
entity_resolution: dict[str, Any] | None = None # filled by resolve.py on merge, never at fetch time
|
||||
artifact_ref: str | None = None # delivery artifact reference (ST0RE), set at delivery time — not yet wired
|
||||
|
||||
|
||||
def make_provenance(
|
||||
field_name: str,
|
||||
source_id: str,
|
||||
confidence: float,
|
||||
level: int,
|
||||
*,
|
||||
transform: str = "raw_fetch",
|
||||
) -> Provenance:
|
||||
"""Factory used by every connector below so the new provenance fields
|
||||
(retrieved_at, connector_version, producer) are populated consistently
|
||||
instead of each call site re-deriving them ad hoc."""
|
||||
now = datetime.now(timezone.utc)
|
||||
return Provenance(
|
||||
field_name=field_name,
|
||||
source_id=source_id,
|
||||
confidence=confidence,
|
||||
level=level,
|
||||
freshness=now.date().isoformat(),
|
||||
retrieved_at=now.isoformat(),
|
||||
transform=transform,
|
||||
connector_version=CONNECTOR_VERSIONS.get(source_id),
|
||||
producer=PRODUCER,
|
||||
)
|
||||
|
||||
|
||||
@dataclass
|
||||
class CompanyResult:
|
||||
org_nr: str | None = None
|
||||
market: str | None = None
|
||||
lei: str | None = None
|
||||
name: str | None = None
|
||||
status: str | None = None
|
||||
address: dict[str, Any] | None = None
|
||||
website: str | None = None
|
||||
provenance: list[Provenance] = field(default_factory=list)
|
||||
source: str = ""
|
||||
|
||||
|
||||
# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
|
||||
async def gleif_by_lei(lei: str) -> CompanyResult | None:
|
||||
url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
|
||||
_raise_if_blocked("gleif", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
a = (r.json().get("data") or {}).get("attributes") or {}
|
||||
entity = a.get("entity") or {}
|
||||
legal = entity.get("legalAddress") or {}
|
||||
res = CompanyResult(
|
||||
lei=lei,
|
||||
name=(entity.get("legalName") or {}).get("name"),
|
||||
status=entity.get("status"),
|
||||
address={"lines": legal.get("addressLines"), "city": legal.get("city"),
|
||||
"country": legal.get("country"), "postal": legal.get("postalCode")},
|
||||
source="gleif",
|
||||
)
|
||||
res.provenance += [
|
||||
make_provenance("lei", "gleif", 0.99, 3),
|
||||
make_provenance("name", "gleif", 0.97, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
|
||||
# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ─────────────
|
||||
async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
|
||||
url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("brreg", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
res = CompanyResult(
|
||||
org_nr=orgnr, market="NO",
|
||||
name=d.get("navn"),
|
||||
status="dissolved" if d.get("konkurs") else "active",
|
||||
address={"raw": d.get("forretningsadresse")},
|
||||
source="brreg",
|
||||
)
|
||||
res.provenance += [
|
||||
make_provenance("name", "brreg", 0.97, 3),
|
||||
make_provenance("status", "brreg", 0.95, 3),
|
||||
make_provenance("address", "brreg", 0.95, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
|
||||
# ── Companies House (UK) — free API, key required. VERIFY terms. ────────────
|
||||
async def companies_house_by_number(number: str) -> CompanyResult | None:
|
||||
key = os.getenv("COMPANIES_HOUSE_API_KEY")
|
||||
if not key:
|
||||
# No key -> deliver honestly as unavailable, never fabricate.
|
||||
return CompanyResult(org_nr=number, market="GB", source="companies_house",
|
||||
provenance=[make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")])
|
||||
url = f"https://api.company-information.service.gov.uk/company/{number}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url, auth=(key, ""))
|
||||
_raise_if_blocked("companies_house", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
res = CompanyResult(
|
||||
org_nr=number, market="GB",
|
||||
name=d.get("company_name"),
|
||||
status=d.get("company_status"),
|
||||
address={"raw": d.get("registered_office_address")},
|
||||
source="companies_house",
|
||||
)
|
||||
res.provenance += [
|
||||
make_provenance("name", "companies_house", 0.97, 3),
|
||||
make_provenance("status", "companies_house", 0.95, 3),
|
||||
]
|
||||
return res
|
||||
|
||||
|
||||
# ── PRH avoindata (FI) — open API (CC-BY). VERIFY terms. ────────────────────
|
||||
async def prh_by_businessid(business_id: str) -> CompanyResult | None:
|
||||
url = f"https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId={business_id}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("prh", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
items = r.json().get("companies") or r.json().get("results") or []
|
||||
if not items:
|
||||
return None
|
||||
d = items[0]
|
||||
names = d.get("names") or []
|
||||
name = names[0].get("name") if names else d.get("name")
|
||||
res = CompanyResult(org_nr=business_id, market="FI", name=name, status=d.get("status"), source="prh")
|
||||
if name:
|
||||
res.provenance.append(make_provenance("name", "prh", 0.95, 3))
|
||||
return res
|
||||
|
||||
|
||||
# ── CVR (DK) via cvrapi.dk — free for low volume (descriptive UA). VERIFY. ───
|
||||
async def cvr_by_number(cvr: str) -> CompanyResult | None:
|
||||
url = f"https://cvrapi.dk/api?search={cvr}&country=dk"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
_raise_if_blocked("cvr", r)
|
||||
if r.status_code != 200:
|
||||
return None
|
||||
d = r.json()
|
||||
if not d.get("name"):
|
||||
return None
|
||||
res = CompanyResult(
|
||||
org_nr=cvr, market="DK", name=d.get("name"),
|
||||
status="ceased" if d.get("enddate") else "active",
|
||||
address={"raw": d.get("address")}, source="cvr",
|
||||
)
|
||||
res.provenance += [make_provenance("name", "cvr", 0.93, 3), make_provenance("address", "cvr", 0.9, 2)]
|
||||
return res
|
||||
|
||||
|
||||
# ── RDW (NL) open vehicle data — Socrata. Technical/fleet only, no owner PII. ─
|
||||
async def rdw_vehicle(kenteken: str) -> dict[str, Any] | None:
|
||||
plate = kenteken.upper().replace("-", "").replace(" ", "")
|
||||
url = f"https://opendata.rdw.nl/resource/m9d7-ebf2.json?kenteken={plate}"
|
||||
async with _client() as c:
|
||||
r = await c.get(url)
|
||||
r.raise_for_status()
|
||||
arr = r.json()
|
||||
if not isinstance(arr, list) or any(not isinstance(item, dict) for item in arr):
|
||||
raise ValueError("invalid RDW response")
|
||||
if not arr:
|
||||
return None
|
||||
d = arr[0]
|
||||
p = make_provenance("make", "rdw", 0.9, 3)
|
||||
return {
|
||||
"reg_plate": kenteken, "market": "NL",
|
||||
"make": d.get("merk"), "model": d.get("handelsbenaming"),
|
||||
"year": (d.get("datum_eerste_toelating") or "")[:4] or None,
|
||||
"source": "rdw",
|
||||
# Dict shape (not the Provenance dataclass) — this endpoint predates it
|
||||
# and is returned directly as an API response body, not merged through
|
||||
# resolve.py. Field names kept aligned with Provenance's for consistency.
|
||||
"provenance": [{
|
||||
"field": p.field_name, "source_id": p.source_id, "confidence": p.confidence, "level": p.level,
|
||||
"freshness": p.freshness, "retrieved_at": p.retrieved_at, "transform": p.transform,
|
||||
"connector_version": p.connector_version, "producer": p.producer,
|
||||
}],
|
||||
}
|
||||
|
||||
|
||||
# ── OpenSanctions screening — search API. VERIFY commercial terms / key. ─────
|
||||
async def opensanctions_screen(name: str) -> dict[str, Any]:
|
||||
url = "https://api.opensanctions.org/search/default"
|
||||
async with _client() as c:
|
||||
r = await c.get(url, params={"q": name, "limit": 5})
|
||||
r.raise_for_status()
|
||||
payload = r.json()
|
||||
if not isinstance(payload, dict) or not isinstance(payload.get("results"), list) or any(
|
||||
not isinstance(item, dict) for item in payload["results"]
|
||||
):
|
||||
raise ValueError("invalid OpenSanctions response")
|
||||
results = payload["results"]
|
||||
return {
|
||||
"source": "opensanctions",
|
||||
"matches": [{"name": x.get("caption"), "score": x.get("score"), "schema": x.get("schema")} for x in results],
|
||||
}
|
||||
|
||||
|
||||
CONNECTORS = {
|
||||
"gleif": gleif_by_lei,
|
||||
"brreg": brreg_by_orgnr,
|
||||
"companies_house": companies_house_by_number,
|
||||
"prh": prh_by_businessid,
|
||||
"cvr": cvr_by_number,
|
||||
}
|
||||
@@ -0,0 +1,34 @@
|
||||
"""
|
||||
Structured financials (docs/06). XBRL/iXBRL is the fast, exact lane (not OCR).
|
||||
Companies House publishes accounts as iXBRL; this fetches the accounts filing
|
||||
history (real, key-gated) and documents the XBRL extraction step.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX)"
|
||||
|
||||
|
||||
async def companies_house_filings(number: str) -> dict[str, Any]:
|
||||
key = os.getenv("COMPANIES_HOUSE_API_KEY")
|
||||
if not key:
|
||||
return {"company": number, "filings": [], "note": "level0: kräver COMPANIES_HOUSE_API_KEY"}
|
||||
url = f"https://api.company-information.service.gov.uk/company/{number}/filing-history?category=accounts"
|
||||
async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}, timeout=15) as c:
|
||||
r = await c.get(url, auth=(key, ""))
|
||||
if r.status_code != 200:
|
||||
return {"company": number, "filings": []}
|
||||
items = r.json().get("items", [])
|
||||
return {
|
||||
"company": number,
|
||||
"source": "companies_house",
|
||||
"filings": [{"date": i.get("date"), "type": i.get("type"), "description": i.get("description")} for i in items[:10]],
|
||||
}
|
||||
|
||||
# XBRL extraction (extension): with `arelle` installed, download each accounts
|
||||
# filing's iXBRL document and pull tagged facts (turnover, profit, employees)
|
||||
# into fact_financials with provenance. Structured, exact — never OCR for XBRL.
|
||||
@@ -0,0 +1,99 @@
|
||||
# 0pt data engine — sources registry (docs/06). Drives the router, /sources API,
|
||||
# and the public sourcing-map. access: api | bulk | authorized_web | licensed.
|
||||
# VERIFY current access/licence/robots per source before relying — terms change.
|
||||
sources:
|
||||
- source_id: gleif
|
||||
name: GLEIF (LEI)
|
||||
market: global
|
||||
access: api
|
||||
cadence: daily
|
||||
licence: "open (CC0) — VERIFY"
|
||||
entities: [company]
|
||||
fields: [lei, legal_name, address, status]
|
||||
base_url: https://api.gleif.org
|
||||
robots_ok: true
|
||||
notes: "cross-border key"
|
||||
|
||||
- source_id: brreg
|
||||
name: Brønnøysundregistrene (Enhetsregisteret)
|
||||
market: NO
|
||||
access: api
|
||||
cadence: daily
|
||||
licence: "NLOD open — VERIFY"
|
||||
entities: [company]
|
||||
fields: [name, status, address, nace]
|
||||
base_url: https://data.brreg.no
|
||||
robots_ok: true
|
||||
notes: "open API + bulk download"
|
||||
|
||||
- source_id: companies_house
|
||||
name: Companies House
|
||||
market: GB
|
||||
access: api
|
||||
cadence: daily
|
||||
licence: "free, API key required — VERIFY"
|
||||
entities: [company, officers, financials]
|
||||
fields: [name, status, address, officers]
|
||||
base_url: https://api.company-information.service.gov.uk
|
||||
robots_ok: true
|
||||
notes: "key via env COMPANIES_HOUSE_API_KEY"
|
||||
|
||||
- source_id: prh
|
||||
name: PRH avoindata (Trade Register)
|
||||
market: FI
|
||||
access: api
|
||||
cadence: daily
|
||||
licence: "CC-BY — VERIFY"
|
||||
entities: [company]
|
||||
fields: [name, status]
|
||||
base_url: https://avoindata.prh.fi
|
||||
robots_ok: true
|
||||
notes: "connector pending verification"
|
||||
|
||||
- source_id: cvr
|
||||
name: CVR / Virk
|
||||
market: DK
|
||||
access: api
|
||||
cadence: daily
|
||||
licence: "terms apply (Virk) — VERIFY"
|
||||
entities: [company]
|
||||
fields: [name, status]
|
||||
base_url: https://datacvr.virk.dk
|
||||
robots_ok: true
|
||||
notes: "connector pending verification"
|
||||
|
||||
- source_id: bolagsverket
|
||||
name: Bolagsverket
|
||||
market: SE
|
||||
access: licensed
|
||||
cadence: daily
|
||||
licence: "restricted for bulk — partner/licensed route"
|
||||
entities: [company]
|
||||
fields: [name, status]
|
||||
base_url: https://bolagsverket.se
|
||||
robots_ok: true
|
||||
notes: "no bulk scrape; licensed/partner only"
|
||||
|
||||
- source_id: rdw
|
||||
name: RDW open vehicle data
|
||||
market: NL
|
||||
access: bulk
|
||||
cadence: daily
|
||||
licence: "open — VERIFY"
|
||||
entities: [vehicle]
|
||||
fields: [make, model, year, specs]
|
||||
base_url: https://opendata.rdw.nl
|
||||
robots_ok: true
|
||||
notes: "technical/fleet vehicle data, not owner PII"
|
||||
|
||||
- source_id: opensanctions
|
||||
name: OpenSanctions
|
||||
market: global
|
||||
access: bulk
|
||||
cadence: daily
|
||||
licence: "open — VERIFY commercial terms"
|
||||
entities: [company, person]
|
||||
fields: [sanction, pep]
|
||||
base_url: https://www.opensanctions.org
|
||||
robots_ok: true
|
||||
notes: "screening/diligence enrichment"
|
||||
@@ -0,0 +1,77 @@
|
||||
"""
|
||||
P014: adaptive_router.py previously had ZERO test coverage and a docstring
|
||||
("all methods, no compliance restrictions. See b0ts for scraping") that
|
||||
directly contradicted this product's own compliance stance and pointed at a
|
||||
forbidden, separate, non-compliant repo. This file tests the corrected
|
||||
version: plan_for_market's static per-market plan, and the new block-
|
||||
tracking (record_blocked/is_blocked/cooldown) that makes the engine actually
|
||||
back off a source that just told it to stop, instead of retrying it with no
|
||||
memory at all.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
from workers.adaptive_router import AdaptiveRouter, MARKET_SOURCES, plan_for_market
|
||||
|
||||
|
||||
class TestPlanForMarket:
|
||||
def test_returns_the_full_static_plan_when_nothing_is_blocked(self):
|
||||
router = AdaptiveRouter()
|
||||
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
|
||||
|
||||
def test_unknown_market_falls_back_to_gleif_only(self):
|
||||
router = AdaptiveRouter()
|
||||
assert plan_for_market("ZZ", router=router) == ["gleif"]
|
||||
|
||||
def test_defaults_to_the_process_wide_router_when_none_is_passed(self):
|
||||
# Just proves the call doesn't blow up and returns a real plan --
|
||||
# not asserting anything about global state here (other tests use
|
||||
# their own injected router precisely to avoid cross-test pollution
|
||||
# of the shared default_router).
|
||||
assert plan_for_market("FI") == MARKET_SOURCES["FI"]
|
||||
|
||||
|
||||
class TestBlockTracking:
|
||||
def test_a_freshly_blocked_source_is_excluded_from_the_plan(self):
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "companies_house")
|
||||
plan = plan_for_market("GB", router=router)
|
||||
assert "companies_house" not in plan
|
||||
assert "gleif" in plan # only the blocked source is removed, not the whole market
|
||||
|
||||
def test_is_blocked_is_false_before_any_block_is_recorded(self):
|
||||
router = AdaptiveRouter()
|
||||
assert router.is_blocked("GB", "companies_house") is False
|
||||
|
||||
def test_blocking_is_scoped_per_market_not_global_to_the_source_id(self):
|
||||
# gleif appears in every market's plan -- blocking it for GB must not
|
||||
# silently remove it from FI/SE/etc too.
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "gleif")
|
||||
assert "gleif" not in plan_for_market("GB", router=router)
|
||||
assert "gleif" in plan_for_market("FI", router=router)
|
||||
assert "gleif" in plan_for_market("SE", router=router)
|
||||
|
||||
def test_cooldown_expires_and_the_source_becomes_eligible_again(self):
|
||||
router = AdaptiveRouter(cooldown_seconds=60.0)
|
||||
router.record_blocked("GB", "companies_house", now=1000.0)
|
||||
assert router.is_blocked("GB", "companies_house", now=1030.0) is True # still within cooldown
|
||||
assert router.is_blocked("GB", "companies_house", now=1061.0) is False # cooldown elapsed
|
||||
# And the plan reflects that:
|
||||
assert "companies_house" in plan_for_market("GB", router=router)
|
||||
|
||||
def test_clear_forgets_every_recorded_block(self):
|
||||
router = AdaptiveRouter()
|
||||
router.record_blocked("GB", "companies_house")
|
||||
router.record_blocked("NO", "brreg")
|
||||
router.clear()
|
||||
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
|
||||
assert plan_for_market("NO", router=router) == MARKET_SOURCES["NO"]
|
||||
|
||||
def test_two_router_instances_never_share_state(self):
|
||||
# Regression guard for a shared-mutable-default bug class: each
|
||||
# AdaptiveRouter() must own its own dict, not a class-level one.
|
||||
a = AdaptiveRouter()
|
||||
b = AdaptiveRouter()
|
||||
a.record_blocked("GB", "companies_house")
|
||||
assert a.is_blocked("GB", "companies_house") is True
|
||||
assert b.is_blocked("GB", "companies_house") is False
|
||||
@@ -0,0 +1,104 @@
|
||||
"""
|
||||
P014: every connector used by pipeline.py's adaptive plan (gleif, brreg,
|
||||
companies_house, prh, cvr) used to collapse EVERY non-200 response into a
|
||||
bare `None`, indistinguishable from "record genuinely doesn't exist". This
|
||||
tests that the block-status subset (401/403/429/451) now raises
|
||||
SourceBlockedError instead, while every other non-200 status (404, 500, ...)
|
||||
still returns None exactly as before -- this is an ADDITIVE signal, not a
|
||||
behavior change for the non-block case.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from sources.connectors import (
|
||||
SourceBlockedError,
|
||||
brreg_by_orgnr,
|
||||
companies_house_by_number,
|
||||
cvr_by_number,
|
||||
gleif_by_lei,
|
||||
prh_by_businessid,
|
||||
)
|
||||
|
||||
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/5493001KJTIIGC8Y1R12"
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
|
||||
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
|
||||
CVR_URL = "https://cvrapi.dk/api?search=10150817&country=dk"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
@pytest.mark.parametrize("status", [401, 403, 429, 451])
|
||||
async def test_gleif_raises_source_blocked_on_block_statuses(status):
|
||||
respx.get(GLEIF_URL).mock(return_value=httpx.Response(status, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await gleif_by_lei("5493001KJTIIGC8Y1R12")
|
||||
assert exc_info.value.source_id == "gleif"
|
||||
assert exc_info.value.status_code == status
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_gleif_a_404_still_returns_none_not_blocked(monkeypatch):
|
||||
respx.get(GLEIF_URL).mock(return_value=httpx.Response(404, json={}))
|
||||
result = await gleif_by_lei("5493001KJTIIGC8Y1R12")
|
||||
assert result is None # unchanged pre-P014 behavior for a genuine "not found"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_brreg_raises_source_blocked_on_429():
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(429, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await brreg_by_orgnr("923609016")
|
||||
assert exc_info.value.source_id == "brreg"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_companies_house_raises_source_blocked_on_403(monkeypatch):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(403, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await companies_house_by_number("00000006")
|
||||
assert exc_info.value.source_id == "companies_house"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_companies_house_missing_key_still_delivers_honest_level_0_not_a_block():
|
||||
# No COMPANIES_HOUSE_API_KEY set -- this is a config gap, not a source
|
||||
# block, and must keep returning the honest "unavailable" result it
|
||||
# already did (never raise SourceBlockedError for a problem on OUR side).
|
||||
result = await companies_house_by_number("00000006")
|
||||
assert result is not None
|
||||
assert result.provenance[0].level == 0
|
||||
assert result.provenance[0].transform == "unavailable_no_key"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_prh_raises_source_blocked_on_401():
|
||||
respx.get(PRH_URL).mock(return_value=httpx.Response(401, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await prh_by_businessid("2077474-0")
|
||||
assert exc_info.value.source_id == "prh"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_cvr_raises_source_blocked_on_451():
|
||||
respx.get(CVR_URL).mock(return_value=httpx.Response(451, json={}))
|
||||
with pytest.raises(SourceBlockedError) as exc_info:
|
||||
await cvr_by_number("10150817")
|
||||
assert exc_info.value.source_id == "cvr"
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_cvr_a_500_still_returns_none_not_blocked():
|
||||
respx.get(CVR_URL).mock(return_value=httpx.Response(500, json={}))
|
||||
result = await cvr_by_number("10150817")
|
||||
assert result is None
|
||||
@@ -0,0 +1,58 @@
|
||||
"""
|
||||
Verifies the P010 provenance additions (BEFORE_STATE.md §3: source,
|
||||
retrieval timestamp, transform, version, producer were absent from the real
|
||||
dataflow) land correctly on every real connector's output — mocked HTTP,
|
||||
real parsing code.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from sources.connectors import brreg_by_orgnr, make_provenance, CONNECTOR_VERSIONS
|
||||
|
||||
|
||||
class TestMakeProvenanceFactory:
|
||||
def test_populates_all_new_fields_honestly_and_leaves_the_rest_none(self):
|
||||
p = make_provenance("name", "brreg", 0.97, 3)
|
||||
assert p.field_name == "name"
|
||||
assert p.source_id == "brreg"
|
||||
assert p.retrieved_at is not None
|
||||
assert p.freshness == p.retrieved_at[:10] # date prefix of the same instant
|
||||
assert p.transform == "raw_fetch"
|
||||
assert p.connector_version == CONNECTOR_VERSIONS["brreg"]
|
||||
assert p.producer == "0pt-engine"
|
||||
# Never guessed — these are filled by later pipeline stages, not at fetch time.
|
||||
assert p.quality is None
|
||||
assert p.entity_resolution is None
|
||||
assert p.artifact_ref is None
|
||||
|
||||
def test_transform_override_is_honored(self):
|
||||
p = make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")
|
||||
assert p.transform == "unavailable_no_key"
|
||||
|
||||
def test_unknown_source_id_has_no_fabricated_version(self):
|
||||
p = make_provenance("x", "some_future_source", 0.5, 1)
|
||||
assert p.connector_version is None # honestly unknown, not guessed
|
||||
|
||||
|
||||
class TestBrregConnectorProvenance:
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_real_parsing_path_produces_fully_populated_provenance(self):
|
||||
respx.get("https://data.brreg.no/enhetsregisteret/api/enheter/923609016").mock(
|
||||
return_value=httpx.Response(200, json={
|
||||
"navn": "Volvo Personvagnar Norge AS", "konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"]},
|
||||
})
|
||||
)
|
||||
res = await brreg_by_orgnr("923609016")
|
||||
assert res is not None
|
||||
assert len(res.provenance) == 3 # name, status, address
|
||||
for p in res.provenance:
|
||||
assert p.source_id == "brreg"
|
||||
assert p.connector_version == "brreg-connector@1"
|
||||
assert p.producer == "0pt-engine"
|
||||
assert p.retrieved_at is not None
|
||||
assert p.transform == "raw_fetch"
|
||||
@@ -0,0 +1,93 @@
|
||||
from workers.entity_resolution import (
|
||||
EntityCandidate,
|
||||
ResolutionDecision,
|
||||
rank_resolution_candidates,
|
||||
resolve_entity_pair,
|
||||
)
|
||||
|
||||
|
||||
def organization(entity_id: str, **overrides):
|
||||
values = {
|
||||
"entity_id": entity_id,
|
||||
"entity_type": "organization",
|
||||
"name": "Example Technology AB",
|
||||
"source_ref": f"source:{entity_id}",
|
||||
"registration_country": "SE",
|
||||
"registration_number": None,
|
||||
"website": None,
|
||||
"city": "Stockholm",
|
||||
}
|
||||
values.update(overrides)
|
||||
return EntityCandidate(**values)
|
||||
|
||||
|
||||
def person(entity_id: str, **overrides):
|
||||
values = {
|
||||
"entity_id": entity_id,
|
||||
"entity_type": "person",
|
||||
"name": "Anna Andersson",
|
||||
"source_ref": f"source:{entity_id}",
|
||||
"email": None,
|
||||
"organization_id": None,
|
||||
}
|
||||
values.update(overrides)
|
||||
return EntityCandidate(**values)
|
||||
|
||||
|
||||
def test_registration_number_is_a_strong_organization_identifier():
|
||||
result = resolve_entity_pair(
|
||||
organization("left", registration_number="556123-4567"),
|
||||
organization("right", registration_number="5561234567"),
|
||||
)
|
||||
assert result.decision is ResolutionDecision.EXACT_LINK
|
||||
assert result.score == 1.0
|
||||
assert result.requires_human_review is False
|
||||
|
||||
|
||||
def test_conflicting_registration_numbers_fail_distinct_even_when_names_match():
|
||||
result = resolve_entity_pair(
|
||||
organization("left", registration_number="5561234567"),
|
||||
organization("right", registration_number="5569999999"),
|
||||
)
|
||||
assert result.decision is ResolutionDecision.DISTINCT
|
||||
assert result.score == 0.0
|
||||
|
||||
|
||||
def test_same_person_email_is_exact_link():
|
||||
result = resolve_entity_pair(
|
||||
person("left", email="Anna@example.com"),
|
||||
person("right", email=" anna@EXAMPLE.com "),
|
||||
)
|
||||
assert result.decision is ResolutionDecision.EXACT_LINK
|
||||
assert result.requires_human_review is False
|
||||
|
||||
|
||||
def test_fuzzy_organization_similarity_never_auto_merges():
|
||||
result = resolve_entity_pair(
|
||||
organization("left", name="Example Technology AB", website="example.se"),
|
||||
organization("right", name="Example Technologies AB", website="https://www.example.se"),
|
||||
)
|
||||
assert result.score > 0.72
|
||||
assert result.decision is ResolutionDecision.REVIEW
|
||||
assert result.requires_human_review is True
|
||||
|
||||
|
||||
def test_different_entity_types_are_distinct():
|
||||
result = resolve_entity_pair(
|
||||
organization("org"),
|
||||
person("person"),
|
||||
)
|
||||
assert result.decision is ResolutionDecision.DISTINCT
|
||||
|
||||
|
||||
def test_candidate_ranking_prefers_exact_then_review_and_is_stable():
|
||||
query = organization("query", registration_number="5561234567", website="example.se")
|
||||
candidates = [
|
||||
organization("weak", name="Completely Different Company", registration_country="NO", city="Oslo"),
|
||||
organization("review", name="Example Technologies AB", website="example.se"),
|
||||
organization("exact", registration_number="556123-4567"),
|
||||
]
|
||||
ranked = rank_resolution_candidates(query, candidates)
|
||||
assert [result.right_id for result in ranked] == ["exact", "review", "weak"]
|
||||
assert ranked[0].decision is ResolutionDecision.EXACT_LINK
|
||||
assert ranked[1].decision is ResolutionDecision.REVIEW
|
||||
@@ -0,0 +1,70 @@
|
||||
"""The engine must not accept anonymous traffic when INTERNAL_TOKEN is set.
|
||||
|
||||
Background (2026-09-15): lib/engine.ts on the edge has always SENT
|
||||
`X-Internal-Token`, but the engine never verified it — a deployed engine would
|
||||
have accepted anyone who could reach the port. The gate is fail-closed:
|
||||
|
||||
* INTERNAL_TOKEN set -> every route except /health requires the EXACT
|
||||
token; wrong/missing = 401.
|
||||
* INTERNAL_TOKEN unset -> gate inert (local dev); production compose always
|
||||
sets it.
|
||||
|
||||
`/health` stays public on purpose: it is the liveness probe that
|
||||
`lib/health-checks.ts` calls from the edge, and it leaks nothing but booleans.
|
||||
|
||||
The token is read at module import (the same value the running process serves
|
||||
with), so each test reloads the module with the env it wants — and the
|
||||
teardown reloads it one more time with the env CLEARED so later test files in
|
||||
the same session never inherit a baked-in token.
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import importlib
|
||||
import os
|
||||
|
||||
import pytest
|
||||
from fastapi.testclient import TestClient
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def engine(monkeypatch):
|
||||
monkeypatch.setenv("INTERNAL_TOKEN", "s3cret")
|
||||
import orchestrator.main as main
|
||||
|
||||
importlib.reload(main)
|
||||
yield TestClient(main.app)
|
||||
os.environ.pop("INTERNAL_TOKEN", None)
|
||||
importlib.reload(main)
|
||||
|
||||
|
||||
@pytest.fixture()
|
||||
def engine_no_token(monkeypatch):
|
||||
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
|
||||
import orchestrator.main as main
|
||||
|
||||
importlib.reload(main)
|
||||
yield TestClient(main.app)
|
||||
os.environ.pop("INTERNAL_TOKEN", None)
|
||||
importlib.reload(main)
|
||||
|
||||
|
||||
def test_gate_off_when_internal_token_unset(engine_no_token):
|
||||
assert engine_no_token.get("/health").status_code == 200
|
||||
assert engine_no_token.get("/sources").status_code == 200
|
||||
|
||||
|
||||
def test_health_stays_public_but_sources_require_token(engine):
|
||||
# liveness probe is public by design
|
||||
assert engine.get("/health").status_code == 200
|
||||
# anonymous call to a data route -> 401 (was 200 before 2026-09-15)
|
||||
r = engine.get("/sources")
|
||||
assert r.status_code == 401
|
||||
assert r.json()["error"]["code"] == "unauthorized"
|
||||
|
||||
|
||||
def test_wrong_token_is_401_and_correct_token_passes(engine):
|
||||
wrong = engine.get("/estimates?markets=SE", headers={"X-Internal-Token": "nope"})
|
||||
assert wrong.status_code == 401
|
||||
right = engine.get("/sources", headers={"X-Internal-Token": "s3cret"})
|
||||
assert right.status_code == 200
|
||||
@@ -0,0 +1,53 @@
|
||||
"""
|
||||
Honest documentation of two of the seven requested failure-test categories
|
||||
that are NOT implemented as real fault-injection tests in this pass, and
|
||||
why — matching this repo's established convention of stating a gap in code
|
||||
(BEFORE_STATE.md, docs/P0_DATASET_QUALITY_FLOW.md) rather than silently
|
||||
dropping it or faking coverage. These are `pytest.skip` markers, not
|
||||
disguised passes, so they show up in the test report as explicitly
|
||||
unaddressed rather than vanishing from view.
|
||||
|
||||
* CDC-interruption: this repo has NO change-data-capture pipeline
|
||||
anywhere — no Debezium/WAL-slot consumer, no CDC producer, no
|
||||
replication-slot code, confirmed by a full-repo search
|
||||
(`grep -rn "debezium\\|wal_level\\|replication.slot\\|cdc" services/ src/`
|
||||
finds nothing on this topic). A "CDC-interruption" fault-injection
|
||||
test needs a CDC stream to interrupt; fabricating one to have
|
||||
something to test would be building fictional infrastructure and
|
||||
testing the fiction, not this codebase — the exact failure mode this
|
||||
project's CLAUDE.md and BEFORE_STATE.md explicitly warn against
|
||||
("never fabricate").
|
||||
* True backfill idempotency (dedup against a delivery store): the
|
||||
pipeline has no persistence layer at all (see workers/pipeline.py —
|
||||
every run_job() call recomputes from scratch, nothing is written to a
|
||||
database). "Backfill doesn't create duplicate deliveries" requires a
|
||||
delivery store to check duplicates against, which doesn't exist.
|
||||
test_pipeline_fault_injection.py::TestBackfillReplay covers the
|
||||
narrower, real property that DOES hold today (replaying an identical
|
||||
order config is compute-idempotent — byte-identical output, no drift)
|
||||
and documents this exact distinction inline.
|
||||
"""
|
||||
import pytest
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"No CDC pipeline exists anywhere in this repo (verified by full-repo "
|
||||
"search) — nothing to interrupt without inventing fictional "
|
||||
"infrastructure. See this file's module docstring."
|
||||
)
|
||||
)
|
||||
def test_cdc_interruption_not_implementable_against_real_code():
|
||||
pass
|
||||
|
||||
|
||||
@pytest.mark.skip(
|
||||
reason=(
|
||||
"No delivery-store persistence layer exists to dedupe a backfill "
|
||||
"against — the compute-idempotency property that DOES exist is "
|
||||
"covered for real in test_pipeline_fault_injection.py::"
|
||||
"TestBackfillReplay. See this file's module docstring."
|
||||
)
|
||||
)
|
||||
def test_true_backfill_idempotency_not_implementable_against_real_code():
|
||||
pass
|
||||
@@ -0,0 +1,112 @@
|
||||
"""
|
||||
P014: services/api/orchestrator/main.py had ZERO test coverage before this
|
||||
pass, and `POST /orders/{oid}/run` had no idempotency protection at all --
|
||||
calling it twice for the same order re-ran the full compliant sourcing
|
||||
pipeline a second time, silently overwriting `_JOBS[oid]` with a second,
|
||||
independently-produced delivery. This is the actual "duplicate delivery"
|
||||
gap P014's recon flagged NOT MET (nothing downstream of order creation was
|
||||
idempotent). These tests exercise the real ASGI app end to end (httpx
|
||||
ASGITransport, no mocked HTTP layer) and prove: a sequential retry returns
|
||||
the cached job without re-running the pipeline, a TRUE CONCURRENT race
|
||||
(two requests in flight at once) is also closed by the per-order asyncio
|
||||
lock, and an unknown order still 404s exactly as before.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
|
||||
from orchestrator.main import app
|
||||
|
||||
|
||||
def _fake_job_result(calls: list[str], order_id: str) -> dict:
|
||||
calls.append(order_id)
|
||||
return {
|
||||
"stage": "qa",
|
||||
"pct": 100,
|
||||
"state": "done",
|
||||
"metrics": {"rows": 1, "dropped_below_threshold": 0, "per_market": {"NO": 1}, "avg_confidence": 0.95},
|
||||
"records": [{"market": "NO", "name": "Test AS", "confidence": 0.95}],
|
||||
}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_sequential_retry_returns_cached_job_without_rerunning_pipeline(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
|
||||
oid = created.json()["order_id"]
|
||||
|
||||
r1 = await client.post(f"/orders/{oid}/run")
|
||||
r2 = await client.post(f"/orders/{oid}/run")
|
||||
r3 = await client.post(f"/orders/{oid}/run")
|
||||
|
||||
assert r1.status_code == 200
|
||||
assert r1.json() == r2.json() == r3.json()
|
||||
assert calls == [oid] # run_job invoked exactly once across three /run calls
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_true_concurrent_race_still_invokes_run_job_exactly_once(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
# Widen the race window: both requests reach this await before
|
||||
# either has written to _JOBS, which is exactly the race a bare
|
||||
# `if oid in _JOBS` check (no lock) would lose.
|
||||
await asyncio.sleep(0.05)
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
|
||||
oid = created.json()["order_id"]
|
||||
|
||||
r1, r2 = await asyncio.gather(
|
||||
client.post(f"/orders/{oid}/run"),
|
||||
client.post(f"/orders/{oid}/run"),
|
||||
)
|
||||
|
||||
assert r1.status_code == 200 and r2.status_code == 200
|
||||
assert r1.json() == r2.json()
|
||||
assert calls == [oid] # THE regression this test guards: exactly one real pipeline run
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_run_on_unknown_order_still_404s():
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
res = await client.post("/orders/does-not-exist/run")
|
||||
assert res.status_code == 404
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_different_orders_each_get_their_own_real_run(monkeypatch):
|
||||
calls: list[str] = []
|
||||
|
||||
async def fake_run_job(order_id: str, config: dict) -> dict:
|
||||
return _fake_job_result(calls, order_id)
|
||||
|
||||
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
|
||||
|
||||
transport = httpx.ASGITransport(app=app)
|
||||
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
|
||||
o1 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})).json()["order_id"]
|
||||
o2 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["GB"]}})).json()["order_id"]
|
||||
|
||||
await client.post(f"/orders/{o1}/run")
|
||||
await client.post(f"/orders/{o2}/run")
|
||||
|
||||
# Idempotency is per-order, not global -- two distinct orders both really run.
|
||||
assert sorted(calls) == sorted([o1, o2])
|
||||
@@ -0,0 +1,162 @@
|
||||
"""
|
||||
Fault-injection tests for workers/pipeline.py's run_job — the REAL order
|
||||
pipeline (not a mock of it). BEFORE_STATE.md §4 found schema-evolution,
|
||||
CDC-interruption, backfill, and restart entirely untested. This file closes
|
||||
what can be closed against code that actually exists, and says explicitly
|
||||
why the rest can't be, instead of fabricating a harness for infrastructure
|
||||
this repo doesn't have.
|
||||
|
||||
All external HTTP is mocked with respx (never hits real registries) so this
|
||||
suite is deterministic and offline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.pipeline import run_job
|
||||
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
|
||||
|
||||
VALID_BRREG_BODY = {
|
||||
"navn": "Volvo Personvagnar Norge AS",
|
||||
"konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"], "poststed": "OSLO"},
|
||||
}
|
||||
|
||||
|
||||
class TestRestartMidFetch:
|
||||
"""A worker crash/restart mid-job is not literally simulate-able (no
|
||||
checkpointing exists to resume from — the pipeline is a single async
|
||||
function with no persisted intermediate state). What this DOES test,
|
||||
honestly: a connector raising mid-job (network reset, upstream 5xx,
|
||||
connection drop — the kind of fault a real restart would also surface
|
||||
as) never crashes the whole order; the job completes and delivers
|
||||
whatever it validly could."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_connector_exception_does_not_crash_the_order(self):
|
||||
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("connection reset"))
|
||||
respx.get(PRH_URL).mock(return_value=httpx.Response(200, json={
|
||||
"companies": [{"names": [{"name": "Suomi Oy"}], "status": "active"}],
|
||||
}))
|
||||
result = await run_job(
|
||||
"order_restart_1",
|
||||
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
# NO failed entirely (connector raised); FI still delivered.
|
||||
assert result["metrics"]["per_market"]["NO"] == 0
|
||||
assert result["metrics"]["per_market"]["FI"] == 1
|
||||
assert len(result["records"]) == 1
|
||||
assert result["records"][0]["market"] == "FI"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_every_connector_failing_delivers_zero_rows_not_a_crash(self):
|
||||
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("down"))
|
||||
respx.get(PRH_URL).mock(side_effect=httpx.ConnectError("down"))
|
||||
result = await run_job(
|
||||
"order_restart_2",
|
||||
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == []
|
||||
assert result["metrics"]["rows"] == 0
|
||||
|
||||
|
||||
class TestSchemaEvolution:
|
||||
"""Upstream registries adding/removing/renaming fields without notice is
|
||||
a real risk (BEFORE_STATE.md flagged this as wholly untested). Verified
|
||||
against the actual HTTP-parsing boundary in sources/connectors.py, via
|
||||
respx-mocked responses shaped like plausible upstream drift — not a
|
||||
fabricated schema-registry harness (this repo has no schema
|
||||
versioning/CDC system to test against; see test_known_gaps.py)."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_adding_unexpected_new_fields_does_not_break_parsing(self):
|
||||
drifted_body = {
|
||||
**VALID_BRREG_BODY,
|
||||
# Simulated schema drift: brreg adds fields this connector never asked for.
|
||||
"stiftelsesdato": "2020-01-01",
|
||||
"naeringskode1": {"kode": "45.11", "beskrivelse": "Handel"},
|
||||
"nested_new_thing": {"a": {"b": [1, 2, 3]}},
|
||||
}
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=drifted_body))
|
||||
result = await run_job(
|
||||
"order_schema_1",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert len(result["records"]) == 1
|
||||
assert result["records"][0]["name"] == "Volvo Personvagnar Norge AS"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_removing_an_expected_field_degrades_honestly_not_a_crash(self):
|
||||
# Real drift case: upstream drops "navn" (renamed/moved). The
|
||||
# connector must not crash, and must not fabricate a name — the
|
||||
# record is simply not produced (pipeline's existing `if res and
|
||||
# res.name` gate), which is the correct "level 0 / not available"
|
||||
# behavior, not a bug to work around.
|
||||
stripped_body = {k: v for k, v in VALID_BRREG_BODY.items() if k != "navn"}
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=stripped_body))
|
||||
result = await run_job(
|
||||
"order_schema_2",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == [] # honestly nothing delivered, not a fabricated name
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_upstream_returning_a_wholly_different_shape_does_not_crash(self):
|
||||
# Worst case: upstream returns something structurally alien (e.g. a
|
||||
# maintenance-page JSON blob instead of a company record).
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json={"error": "maintenance", "retry_after": 60}))
|
||||
result = await run_job(
|
||||
"order_schema_3",
|
||||
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
|
||||
)
|
||||
assert result["state"] == "done"
|
||||
assert result["records"] == []
|
||||
|
||||
|
||||
class TestBackfillReplay:
|
||||
"""This pipeline has NO persistence layer (BEFORE_STATE.md — every
|
||||
run_job call recomputes from scratch; there is no delivery store to
|
||||
dedupe a backfill against). A true "backfill doesn't create duplicate
|
||||
deliveries" test would need that store and is NOT fabricated here. What
|
||||
IS real and testable: replaying the identical order config (the exact
|
||||
scenario a backfill re-run performs) is idempotent AT THE COMPUTE LEVEL
|
||||
— same input, byte-identical output — so a backfill re-run at least
|
||||
doesn't silently drift or become order-dependent across replays."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_replaying_the_same_order_config_yields_byte_identical_delivery(self):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
config = {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}}
|
||||
|
||||
first = await run_job("order_backfill", config)
|
||||
second = await run_job("order_backfill", config)
|
||||
|
||||
# Strip the per-run artifact_ref/timestamps (legitimately differ —
|
||||
# every replay IS a new delivery event) before comparing the
|
||||
# substantive content.
|
||||
def strip_run_identity(result):
|
||||
recs = []
|
||||
for r in result["records"]:
|
||||
rec = {k: v for k, v in r.items()}
|
||||
rec["provenance"] = [
|
||||
{k: v for k, v in p.items() if k not in ("retrieved_at", "artifact_ref")} for p in rec["provenance"]
|
||||
]
|
||||
recs.append(rec)
|
||||
return recs
|
||||
|
||||
assert strip_run_identity(first) == strip_run_identity(second)
|
||||
assert first["metrics"]["rows"] == second["metrics"]["rows"] == 1
|
||||
@@ -0,0 +1,114 @@
|
||||
"""
|
||||
P014: integration proof that a source signaling a block (HTTP 401/403/429/451)
|
||||
during a real run_job() call (a) never crashes or fabricates a result, (b) is
|
||||
recorded on the process-wide adaptive_router so the engine backs off that
|
||||
source, and (c) the pipeline still delivers from the next compliant source in
|
||||
the plan -- exactly the "source-racing several authorized sources" behavior
|
||||
the siax-0pt skill documents, now with an honest memory of blocks instead of
|
||||
silently re-trying a rate-limited source with none.
|
||||
|
||||
Uses the real default_router (module-level singleton pipeline.py actually
|
||||
uses), cleared before and after every test in this file so this suite never
|
||||
leaks state into -- or picks up state from -- any other test module that
|
||||
imports workers.pipeline.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.adaptive_router import default_router
|
||||
from workers.pipeline import run_job
|
||||
|
||||
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
|
||||
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/00000006"
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_router():
|
||||
default_router.clear()
|
||||
yield
|
||||
default_router.clear()
|
||||
|
||||
|
||||
class TestSourceBlockedDuringARealRun:
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_429_from_one_source_is_recorded_as_blocked_and_the_run_still_delivers_from_the_next_source(
|
||||
self, monkeypatch
|
||||
):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={"error": "rate limited"}))
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(
|
||||
200,
|
||||
json={"data": {"attributes": {"entity": {"legalName": {"name": "Fallback Ltd"}, "status": "ACTIVE", "legalAddress": {}}}}},
|
||||
)
|
||||
)
|
||||
|
||||
assert default_router.is_blocked("GB", "companies_house") is False # sanity: clean slate
|
||||
|
||||
result = await run_job("order_block_1", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert result["metrics"]["rows"] == 1
|
||||
assert result["records"][0]["name"] == "Fallback Ltd"
|
||||
assert result["records"][0]["source"] == "gleif" # companies_house was blocked, gleif answered
|
||||
|
||||
# THE regression this test guards: the block was actually recorded,
|
||||
# not silently discarded the way a bare `except Exception: res = None`
|
||||
# would (that's still the fallback for genuine errors, but a block is
|
||||
# now distinguishable from one).
|
||||
assert default_router.is_blocked("GB", "companies_house") is True
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_blocked_source_is_excluded_from_the_plan_on_the_very_next_key_in_the_same_run(self, monkeypatch):
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
gb_second_url = "https://api.company-information.service.gov.uk/company/11111111"
|
||||
gleif_second_url = "https://api.gleif.org/api/v1/lei-records/11111111"
|
||||
|
||||
companies_house_calls = respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={}))
|
||||
# No mock registered for gb_second_url at all -- if the pipeline tried
|
||||
# companies_house AGAIN for the second key after it was just blocked,
|
||||
# respx would raise on the unmocked request and this test would fail
|
||||
# with a connection error, not a clean assertion. That absence IS the
|
||||
# assertion: the plan excluded the just-blocked source.
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "First"}, "legalAddress": {}}}}})
|
||||
)
|
||||
respx.get(gleif_second_url).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "Second"}, "legalAddress": {}}}}})
|
||||
)
|
||||
|
||||
result = await run_job(
|
||||
"order_block_2",
|
||||
{"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006", "11111111"]}},
|
||||
)
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert result["metrics"]["rows"] == 2
|
||||
names = {r["name"] for r in result["records"]}
|
||||
assert names == {"First", "Second"}
|
||||
# companies_house was hit exactly once (for the first key, where it
|
||||
# got blocked) -- never retried for the second key in this run.
|
||||
assert companies_house_calls.call_count == 1
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_a_plain_5xx_is_not_treated_as_a_block(self, monkeypatch):
|
||||
# A genuine server error is a different signal from a compliance
|
||||
# block -- it should NOT be recorded via record_blocked (that would
|
||||
# incorrectly suppress a source that's just having a bad moment, not
|
||||
# one that told the engine to stop).
|
||||
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
|
||||
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(500, json={}))
|
||||
respx.get(GLEIF_URL).mock(
|
||||
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "X"}, "legalAddress": {}}}}})
|
||||
)
|
||||
|
||||
result = await run_job("order_block_3", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
|
||||
|
||||
assert result["state"] == "done"
|
||||
assert default_router.is_blocked("GB", "companies_house") is False
|
||||
@@ -0,0 +1,164 @@
|
||||
"""
|
||||
Tests for workers/quality_bridge.py (the Python half of the P010 TS/Python
|
||||
bridge) and its wiring into workers/pipeline.run_job. Verifies the
|
||||
fail-open contract end to end: disabled, not configured, timeout,
|
||||
non-2xx, malformed body, and success — and, critically, that `delivered`
|
||||
rows from run_job are IDENTICAL regardless of which of those the bridge
|
||||
call resolves to (the bridge is advisory, never a second gate).
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.pipeline import run_job
|
||||
from workers.quality_bridge import run_quality_bridge
|
||||
|
||||
BRIDGE_URL = "http://edge.local/api/0pt/quality/bridge-run"
|
||||
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
|
||||
|
||||
VALID_BRREG_BODY = {
|
||||
"navn": "Volvo Personvagnar Norge AS",
|
||||
"konkurs": False,
|
||||
"forretningsadresse": {"adresse": ["Storgata 1"]},
|
||||
}
|
||||
|
||||
|
||||
@pytest.fixture(autouse=True)
|
||||
def _clean_env(monkeypatch):
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_URL", raising=False)
|
||||
monkeypatch.delenv("EDGE_URL", raising=False)
|
||||
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
|
||||
|
||||
|
||||
class TestBridgeDirectly:
|
||||
@pytest.mark.asyncio
|
||||
async def test_disabled_by_default_makes_zero_network_calls(self, monkeypatch):
|
||||
# No respx mock registered at all — if this made an HTTP call it would raise.
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "disabled"
|
||||
assert result.score is None
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_enabled_but_no_url_configured(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "not_configured"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_successful_run_is_parsed(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
|
||||
"run": {"run_id": "dqrun_1", "status": "succeeded", "score": 100},
|
||||
"issues": [],
|
||||
}))
|
||||
result = await run_quality_bridge(
|
||||
tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[{"name": "X", "confidence": 0.9}]
|
||||
)
|
||||
assert result.status == "ok"
|
||||
assert result.score == 100
|
||||
assert result.run_id == "dqrun_1"
|
||||
assert result.issue_count == 0
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_timeout_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.TimeoutException("timed out"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
assert result.detail == "timeout"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_non_2xx_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(500, json={"error": "boom"}))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
assert "500" in result.detail
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_malformed_body_fails_open_not_raises(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, content=b"not json"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_connection_refused_fails_open(self, monkeypatch):
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("refused"))
|
||||
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
|
||||
assert result.status == "unavailable"
|
||||
|
||||
|
||||
class TestBridgeWiredIntoThePipeline:
|
||||
"""The part that matters most: whatever the bridge does, `delivered`
|
||||
(the rows the customer actually receives) must not change."""
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_missing_tenant_id_skips_the_bridge_honestly(self):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
result = await run_job("order_notenant", {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}})
|
||||
assert result["metrics"]["quality_bridge"]["status"] == "not_configured"
|
||||
assert len(result["records"]) == 1 # delivery unaffected
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_bridge_disabled_delivers_identically_to_bridge_enabled_and_failing(self, monkeypatch):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
config = {
|
||||
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
|
||||
"tenant_id": "t_bridge_pipeline", "project_id": "p_1",
|
||||
}
|
||||
|
||||
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
|
||||
disabled_result = await run_job("order_bridge_off", config)
|
||||
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("edge unreachable"))
|
||||
failing_result = await run_job("order_bridge_fail", config)
|
||||
|
||||
assert disabled_result["metrics"]["rows"] == failing_result["metrics"]["rows"] == 1
|
||||
assert [r["name"] for r in disabled_result["records"]] == [r["name"] for r in failing_result["records"]]
|
||||
assert disabled_result["metrics"]["quality_bridge"]["status"] == "disabled"
|
||||
assert failing_result["metrics"]["quality_bridge"]["status"] == "unavailable"
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_bridge_success_stamps_quality_provenance_without_dropping_or_adding_rows(self, monkeypatch):
|
||||
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
|
||||
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
|
||||
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
|
||||
"run": {"run_id": "dqrun_9", "status": "succeeded", "score": 100},
|
||||
"issues": [],
|
||||
}))
|
||||
result = await run_job(
|
||||
"order_bridge_ok",
|
||||
{
|
||||
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
|
||||
"tenant_id": "t_bridge_ok", "project_id": "p_1",
|
||||
},
|
||||
)
|
||||
assert result["metrics"]["rows"] == 1
|
||||
assert result["metrics"]["quality_bridge"]["status"] == "ok"
|
||||
assert result["metrics"]["quality_bridge"]["score"] == 100
|
||||
for p in result["records"][0]["provenance"]:
|
||||
assert p["quality"] == {
|
||||
"score": 100, "run_id": "dqrun_9", "run_status": "succeeded", "checked_by": "0pt-quality-bridge",
|
||||
}
|
||||
assert p["artifact_ref"] == "0pt:pipeline_delivery:order_bridge_ok"
|
||||
@@ -0,0 +1,166 @@
|
||||
"""Regression tests for the Python adapter to canonical TypeScript ER.
|
||||
|
||||
The removed Python scorer is deliberately not tested because it no longer
|
||||
exists. These tests pin the load-bearing adapter semantics instead:
|
||||
canonical match edges merge; review/non-match edges do not; hard negative
|
||||
identity evidence constrains the whole cluster; bridge failure leaves records
|
||||
unmerged.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
|
||||
from workers.resolve import _apply_matches, resolve
|
||||
|
||||
|
||||
def _prov(field_name: str, source_id: str, confidence: float) -> dict:
|
||||
return {
|
||||
"field_name": field_name,
|
||||
"source_id": source_id,
|
||||
"confidence": confidence,
|
||||
"entity_resolution": None,
|
||||
}
|
||||
|
||||
|
||||
def _rec(name: str, source: str, confidence: float = 0.9) -> dict:
|
||||
return {
|
||||
"market": "SE",
|
||||
"name": name,
|
||||
"status": "active",
|
||||
"lei": None,
|
||||
"org_nr": None,
|
||||
"source": source,
|
||||
"confidence": confidence,
|
||||
"provenance": [_prov("name", source, confidence)],
|
||||
}
|
||||
|
||||
|
||||
def _pair(a: str, b: str, status: str = "match", decided_by: str = "score", score: float = 0.97) -> dict:
|
||||
return {
|
||||
"candidate_id": f"c_{a}_{b}",
|
||||
"record_a": a,
|
||||
"record_b": b,
|
||||
"score": score,
|
||||
"status": status,
|
||||
"decided_by": decided_by,
|
||||
}
|
||||
|
||||
|
||||
def test_apply_matches_is_deterministic_and_uses_canonical_edges_only():
|
||||
records = [_rec("Acme AB", "source-a", 0.91), _rec("ACME Aktiebolag", "source-b", 0.96)]
|
||||
assert len(_apply_matches(copy.deepcopy(records), [])) == 2
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_canonical_match_merges_and_stamps_provenance(monkeypatch):
|
||||
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
|
||||
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
|
||||
records = [_rec("Acme AB", "source-a", 0.91), _rec("Acme AB", "source-b", 0.96)]
|
||||
|
||||
route = respx.post("https://edge.example.test/api/0pt/internal/er/candidates")
|
||||
|
||||
def responder(request: httpx.Request) -> httpx.Response:
|
||||
body = __import__("json").loads(request.content)
|
||||
assert request.headers["X-Internal-Token"] == "test-internal-token"
|
||||
assert body["tenant_id"] == "tenant-a"
|
||||
assert body["blocking_fields"] == ["market", "block_initial"]
|
||||
a, b = body["records"][0]["record_id"], body["records"][1]["record_id"]
|
||||
# Endast hard-rule-beslut (decided_by != score/None) far skapa en
|
||||
# canonical identity-lank - consumenten ignorerar medvetet score/"match".
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={"pairs": [_pair(a, b, "match", "0pt.company.stable-identifiers.v3", 0.99)], "review_queue": []},
|
||||
)
|
||||
|
||||
route.side_effect = responder
|
||||
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
|
||||
|
||||
assert len(result) == 1
|
||||
assert result[0]["agreed_sources"] == 2
|
||||
assert result[0]["confidence"] == pytest.approx(0.99)
|
||||
assert set(result[0]["merged_from"]) <= {"source-a", "source-b"}
|
||||
for provenance in result[0]["provenance"]:
|
||||
er = provenance["entity_resolution"]
|
||||
assert er["method"] == "canonical_ts_entity_resolution"
|
||||
assert set(er["merged_sources"]) == {"source-a", "source-b"}
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_pending_review_never_auto_merges(monkeypatch):
|
||||
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
|
||||
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
|
||||
records = [_rec("Acme AB", "source-a"), _rec("Acme Group AB", "source-b")]
|
||||
|
||||
def responder(request: httpx.Request) -> httpx.Response:
|
||||
body = __import__("json").loads(request.content)
|
||||
a, b = body["records"][0]["record_id"], body["records"][1]["record_id"]
|
||||
pair = _pair(a, b, "pending_review")
|
||||
return httpx.Response(200, json={"pairs": [pair], "review_queue": [pair]})
|
||||
|
||||
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").side_effect = responder
|
||||
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
|
||||
assert len(result) == 2
|
||||
assert all(not r.get("merged_from") for r in result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_hard_non_match_blocks_transitive_cluster_merge(monkeypatch):
|
||||
"""A-B and B-C matches may never imply A=C when canonical hard evidence says A!=C."""
|
||||
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
|
||||
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
|
||||
records = [
|
||||
_rec("Acme AB", "source-a", 0.95),
|
||||
_rec("Acme AB", "source-b", 0.94),
|
||||
_rec("Acme AB", "source-c", 0.93),
|
||||
]
|
||||
|
||||
def responder(request: httpx.Request) -> httpx.Response:
|
||||
body = __import__("json").loads(request.content)
|
||||
a, b, c = [item["record_id"] for item in body["records"]]
|
||||
return httpx.Response(
|
||||
200,
|
||||
json={
|
||||
"pairs": [
|
||||
_pair(a, b, "match", "0pt.company.stable-identifiers.v3", 0.98),
|
||||
_pair(b, c, "match", "0pt.company.stable-identifiers.v3", 0.97),
|
||||
_pair(a, c, "non_match", "0pt.company.stable-identifiers.v2", 0.0),
|
||||
],
|
||||
"review_queue": [],
|
||||
},
|
||||
)
|
||||
|
||||
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").side_effect = responder
|
||||
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
|
||||
assert len(result) == 2
|
||||
assert max(r.get("agreed_sources", 1) for r in result) == 2
|
||||
assert not any(set(r.get("merged_from", [])) >= {"source-b", "source-c"} for r in result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@respx.mock
|
||||
async def test_bridge_outage_fails_closed_to_unmerged_records(monkeypatch):
|
||||
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
|
||||
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
|
||||
records = [_rec("Acme AB", "source-a"), _rec("Acme AB", "source-b")]
|
||||
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").mock(side_effect=httpx.ConnectError("offline"))
|
||||
|
||||
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
|
||||
assert len(result) == 2
|
||||
assert all(not r.get("merged_from") for r in result)
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_missing_internal_configuration_makes_zero_network_calls(monkeypatch):
|
||||
monkeypatch.delenv("OPT_ER_BRIDGE_URL", raising=False)
|
||||
monkeypatch.delenv("EDGE_URL", raising=False)
|
||||
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
|
||||
records = [_rec("Acme AB", "source-a"), _rec("Acme AB", "source-b")]
|
||||
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
|
||||
assert len(result) == 2
|
||||
@@ -0,0 +1,53 @@
|
||||
"""A failed or malformed source must never become a verified empty result."""
|
||||
import httpx
|
||||
import pytest
|
||||
import respx
|
||||
from fastapi.testclient import TestClient
|
||||
from orchestrator import main
|
||||
from sources.connectors import opensanctions_screen, rdw_vehicle
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
@pytest.mark.parametrize("status", [403, 429, 500])
|
||||
async def test_source_http_failure_raises(status):
|
||||
with respx.mock:
|
||||
respx.get("https://api.opensanctions.org/search/default").respond(status)
|
||||
respx.get("https://opendata.rdw.nl/resource/m9d7-ebf2.json").respond(status)
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await opensanctions_screen("example")
|
||||
with pytest.raises(httpx.HTTPStatusError):
|
||||
await rdw_vehicle("AB123")
|
||||
|
||||
|
||||
@pytest.mark.asyncio
|
||||
async def test_malformed_and_valid_empty_are_distinct():
|
||||
with respx.mock:
|
||||
screen = respx.get("https://api.opensanctions.org/search/default").respond(200, json={})
|
||||
vehicle = respx.get("https://opendata.rdw.nl/resource/m9d7-ebf2.json").respond(200, json={})
|
||||
with pytest.raises(ValueError):
|
||||
await opensanctions_screen("example")
|
||||
with pytest.raises(ValueError):
|
||||
await rdw_vehicle("AB123")
|
||||
screen.respond(200, json={"results": []})
|
||||
vehicle.respond(200, json=[])
|
||||
assert (await opensanctions_screen("example"))["matches"] == []
|
||||
assert await rdw_vehicle("AB123") is None
|
||||
|
||||
|
||||
def test_routes_return_503_on_source_failure_and_mark_verified_empty(monkeypatch):
|
||||
async def failed(*args):
|
||||
raise RuntimeError("fixture failure")
|
||||
async def empty_screen(*args):
|
||||
return {"source": "opensanctions", "matches": []}
|
||||
async def empty_vehicle(*args):
|
||||
return None
|
||||
monkeypatch.setattr(main, "_INTERNAL_TOKEN", "")
|
||||
monkeypatch.setattr(main, "opensanctions_screen", failed)
|
||||
monkeypatch.setattr(main, "rdw_vehicle", failed)
|
||||
client = TestClient(main.app)
|
||||
assert client.get('/screen?q=example').status_code == 503
|
||||
assert client.get('/vehicles/AB123').status_code == 503
|
||||
monkeypatch.setattr(main, "opensanctions_screen", empty_screen)
|
||||
monkeypatch.setattr(main, "rdw_vehicle", empty_vehicle)
|
||||
assert client.get('/screen?q=example').json() == {"source": "opensanctions", "source_status": "ok", "matches": []}
|
||||
assert client.get('/vehicles/AB123').json() == {"vehicle": None, "source_status": "ok"}
|
||||
@@ -0,0 +1 @@
|
||||
# 0pt data engine — workers package (router + pipeline).
|
||||
@@ -0,0 +1,97 @@
|
||||
"""
|
||||
Adaptive router — per-market compliant-source planner.
|
||||
|
||||
P014 CORRECTION: this file's previous docstring read "all methods, no
|
||||
compliance restrictions. See b0ts for scraping." That phrasing directly
|
||||
contradicts this product's own compliance stance (CLAUDE.md §1: official
|
||||
API/bulk -> authorized public web -> licensed partner data -> otherwise
|
||||
level 0, never a bypass) and, worse, pointed a future reader at `b0ts` -- a
|
||||
SEPARATE repo that does non-compliant scraping and is out of scope/forbidden
|
||||
to touch from here. Nothing in this file ever called b0ts or any bypass
|
||||
path; the line was simply wrong and is corrected, not extended.
|
||||
|
||||
What this file actually does: `plan_for_market(market)` returns the ordered
|
||||
list of registered source_ids `pipeline.py._fetch` tries, in priority order,
|
||||
for a given market. Every entry is a name pipeline.py may or may not have an
|
||||
implemented connector for (see sources/connectors.py's CONNECTORS dict) --
|
||||
an unimplemented source_id is simply skipped by `_fetch`, it is never a
|
||||
license to build a bypass in its place.
|
||||
|
||||
P014 ADDITION: `record_blocked` / `is_blocked` give this planner a memory of
|
||||
which (market, source_id) pairs recently told the engine to stop (HTTP
|
||||
401/403/429/451 -- see sources/connectors.py's SourceBlockedError). Before
|
||||
this pass, a source that started rate-limiting or blocking the engine mid-run
|
||||
was retried on every subsequent record with no memory at all -- not a
|
||||
compliance bypass, but not "respecting the block as a stop signal" either
|
||||
(the same principle 0pt-motor's AdaptiveRouter documents for its own,
|
||||
separate method-tier selector). `plan_for_market` now filters a currently-
|
||||
blocked source out of the plan for the rest of the block's cooldown window,
|
||||
so the engine backs off instead of hammering it. This is process-local,
|
||||
in-memory state (matches every other piece of in-memory state this engine
|
||||
documents honestly elsewhere, e.g. orchestrator/main.py's _ORDERS/_JOBS) --
|
||||
not a persistent, cross-process reputation system.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
|
||||
MARKET_SOURCES: dict[str, list[str]] = {
|
||||
"NO": ["brreg", "brreg_under", "proff", "1881", "gulesider", "flowfirma", "selskapssok", "regnskapsbasen", "okonomi", "visverdi", "gleif"],
|
||||
"GB": ["companies_house", "gleif"],
|
||||
"FI": ["prh", "gleif"],
|
||||
"DK": ["cvr", "krak", "proff_dk", "118", "degulesider", "companydata", "grundigt", "datafordeler", "virk", "gleif"],
|
||||
"DE": ["northdata", "handelsregister", "unternehmensregister", "moneyhouse", "creditreform", "bundesanzeiger", "registercheck", "personensuche", "whitepages", "companyhouse", "gleif"],
|
||||
"SE": ["gleif"],
|
||||
}
|
||||
|
||||
# How long a (market, source_id) pair is excluded from plan_for_market after
|
||||
# being reported blocked. A fixed cooldown, not exponential backoff -- this
|
||||
# is a planner, not a retry/backoff controller; a source is free to be tried
|
||||
# again on the next run once the window elapses.
|
||||
DEFAULT_BLOCK_COOLDOWN_SECONDS = 300.0
|
||||
|
||||
|
||||
class AdaptiveRouter:
|
||||
"""Tracks which (market, source_id) pairs are currently in a block cooldown."""
|
||||
|
||||
def __init__(self, cooldown_seconds: float = DEFAULT_BLOCK_COOLDOWN_SECONDS):
|
||||
self._cooldown = cooldown_seconds
|
||||
self._blocked_until: dict[tuple[str, str], float] = {}
|
||||
|
||||
def record_blocked(self, market: str, source_id: str, *, now: float | None = None) -> None:
|
||||
"""Mark (market, source_id) as blocked starting now."""
|
||||
self._blocked_until[(market, source_id)] = (now if now is not None else time.monotonic()) + self._cooldown
|
||||
|
||||
def is_blocked(self, market: str, source_id: str, *, now: float | None = None) -> bool:
|
||||
deadline = self._blocked_until.get((market, source_id))
|
||||
if deadline is None:
|
||||
return False
|
||||
current = now if now is not None else time.monotonic()
|
||||
if current >= deadline:
|
||||
# Cooldown elapsed -- clear it so the map doesn't grow forever.
|
||||
del self._blocked_until[(market, source_id)]
|
||||
return False
|
||||
return True
|
||||
|
||||
def clear(self) -> None:
|
||||
"""Test/process-reset hook -- forgets every recorded block."""
|
||||
self._blocked_until.clear()
|
||||
|
||||
|
||||
# Process-wide singleton, mirroring the in-memory-state convention already
|
||||
# used elsewhere in this engine (e.g. orchestrator/main.py's _ORDERS/_JOBS).
|
||||
# Injectable in `plan_for_market` for tests rather than only reachable via
|
||||
# global mutable state.
|
||||
default_router = AdaptiveRouter()
|
||||
|
||||
|
||||
def plan_for_market(market: str, *, router: AdaptiveRouter | None = None) -> list[str]:
|
||||
"""
|
||||
Return the ordered source plan for a market, minus any source_id
|
||||
currently in a block cooldown for that market. `router` defaults to the
|
||||
process-wide `default_router`; tests inject their own instance so they
|
||||
never share state with each other or with production.
|
||||
"""
|
||||
r = router if router is not None else default_router
|
||||
plan = MARKET_SOURCES.get(market, ["gleif"])
|
||||
return [source_id for source_id in plan if not r.is_blocked(market, source_id)]
|
||||
@@ -0,0 +1,217 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from dataclasses import dataclass, field
|
||||
from enum import Enum
|
||||
from typing import Literal
|
||||
from urllib.parse import urlparse
|
||||
import re
|
||||
import unicodedata
|
||||
|
||||
from rapidfuzz.fuzz import ratio
|
||||
|
||||
EntityType = Literal["organization", "person"]
|
||||
|
||||
|
||||
class ResolutionDecision(str, Enum):
|
||||
EXACT_LINK = "exact_link"
|
||||
REVIEW = "review"
|
||||
DISTINCT = "distinct"
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class EntityCandidate:
|
||||
entity_id: str
|
||||
entity_type: EntityType
|
||||
name: str
|
||||
source_ref: str
|
||||
registration_country: str | None = None
|
||||
registration_number: str | None = None
|
||||
email: str | None = None
|
||||
website: str | None = None
|
||||
organization_id: str | None = None
|
||||
city: str | None = None
|
||||
metadata: dict[str, object] = field(default_factory=dict)
|
||||
|
||||
|
||||
@dataclass(frozen=True)
|
||||
class ResolutionResult:
|
||||
left_id: str
|
||||
right_id: str
|
||||
decision: ResolutionDecision
|
||||
score: float
|
||||
reasons: tuple[str, ...]
|
||||
requires_human_review: bool
|
||||
algorithm_version: str = "entity-resolution-v1"
|
||||
|
||||
|
||||
def _norm(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
normalized = unicodedata.normalize("NFKD", value)
|
||||
ascii_text = "".join(char for char in normalized if not unicodedata.combining(char))
|
||||
return re.sub(r"[^a-z0-9]+", " ", ascii_text.lower()).strip()
|
||||
|
||||
|
||||
def _norm_registration_number(value: str | None) -> str:
|
||||
return re.sub(r"[^0-9a-z]+", "", (value or "").lower())
|
||||
|
||||
|
||||
def _norm_email(value: str | None) -> str:
|
||||
return (value or "").strip().lower()
|
||||
|
||||
|
||||
def _domain(value: str | None) -> str:
|
||||
if not value:
|
||||
return ""
|
||||
raw = value.strip().lower()
|
||||
if "@" in raw and "://" not in raw:
|
||||
raw = raw.rsplit("@", 1)[-1]
|
||||
if "://" not in raw:
|
||||
raw = f"https://{raw}"
|
||||
host = (urlparse(raw).hostname or "").lower()
|
||||
return host.removeprefix("www.")
|
||||
|
||||
|
||||
def _exact_organization(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult | None:
|
||||
left_number = _norm_registration_number(left.registration_number)
|
||||
right_number = _norm_registration_number(right.registration_number)
|
||||
left_country = _norm(left.registration_country)
|
||||
right_country = _norm(right.registration_country)
|
||||
if left_number and right_number and left_country and right_country:
|
||||
if left_country == right_country and left_number == right_number:
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
ResolutionDecision.EXACT_LINK,
|
||||
1.0,
|
||||
("same registration country and registration number",),
|
||||
False,
|
||||
)
|
||||
if left_country == right_country and left_number != right_number:
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
ResolutionDecision.DISTINCT,
|
||||
0.0,
|
||||
("different registration numbers in the same country",),
|
||||
False,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _exact_person(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult | None:
|
||||
left_email = _norm_email(left.email)
|
||||
right_email = _norm_email(right.email)
|
||||
if left_email and right_email:
|
||||
if left_email == right_email:
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
ResolutionDecision.EXACT_LINK,
|
||||
1.0,
|
||||
("same normalized email address",),
|
||||
False,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def resolve_entity_pair(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult:
|
||||
if left.entity_type != right.entity_type:
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
ResolutionDecision.DISTINCT,
|
||||
0.0,
|
||||
("different entity types",),
|
||||
False,
|
||||
)
|
||||
|
||||
exact = _exact_organization(left, right) if left.entity_type == "organization" else _exact_person(left, right)
|
||||
if exact is not None:
|
||||
return exact
|
||||
|
||||
left_name = _norm(left.name)
|
||||
right_name = _norm(right.name)
|
||||
if not left_name or not right_name:
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
ResolutionDecision.REVIEW,
|
||||
0.0,
|
||||
("missing normalized name prevents deterministic comparison",),
|
||||
True,
|
||||
)
|
||||
|
||||
name_similarity = ratio(left_name, right_name) / 100.0
|
||||
reasons = [f"name similarity {name_similarity:.3f}"]
|
||||
score = name_similarity * 0.65
|
||||
|
||||
if left.entity_type == "organization":
|
||||
left_domain = _domain(left.website or left.email)
|
||||
right_domain = _domain(right.website or right.email)
|
||||
if left_domain and right_domain:
|
||||
if left_domain == right_domain:
|
||||
score += 0.25
|
||||
reasons.append("same normalized domain")
|
||||
else:
|
||||
score -= 0.1
|
||||
reasons.append("different normalized domains")
|
||||
if _norm(left.registration_country) and _norm(left.registration_country) == _norm(right.registration_country):
|
||||
score += 0.05
|
||||
reasons.append("same registration country")
|
||||
if _norm(left.city) and _norm(left.city) == _norm(right.city):
|
||||
score += 0.05
|
||||
reasons.append("same city")
|
||||
else:
|
||||
if left.organization_id and right.organization_id:
|
||||
if left.organization_id == right.organization_id:
|
||||
score += 0.25
|
||||
reasons.append("same known organization")
|
||||
else:
|
||||
score -= 0.1
|
||||
reasons.append("different known organizations")
|
||||
left_domain = _domain(left.email)
|
||||
right_domain = _domain(right.email)
|
||||
if left_domain and right_domain and left_domain == right_domain:
|
||||
score += 0.1
|
||||
reasons.append("same email domain")
|
||||
|
||||
score = max(0.0, min(1.0, score))
|
||||
|
||||
# Only strong identifiers auto-link. Similarity-based matches remain operator-review candidates.
|
||||
if score >= 0.72:
|
||||
decision = ResolutionDecision.REVIEW
|
||||
review = True
|
||||
elif score <= 0.3:
|
||||
decision = ResolutionDecision.DISTINCT
|
||||
review = False
|
||||
else:
|
||||
decision = ResolutionDecision.REVIEW
|
||||
review = True
|
||||
|
||||
return ResolutionResult(
|
||||
left.entity_id,
|
||||
right.entity_id,
|
||||
decision,
|
||||
round(score, 6),
|
||||
tuple(reasons),
|
||||
review,
|
||||
)
|
||||
|
||||
|
||||
def rank_resolution_candidates(
|
||||
query: EntityCandidate,
|
||||
candidates: list[EntityCandidate],
|
||||
*,
|
||||
limit: int = 20,
|
||||
) -> list[ResolutionResult]:
|
||||
results = [resolve_entity_pair(query, candidate) for candidate in candidates if candidate.entity_id != query.entity_id]
|
||||
decision_order = {
|
||||
ResolutionDecision.EXACT_LINK: 0,
|
||||
ResolutionDecision.REVIEW: 1,
|
||||
ResolutionDecision.DISTINCT: 2,
|
||||
}
|
||||
return sorted(
|
||||
results,
|
||||
key=lambda result: (decision_order[result.decision], -result.score, result.right_id),
|
||||
)[: max(0, limit)]
|
||||
@@ -0,0 +1,151 @@
|
||||
"""Production pipeline: sourcing -> extraction -> canonical ER -> QA -> quality.
|
||||
|
||||
Identity decisions are owned by src/er/entity-resolution-service.ts. The
|
||||
Python worker only applies canonical `match` decisions returned by its
|
||||
internal bridge. If tenant context or the canonical bridge is unavailable,
|
||||
records remain unmerged: false identity merges are never used as fallback.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
from typing import Any
|
||||
|
||||
from workers.resolve import resolve
|
||||
from workers.quality_bridge import run_quality_bridge
|
||||
from sources.connectors import (
|
||||
CompanyResult,
|
||||
SourceBlockedError,
|
||||
brreg_by_orgnr,
|
||||
companies_house_by_number,
|
||||
cvr_by_number,
|
||||
gleif_by_lei,
|
||||
prh_by_businessid,
|
||||
)
|
||||
from workers.adaptive_router import default_router, plan_for_market
|
||||
|
||||
logger = logging.getLogger(__name__)
|
||||
PIPELINE_VERSION = "0pt-pipeline@2"
|
||||
|
||||
DEMO_SEEDS: dict[str, list[str]] = {
|
||||
"NO": ["923609016"],
|
||||
"GB": ["00000006"],
|
||||
}
|
||||
|
||||
|
||||
async def _fetch(source_id: str, key: str) -> CompanyResult | None:
|
||||
if source_id == "gleif": return await gleif_by_lei(key)
|
||||
if source_id == "brreg": return await brreg_by_orgnr(key)
|
||||
if source_id == "companies_house": return await companies_house_by_number(key)
|
||||
if source_id == "prh": return await prh_by_businessid(key)
|
||||
if source_id == "cvr": return await cvr_by_number(key)
|
||||
return None
|
||||
|
||||
|
||||
async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]:
|
||||
markets: list[str] = config.get("markets", []) or ["NO"]
|
||||
threshold = float(config.get("confidence", 92)) / 100.0
|
||||
seeds: dict[str, list[str]] = config["seeds"] if config.get("seeds") is not None else DEMO_SEEDS
|
||||
tenant_id = config.get("tenant_id")
|
||||
project_id = config.get("project_id", "p_default")
|
||||
correlation_id = config.get("correlation_id")
|
||||
|
||||
records: list[dict[str, Any]] = []
|
||||
per_market: dict[str, int] = {}
|
||||
|
||||
for market in markets:
|
||||
keys = seeds.get(market, [])
|
||||
got = 0
|
||||
for key in keys:
|
||||
plan = plan_for_market(market)
|
||||
for source_id in plan:
|
||||
try:
|
||||
res = await _fetch(source_id, key)
|
||||
except SourceBlockedError:
|
||||
default_router.record_blocked(market, source_id)
|
||||
res = None
|
||||
except Exception:
|
||||
res = None
|
||||
if res and res.name:
|
||||
records.append({
|
||||
"market": market,
|
||||
"name": res.name,
|
||||
"status": res.status,
|
||||
"lei": res.lei,
|
||||
"org_nr": res.org_nr,
|
||||
"source": res.source,
|
||||
"provenance": [p.__dict__ for p in res.provenance],
|
||||
"confidence": max((p.confidence for p in res.provenance), default=0.0),
|
||||
})
|
||||
got += 1
|
||||
break
|
||||
per_market[market] = got
|
||||
|
||||
# Canonical ER is tenant-bound. Missing tenant context must never be
|
||||
# papered over with a synthetic tenant or the removed Python scorer.
|
||||
if tenant_id:
|
||||
deduped = await resolve(
|
||||
records,
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
er_mode = "canonical_ts"
|
||||
else:
|
||||
deduped = [dict(r) for r in records]
|
||||
er_mode = "fail_closed_unmerged_missing_tenant"
|
||||
|
||||
delivered = [r for r in deduped if r["confidence"] >= threshold]
|
||||
dropped = len(deduped) - len(delivered)
|
||||
avg_conf = round(sum(r["confidence"] for r in delivered) / len(delivered), 3) if delivered else 0.0
|
||||
|
||||
artifact_ref = f"0pt:pipeline_delivery:{order_id}"
|
||||
for r in delivered:
|
||||
for p in r.get("provenance", []):
|
||||
p["artifact_ref"] = artifact_ref
|
||||
|
||||
quality_bridge_metrics: dict[str, Any]
|
||||
if not tenant_id:
|
||||
quality_bridge_metrics = {
|
||||
"status": "not_configured",
|
||||
"detail": "config.tenant_id missing — quality bridge not called",
|
||||
}
|
||||
else:
|
||||
bridge_result = await run_quality_bridge(
|
||||
tenant_id=tenant_id,
|
||||
project_id=project_id,
|
||||
dataset_id=f"0pt-order-{order_id}",
|
||||
rows=delivered,
|
||||
correlation_id=correlation_id,
|
||||
)
|
||||
quality_bridge_metrics = bridge_result.as_metrics()
|
||||
if bridge_result.status == "ok":
|
||||
quality_info = {
|
||||
"score": bridge_result.score,
|
||||
"run_id": bridge_result.run_id,
|
||||
"run_status": bridge_result.run_status,
|
||||
"checked_by": "0pt-quality-bridge",
|
||||
}
|
||||
for r in delivered:
|
||||
for p in r.get("provenance", []):
|
||||
p["quality"] = quality_info
|
||||
elif bridge_result.status != "disabled":
|
||||
logger.warning(
|
||||
"0pt.quality_bridge did not complete for order %s: %s (%s)",
|
||||
order_id, bridge_result.status, bridge_result.detail,
|
||||
)
|
||||
|
||||
return {
|
||||
"stage": "qa",
|
||||
"pct": 100,
|
||||
"state": "done",
|
||||
"metrics": {
|
||||
"rows": len(delivered),
|
||||
"dropped_below_threshold": dropped,
|
||||
"per_market": per_market,
|
||||
"avg_confidence": avg_conf,
|
||||
"pipeline_version": PIPELINE_VERSION,
|
||||
"entity_resolution": er_mode,
|
||||
"quality_bridge": quality_bridge_metrics,
|
||||
},
|
||||
"records": delivered,
|
||||
}
|
||||
@@ -0,0 +1,154 @@
|
||||
"""
|
||||
services/api/workers/quality_bridge.py
|
||||
=========================================
|
||||
P010: the TS/Python split-brain bridge (BEFORE_STATE.md §2). The production
|
||||
order pipeline (workers/pipeline.py) has always run its own one-line QA
|
||||
filter (`[r for r in deduped if r["confidence"] >= threshold]`) and never
|
||||
touched this repo's sophisticated, versioned TS QualityService (393 lines:
|
||||
rule registry, GE delegation, OpenLineage) — "noll bryggor mellan sig", per
|
||||
BEFORE_STATE.md.
|
||||
|
||||
This module is the ADDITIVE bridge, called optionally from pipeline.py. It
|
||||
is NOT a replacement for the Python filter and cannot become a silent one:
|
||||
|
||||
* Feature-flagged OFF by default (OPT_QUALITY_BRIDGE_ENABLED unset/"false").
|
||||
A disabled bridge makes zero network calls and returns immediately.
|
||||
* FAILS OPEN. Any failure mode (bridge URL unset, connection refused,
|
||||
timeout, non-2xx, malformed JSON) is caught here and reported back as
|
||||
`{"status": "unavailable", ...}` — it NEVER raises into pipeline.run_job,
|
||||
and it never changes which rows get delivered. `run_job` remains the
|
||||
single source of truth for delivery; this bridge is read-only/advisory.
|
||||
* ADVISORY ONLY, not a second gate — and this is a real limitation, not a
|
||||
choice made lightly: ExpectationEngine's execute() returns per-rule
|
||||
PASS/FAIL COUNTS and up to 20 unexpected-value SAMPLES
|
||||
(src/quality/expectation-engine.ts), not a per-row boolean array. There
|
||||
is no way, from a dataset-level score + a handful of samples, to know
|
||||
*which* of the N rows in this batch failed which rule — so per-row
|
||||
gating against the TS engine's result cannot be implemented honestly
|
||||
without first extending that engine's output contract (out of scope for
|
||||
this pass; flagged, not worked around with a guess). A future
|
||||
`OPT_QUALITY_BRIDGE_GATE_ENABLED` flag is deliberately NOT implemented
|
||||
here for that reason — see the docstring on `BridgeResult` below.
|
||||
|
||||
Transport: a direct HTTP call to the Next.js edge's
|
||||
POST /api/0pt/quality/bridge-run (app/api/0pt/quality/bridge-run/route.ts),
|
||||
reusing the same `X-Internal-Token` shared-secret convention lib/engine.ts
|
||||
already uses for the (opposite-direction) edge -> engine call, for
|
||||
consistency. This was chosen over an N0D job dispatch (the other option the
|
||||
brief raised) because: the TS QualityService instance the pipeline needs to
|
||||
reach lives in-process inside the Next.js app (src/services/quality.ts, an
|
||||
in-memory singleton, same pattern as the Source Registry before its Postgres
|
||||
persistence landed) — there is no durable store an N0D job could read/write
|
||||
through today, and dispatching a job whose only effect is "call this
|
||||
in-process singleton" would add a queue hop for no isolation benefit. A
|
||||
synchronous HTTP call, wrapped in a short timeout and a hard fail-open, is
|
||||
the simpler and more honest choice for a first cut; nothing here blocks a
|
||||
later move to N0D once the quality state has real (Postgres) persistence.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
_TIMEOUT = httpx.Timeout(4.0, connect=2.0) # short — this must never meaningfully slow delivery
|
||||
|
||||
|
||||
def _enabled() -> bool:
|
||||
return os.getenv("OPT_QUALITY_BRIDGE_ENABLED", "false").strip().lower() in ("1", "true", "yes", "on")
|
||||
|
||||
|
||||
def _bridge_url() -> str | None:
|
||||
base = os.getenv("OPT_QUALITY_BRIDGE_URL") or os.getenv("EDGE_URL")
|
||||
if not base:
|
||||
return None
|
||||
return f"{base.rstrip('/')}/api/0pt/quality/bridge-run"
|
||||
|
||||
|
||||
@dataclass
|
||||
class BridgeResult:
|
||||
"""
|
||||
Outcome of one bridge-run attempt. `status`:
|
||||
- "disabled": OPT_QUALITY_BRIDGE_ENABLED is not set — no call made.
|
||||
- "not_configured": enabled, but no OPT_QUALITY_BRIDGE_URL/EDGE_URL set.
|
||||
- "unavailable": enabled + configured, but the call failed (timeout,
|
||||
connection error, non-2xx, malformed body). Never raises.
|
||||
- "ok": a real run completed; `score`/`run_id`/`issue_count` are populated.
|
||||
|
||||
There is deliberately no `gated`/`dropped_rows` field — see the module
|
||||
docstring's "ADVISORY ONLY" section for why per-row gating isn't
|
||||
implementable against the current TS engine contract.
|
||||
"""
|
||||
status: str
|
||||
score: float | None = None
|
||||
run_id: str | None = None
|
||||
run_status: str | None = None
|
||||
issue_count: int | None = None
|
||||
detail: str | None = None
|
||||
|
||||
def as_metrics(self) -> dict[str, Any]:
|
||||
return {
|
||||
"status": self.status,
|
||||
"score": self.score,
|
||||
"run_id": self.run_id,
|
||||
"run_status": self.run_status,
|
||||
"issue_count": self.issue_count,
|
||||
**({"detail": self.detail} if self.detail else {}),
|
||||
}
|
||||
|
||||
|
||||
async def run_quality_bridge(
|
||||
*,
|
||||
tenant_id: str,
|
||||
project_id: str,
|
||||
dataset_id: str,
|
||||
rows: list[dict[str, Any]],
|
||||
correlation_id: str | None = None,
|
||||
) -> BridgeResult:
|
||||
"""Ask the TS QualityService to grade `rows`. Never raises — see module docstring."""
|
||||
if not _enabled():
|
||||
return BridgeResult(status="disabled")
|
||||
|
||||
url = _bridge_url()
|
||||
if not url:
|
||||
return BridgeResult(status="not_configured", detail="OPT_QUALITY_BRIDGE_URL/EDGE_URL not set")
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"tenant_id": tenant_id,
|
||||
"project_id": project_id,
|
||||
"dataset_id": dataset_id,
|
||||
"rows": rows,
|
||||
}
|
||||
if correlation_id:
|
||||
payload["correlation_id"] = correlation_id
|
||||
|
||||
headers = {
|
||||
"Content-Type": "application/json",
|
||||
"X-Internal-Token": os.getenv("INTERNAL_TOKEN", ""),
|
||||
}
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
r = await client.post(url, json=payload, headers=headers)
|
||||
if r.status_code not in (200, 201):
|
||||
return BridgeResult(status="unavailable", detail=f"HTTP {r.status_code}")
|
||||
body = r.json()
|
||||
run = body.get("run") or {}
|
||||
issues = body.get("issues") or []
|
||||
return BridgeResult(
|
||||
status="ok",
|
||||
score=run.get("score"),
|
||||
run_id=run.get("run_id"),
|
||||
run_status=run.get("status"),
|
||||
issue_count=len(issues),
|
||||
)
|
||||
except httpx.TimeoutException:
|
||||
return BridgeResult(status="unavailable", detail="timeout")
|
||||
except httpx.HTTPError as exc:
|
||||
return BridgeResult(status="unavailable", detail=f"{type(exc).__name__}: {exc}")
|
||||
except (ValueError, KeyError, TypeError) as exc:
|
||||
# Malformed/unexpected response body — treat exactly like network
|
||||
# unavailability, never propagate into the pipeline.
|
||||
return BridgeResult(status="unavailable", detail=f"malformed response: {type(exc).__name__}")
|
||||
@@ -0,0 +1,31 @@
|
||||
"""
|
||||
Lightweight in-process job queue (docs/02). Adequate for single-worker dev; swap
|
||||
the backend for Redis/Celery at scale without changing callers.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
from typing import Any, Awaitable, Callable
|
||||
|
||||
Job = Callable[..., Awaitable[Any]]
|
||||
|
||||
|
||||
class JobQueue:
|
||||
def __init__(self) -> None:
|
||||
self._q: "asyncio.Queue[tuple[Job, tuple[Any, ...]]]" = asyncio.Queue()
|
||||
|
||||
async def enqueue(self, fn: Job, *args: Any) -> None:
|
||||
await self._q.put((fn, args))
|
||||
|
||||
async def worker(self) -> None:
|
||||
while True:
|
||||
fn, args = await self._q.get()
|
||||
try:
|
||||
await fn(*args)
|
||||
except Exception as e: # keep the worker alive on a failed job
|
||||
print("job failed:", repr(e))
|
||||
finally:
|
||||
self._q.task_done()
|
||||
|
||||
|
||||
queue = JobQueue()
|
||||
@@ -0,0 +1,235 @@
|
||||
"""Thin adapter from the Python pipeline to 0pt's canonical TS ER engine.
|
||||
|
||||
Python no longer scores identity. It sends opaque records to
|
||||
/api/0pt/internal/er/candidates, then applies only explicit hard-rule `match`
|
||||
decisions. Score/fuzzy decisions are review evidence only and can never merge
|
||||
records in this adapter, even if an older or compromised bridge returns
|
||||
`status=match, decided_by=score`.
|
||||
|
||||
If the canonical engine is unavailable, records are returned UNMERGED (fail
|
||||
closed against false identity merges).
|
||||
|
||||
A canonical hard non-match is also a cluster-wide constraint: transitive
|
||||
matches are never allowed to pull two records with conflicting stable identity
|
||||
evidence into the same entity.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import copy
|
||||
import hashlib
|
||||
import json
|
||||
import os
|
||||
from typing import Any
|
||||
|
||||
import httpx
|
||||
|
||||
_TIMEOUT = httpx.Timeout(6.0, connect=2.0)
|
||||
|
||||
|
||||
def _url() -> str | None:
|
||||
base = os.getenv("OPT_ER_BRIDGE_URL") or os.getenv("EDGE_URL")
|
||||
return f"{base.rstrip('/')}/api/0pt/internal/er/candidates" if base else None
|
||||
|
||||
|
||||
def _record_id(record: dict[str, Any]) -> str:
|
||||
stable = {
|
||||
"market": record.get("market"),
|
||||
"name": record.get("name"),
|
||||
"status": record.get("status"),
|
||||
"lei": record.get("lei"),
|
||||
"org_nr": record.get("org_nr"),
|
||||
"source": record.get("source"),
|
||||
"provenance": record.get("provenance", []),
|
||||
}
|
||||
raw = json.dumps(stable, sort_keys=True, separators=(",", ":"), default=str).encode()
|
||||
return "erp_" + hashlib.sha256(raw).hexdigest()[:32]
|
||||
|
||||
|
||||
def _unique_record_ids(records: list[dict[str, Any]]) -> list[str]:
|
||||
seen: dict[str, int] = {}
|
||||
unique_ids: list[str] = []
|
||||
for record in records:
|
||||
rid = _record_id(record)
|
||||
n = seen.get(rid, 0)
|
||||
seen[rid] = n + 1
|
||||
unique_ids.append(rid if n == 0 else f"{rid}_{n}")
|
||||
return unique_ids
|
||||
|
||||
|
||||
def _apply_matches(records: list[dict[str, Any]], pairs: list[dict[str, Any]]) -> list[dict[str, Any]]:
|
||||
if not records:
|
||||
return []
|
||||
|
||||
unique_ids = _unique_record_ids(records)
|
||||
known_ids = set(unique_ids)
|
||||
parent = {rid: rid for rid in unique_ids}
|
||||
members = {rid: {rid} for rid in unique_ids}
|
||||
|
||||
hard_non_matches: set[frozenset[str]] = set()
|
||||
for pair in pairs:
|
||||
a, b = pair.get("record_a"), pair.get("record_b")
|
||||
if a not in known_ids or b not in known_ids:
|
||||
continue
|
||||
if pair.get("status") == "non_match" and pair.get("decided_by") not in (None, "score"):
|
||||
hard_non_matches.add(frozenset((a, b)))
|
||||
|
||||
def find(x: str) -> str:
|
||||
while parent[x] != x:
|
||||
parent[x] = parent[parent[x]]
|
||||
x = parent[x]
|
||||
return x
|
||||
|
||||
def components_conflict(left: set[str], right: set[str]) -> bool:
|
||||
return any(frozenset((a, b)) in hard_non_matches for a in left for b in right)
|
||||
|
||||
def union(a: str, b: str) -> bool:
|
||||
ra, rb = find(a), find(b)
|
||||
if ra == rb:
|
||||
return True
|
||||
left, right = members[ra], members[rb]
|
||||
if components_conflict(left, right):
|
||||
return False
|
||||
keep, drop = (ra, rb) if ra < rb else (rb, ra)
|
||||
parent[drop] = keep
|
||||
members[keep] = members[ra] | members[rb]
|
||||
members.pop(drop, None)
|
||||
return True
|
||||
|
||||
# Defence in depth: only stable-identifier hard rules may create canonical
|
||||
# identity links. Score/fuzzy "match" responses are ignored at the consumer
|
||||
# boundary even if an upstream/legacy route emits one.
|
||||
match_pairs = [
|
||||
p for p in pairs
|
||||
if p.get("status") == "match"
|
||||
and p.get("decided_by") not in (None, "score")
|
||||
and p.get("record_a") in known_ids
|
||||
and p.get("record_b") in known_ids
|
||||
]
|
||||
match_pairs.sort(
|
||||
key=lambda p: (
|
||||
-float(p.get("score") or 0.0),
|
||||
str(p.get("candidate_id") or ""),
|
||||
)
|
||||
)
|
||||
accepted_edges: list[dict[str, Any]] = []
|
||||
for pair in match_pairs:
|
||||
a, b = str(pair["record_a"]), str(pair["record_b"])
|
||||
if union(a, b):
|
||||
accepted_edges.append(pair)
|
||||
|
||||
groups: dict[str, list[tuple[str, dict[str, Any]]]] = {}
|
||||
for rid, record in zip(unique_ids, records):
|
||||
groups.setdefault(find(rid), []).append((rid, copy.deepcopy(record)))
|
||||
|
||||
out: list[dict[str, Any]] = []
|
||||
for group in groups.values():
|
||||
group.sort(key=lambda item: (-float(item[1].get("confidence", 0.0)), item[0]))
|
||||
_, anchor = group[0]
|
||||
sources = sorted({str(r.get("source")) for _, r in group if r.get("source")})
|
||||
anchor["agreed_sources"] = len(sources) or 1
|
||||
anchor["confidence"] = min(
|
||||
0.99,
|
||||
float(anchor.get("confidence", 0.0)) + 0.1 * max(0, len(sources) - 1),
|
||||
)
|
||||
anchor["merged_from"] = [r.get("source") for _, r in group[1:] if r.get("source")]
|
||||
|
||||
provenance = list(anchor.get("provenance", []))
|
||||
keys = {(p.get("field_name"), p.get("source_id")) for p in provenance}
|
||||
for _, record in group[1:]:
|
||||
for p in record.get("provenance", []):
|
||||
key = (p.get("field_name"), p.get("source_id"))
|
||||
if key not in keys:
|
||||
provenance.append(copy.deepcopy(p))
|
||||
keys.add(key)
|
||||
|
||||
member_ids = {rid for rid, _ in group}
|
||||
edges = [
|
||||
p for p in accepted_edges
|
||||
if p.get("record_a") in member_ids and p.get("record_b") in member_ids
|
||||
]
|
||||
er_info = {
|
||||
"method": "canonical_ts_entity_resolution",
|
||||
"merged_sources": sources,
|
||||
"matched_edges": [
|
||||
{
|
||||
"candidate_id": p.get("candidate_id"),
|
||||
"score": p.get("score"),
|
||||
"decided_by": p.get("decided_by"),
|
||||
}
|
||||
for p in edges
|
||||
],
|
||||
}
|
||||
if len(group) > 1:
|
||||
for p in provenance:
|
||||
p["entity_resolution"] = er_info
|
||||
anchor["provenance"] = provenance
|
||||
out.append(anchor)
|
||||
|
||||
return sorted(out, key=lambda r: (_record_id(r), str(r.get("source") or "")))
|
||||
|
||||
|
||||
async def resolve(
|
||||
records: list[dict[str, Any]],
|
||||
*,
|
||||
tenant_id: str,
|
||||
project_id: str,
|
||||
correlation_id: str | None = None,
|
||||
) -> list[dict[str, Any]]:
|
||||
"""Return canonically resolved records; never falls back to local scoring."""
|
||||
if len(records) < 2:
|
||||
return [copy.deepcopy(r) for r in records]
|
||||
|
||||
url = _url()
|
||||
token = os.getenv("INTERNAL_TOKEN", "")
|
||||
if not url or not token or not tenant_id or not project_id:
|
||||
return [copy.deepcopy(r) for r in records]
|
||||
|
||||
unique_ids = _unique_record_ids(records)
|
||||
er_records = []
|
||||
for rid, record in zip(unique_ids, records):
|
||||
name = str(record.get("name") or "")
|
||||
market = record.get("market")
|
||||
er_records.append({
|
||||
"record_id": rid,
|
||||
"fields": {
|
||||
"market": market,
|
||||
"registration_country": market,
|
||||
"block_initial": name[:1].upper(),
|
||||
"name": name,
|
||||
"status": record.get("status"),
|
||||
"lei": record.get("lei"),
|
||||
"org_nr": record.get("org_nr"),
|
||||
},
|
||||
})
|
||||
|
||||
payload: dict[str, Any] = {
|
||||
"tenant_id": tenant_id,
|
||||
"project_id": project_id,
|
||||
"records": er_records,
|
||||
"blocking_fields": ["market", "block_initial"],
|
||||
"compare_fields": [{"field": "name", "weight": 1}],
|
||||
"auto_match_threshold": 0.92,
|
||||
"review_threshold": 0.72,
|
||||
}
|
||||
if correlation_id:
|
||||
payload["correlation_id"] = correlation_id
|
||||
|
||||
try:
|
||||
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
|
||||
response = await client.post(
|
||||
url,
|
||||
json=payload,
|
||||
headers={"Content-Type": "application/json", "X-Internal-Token": token},
|
||||
)
|
||||
if response.status_code not in (200, 201):
|
||||
return [copy.deepcopy(r) for r in records]
|
||||
body = response.json()
|
||||
pairs = body.get("pairs")
|
||||
if not isinstance(pairs, list):
|
||||
return [copy.deepcopy(r) for r in records]
|
||||
return _apply_matches(records, pairs)
|
||||
except (httpx.HTTPError, ValueError, TypeError, KeyError):
|
||||
return [copy.deepcopy(r) for r in records]
|
||||
|
||||
|
||||
__all__ = ["resolve", "_apply_matches"]
|
||||
@@ -0,0 +1,21 @@
|
||||
"""
|
||||
Worker entrypoint. In production this drains a queue of order jobs and runs the
|
||||
compliant pipeline. For now it exposes a CLI to run a single job locally.
|
||||
python -m workers.run <order_id>
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import sys
|
||||
|
||||
from workers.pipeline import run_job
|
||||
|
||||
|
||||
async def _main() -> None:
|
||||
order_id = sys.argv[1] if len(sys.argv) > 1 else "demo"
|
||||
result = await run_job(order_id, {"markets": ["NO", "GB"], "confidence": 90})
|
||||
print(result["metrics"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
asyncio.run(_main())
|
||||
Reference in New Issue
Block a user