fix(recovery): restore source removed by production documentation merge

This commit is contained in:
2026-11-10 03:42:10 +01:00
parent 585bdd4b44
commit 3d2fc6f913
957 changed files with 115754 additions and 0 deletions
+21
View File
@@ -0,0 +1,21 @@
# 0pt data engine — API + worker image (runs on a VPS, NOT Vercel; long jobs).
FROM python:3.12-slim
# Commit-kedjan (samma mönster som 0pt-motor/scripts/build-images.sh): samma
# SHA i image-taggen (sätts av byggaren), OCI-label och GET /health.
ARG GIT_SHA=unknown
LABEL org.opencontainers.image.revision=${GIT_SHA}
ENV GIT_SHA=${GIT_SHA}
WORKDIR /app
ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1
COPY requirements.txt .
RUN pip install -r requirements.txt
COPY . .
EXPOSE 8000
# API: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000
# Worker: python -m workers.run
CMD ["uvicorn", "orchestrator.main:app", "--host", "0.0.0.0", "--port", "8000"]
+34
View File
@@ -0,0 +1,34 @@
# 0pt data engine (`services/api`)
The legal data motor (docs/06). FastAPI orchestrator + worker pipeline that turn
**compliant** sources into delivered, provenance-tagged data. Runs on a VPS via
[`../../infra/docker-compose.yml`](../../infra/docker-compose.yml) — **not on Vercel** (long jobs).
## Sourcing boundary (CLAUDE.md §1 — non-negotiable)
Per field × market, in strict order: **official API/bulk → authorized public web
(robots + rate limits) → licensed partner data → otherwise level 0 "ej tillgänglig"**.
We never invent values.
**Deliberately excluded (never add here):** WAF/bot-detection bypass, FlareSolverr,
stealth/undetected browsers, proxy/fingerprint rotation to evade bans, scraping behind
login walls, automated fake-account creation. This is the honest-accuracy moat.
## Layout
- `sources/connectors.py` — compliant connectors (GLEIF, Brønnøysund, Companies House live; PRH/CVR/Bolagsverket gated on verified terms → level 0 until wired).
- `sources/sources.yaml` — source registry (access/licence/cadence; VERIFY terms).
- `orchestrator/main.py` — REST API (health, sources, estimates, orders, jobs) per docs/04.
- `workers/adaptive_router.py` — picks the cheapest **compliant** method per market (api > bulk > authorized_web > licensed). No bypass tiers.
- `workers/pipeline.py` — sourcing → extraction → structuring (dedup) → enrichment/fusion → QA (drops rows below the order's confidence target). Writes provenance per field.
## Run locally
```bash
pip install -r requirements.txt
uvicorn orchestrator.main:app --reload # API on :8000
python -m workers.run # run one demo job
```
## Reconciliation note
Supersedes the earlier `nordic-eu-data-platform` backend: the compliant connectors +
pipeline are migrated here; the FlareSolverr/stealth/proxy/fake-account paths and the
`test_bypass_chain`/`test_proxy_pool` examples are intentionally **not** carried over.
+1
View File
@@ -0,0 +1 @@
# 0pt data engine — orchestrator package (FastAPI app).
+265
View File
@@ -0,0 +1,265 @@
"""
0pt data engine — FastAPI orchestrator (docs/04). Compliant sourcing only.
Runs on a VPS/worker host, NOT on Vercel (long jobs). Amounts in öre.
Reconciles the Next.js /api routes with the Python data engine: the Next layer
is the customer-facing edge; this service runs the actual pipeline.
"""
from __future__ import annotations
import asyncio
import hmac
import os
import uuid
from pathlib import Path
from typing import Any
import yaml
from fastapi import FastAPI, HTTPException
from fastapi.middleware.cors import CORSMiddleware
from fastapi.responses import JSONResponse
from pydantic import BaseModel
from sources.connectors import opensanctions_screen, rdw_vehicle
from workers.pipeline import run_job
app = FastAPI(title="0pt data engine", version="0.1")
app.add_middleware(
CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"],
)
_SOURCES = yaml.safe_load((Path(__file__).resolve().parent.parent / "sources" / "sources.yaml").read_text(encoding="utf-8"))["sources"]
def _coerce_int(v: Any, default: int) -> int:
"""int(v) raises on decimal-formatted strings/floats; coerce via float() first
and fall back to `default` on any unparseable, client-controlled input instead
of a 500."""
try:
return int(float(v))
except (TypeError, ValueError):
return default
# ── estimate model (mirrors apps/web lib/data.computeEstimate) ───────────────
FIELD_RATE = 240
ENTITY_MULT = {"Bolag": 1.0, "Beslutsfattare": 1.4, "Fordon": 1.2}
FIELD_WEIGHT = {"identity": 1.0, "lei": 0.8, "financials": 1.6, "ubo": 1.4, "contact": 1.8, "intent": 1.5}
ENRICH_WEIGHT = {"dedup": 0.1, "crm": 0.2, "sanctions": 0.15, "nace": 0.1, "tech": 0.15}
class EstimateIn(BaseModel):
markets: list[str] = []
entity_type: str = "Bolag"
fields: list[str] = []
enrichment: list[str] = []
volume: int = 1000
frequency: str = "Engångs"
confidence_target: int = 92
def compute_estimate(b: EstimateIn) -> dict[str, Any]:
markets = max(1, len(b.markets))
fields = set(b.fields) | {"identity"}
fw = sum(FIELD_WEIGHT.get(f, 0.5) for f in fields) or 1
em = ENTITY_MULT.get(b.entity_type, 1.0)
cm = 1 + max(0, b.confidence_target - 90) * 0.03
ew = sum(ENRICH_WEIGHT.get(e, 0) for e in b.enrichment)
base = fw * markets * FIELD_RATE * (b.volume / 1000) * em * cm
price = round(base * (1 + ew) / 5) * 5
deposit = round(price / 2)
days = min(14, max(1, round(1 + markets * 0.4 + b.volume / 6000 + len(fields) * 0.25)))
avg_level = sum(3 if f in ("identity", "lei") else 2 if f in ("financials", "ubo") else 1 for f in fields) / len(fields)
ceiling = round(82 + avg_level * 5)
est_conf = min(b.confidence_target, ceiling)
return {
"total_ore": price * 100, "deposit_ore": deposit * 100, "balance_ore": (price - deposit) * 100,
"est_days": days, "est_confidence": est_conf / 100, "recurring": b.frequency != "Engångs",
}
# ── in-memory store (swapped for asyncpg when DATABASE_URL is set) ───────────
_ORDERS: dict[str, dict[str, Any]] = {}
_JOBS: dict[str, dict[str, Any]] = {}
# P014: one asyncio.Lock per order_id, held for the duration of a /run call.
# Process-local only (matches the honest in-memory-store limitation this file
# already documents), but closes the actual race a bare `if oid in _JOBS`
# check would leave open: two /run requests for the same order_id arriving
# back to back both await run_job() before either has stored a result, so a
# read-then-write check alone would let both through. See `run()` below.
_JOB_LOCKS: dict[str, asyncio.Lock] = {}
def _lock_for(oid: str) -> asyncio.Lock:
lock = _JOB_LOCKS.get(oid)
if lock is None:
lock = asyncio.Lock()
_JOB_LOCKS[oid] = lock
return lock
class OrderIn(BaseModel):
flow: str = "custom"
config: dict[str, Any] = {}
# ── internal-token gate (fail-closed) ────────────────────────────────────────
# The edge sends `X-Internal-Token` on every engineFetch (lib/engine.ts), but
# until 2026-09-15 this service never VERIFIED it — a deployed engine accepted
# anonymous traffic from anyone who could reach the port. The gate below
# enforces it when INTERNAL_TOKEN is set: every route except /health (the
# liveness probe) must carry the exact token or gets 401. When INTERNAL_TOKEN
# is unset the gate is inert (local dev / docker compose without auth wiring);
# the production deployment ALWAYS sets it (see infra/docker-compose.yml).
_INTERNAL_TOKEN = os.getenv("INTERNAL_TOKEN", "").strip()
_PUBLIC_PATHS = {"/health", "/docs", "/openapi.json", "/redoc"}
class _InternalTokenGate:
"""Raw-ASGI middleware: constant-time X-Internal-Token check, 401 on
missing/wrong token. Written as a plain ASGI class (not
@app.middleware('http')) so behavior does not depend on the
BaseHTTPMiddleware dispatch quirks of the installed starlette."""
def __init__(self, app) -> None:
self.app = app
async def __call__(self, scope, receive, send):
if scope["type"] == "http" and _INTERNAL_TOKEN and scope["path"] not in _PUBLIC_PATHS:
headers = {}
for k, v in scope.get("headers") or []:
headers.setdefault(k.decode("latin-1").lower(), v.decode("latin-1"))
supplied = headers.get("x-internal-token", "")
if not hmac.compare_digest(supplied, _INTERNAL_TOKEN):
resp = JSONResponse(
status_code=401,
content={"error": {"code": "unauthorized", "message": "missing or invalid X-Internal-Token"}},
)
await resp(scope, receive, send)
return
await self.app(scope, receive, send)
app.add_middleware(_InternalTokenGate)
@app.get("/health")
async def health():
return {
"ok": True,
"service": "0pt-data-engine",
"db": bool(os.getenv("DATABASE_URL")),
"sources": len(_SOURCES),
# Commit-kedjan: samma SHA som image-taggen/OCI-labeln (Dockerfile ARG
# GIT_SHA). 'unknown' gör ett oprovenienserat image synligt istället
# för tyst falskt — samma policy som 0pt-motor/scripts/verify-image.sh.
"revision": os.getenv("GIT_SHA", "unknown"),
}
@app.get("/sources")
async def sources():
return [{k: s.get(k) for k in ("source_id", "name", "market", "access", "licence", "cadence")} for s in _SOURCES]
@app.get("/sources/{market}")
async def sources_for(market: str):
return [s for s in _SOURCES if s.get("market") in (market, "global")]
@app.get("/vehicles/{plate}")
async def vehicle(plate: str):
try:
v = await rdw_vehicle(plate)
except Exception as exc:
raise HTTPException(status_code=503, detail="vehicle source unavailable") from exc
return {**(v or {"vehicle": None}), "source_status": "ok"}
@app.get("/screen")
async def screen(q: str):
try:
result = await opensanctions_screen(q)
return {**result, "source_status": "ok"}
except Exception as exc:
raise HTTPException(status_code=503, detail="screening source unavailable") from exc
@app.post("/estimates")
async def estimates(body: EstimateIn):
return compute_estimate(body)
@app.post("/orders", status_code=201)
async def create_order(body: OrderIn):
est = compute_estimate(EstimateIn(
markets=body.config.get("markets", []),
entity_type=body.config.get("type", "Bolag"),
fields=body.config.get("fieldIds", []),
enrichment=body.config.get("enrichmentIds", []),
volume=_coerce_int(body.config.get("volume", 1000), 1000),
frequency=body.config.get("frequency", "Engångs"),
confidence_target=_coerce_int(body.config.get("confidence", 92), 92),
))
oid = str(uuid.uuid4())
_ORDERS[oid] = {"order_id": oid, "flow": body.flow, "status": "draft", "config": body.config, **est}
return {"order_id": oid, "status": "draft", "deposit_ore": est["deposit_ore"], "total_ore": est["total_ore"]}
@app.get("/orders")
async def list_orders():
return list(_ORDERS.values())
@app.get("/orders/{oid}")
async def get_order(oid: str):
o = _ORDERS.get(oid)
if not o:
raise HTTPException(404, "order saknas")
return {**o, "job": _JOBS.get(oid)}
class StatusIn(BaseModel):
status: str
@app.patch("/orders/{oid}")
async def set_status(oid: str, body: StatusIn):
o = _ORDERS.get(oid)
if not o:
raise HTTPException(404, "order saknas")
o["status"] = body.status
return o
@app.post("/orders/{oid}/run")
async def run(oid: str):
"""
P014: idempotent delivery. An order should be fulfilled/delivered ONCE.
Before this fix, calling this endpoint twice for the same order_id (a
retried request, a duplicate trigger, an operator double-click) re-ran
the full compliant sourcing pipeline a second time and silently
overwrote `_JOBS[oid]` with a second, independently-produced result --
a genuinely different delivery (a second live fetch against upstream
registries, a second QA pass with its own confidence draws), completely
indistinguishable from the first at the API surface. This is the actual
"duplicate delivery" gap P014's recon flagged as NOT MET: nothing
downstream of order creation was idempotent at all. A job already
recorded for this order_id is now returned unchanged; `run_job` is never
invoked twice for the same order in this process's lifetime. The lock
(see `_lock_for` above) closes the race between two requests arriving
concurrently, not just sequential retries.
"""
o = _ORDERS.get(oid)
if not o:
raise HTTPException(404, "order saknas")
async with _lock_for(oid):
if oid in _JOBS:
return _JOBS[oid]
o["status"] = "in_production"
_JOBS[oid] = await run_job(oid, o["config"])
o["status"] = "review"
return _JOBS[oid]
@app.get("/orders/{oid}/status")
async def job_status(oid: str):
return _JOBS.get(oid) or {"stage": "queued", "pct": 0, "state": "queued"}
+11
View File
@@ -0,0 +1,11 @@
[pytest]
# services/api's modules import each other as top-level packages
# (`from workers.pipeline import run_job`, `from sources.connectors import ...`)
# relative to this directory — orchestrator/main.py does the same. Running
# pytest from services/api (or with this file as rootdir) needs that on
# sys.path; pytest 8's rootdir-relative testpaths + this ini being IN
# services/api handles that via pytest's automatic rootdir insertion for
# rootdir-relative imports when there is no conftest.py package structure.
testpaths = tests
asyncio_mode = auto
pythonpath = .
+7
View File
@@ -0,0 +1,7 @@
# Test-only dependencies for services/api. NOT installed in the production
# image (Dockerfile only installs requirements.txt) — see pytest.ini for how
# the test suite is run locally / in CI.
-r requirements.txt
pytest==8.3.4
pytest-asyncio==0.25.2
respx==0.22.0
+10
View File
@@ -0,0 +1,10 @@
# 0pt data engine — compliant stack only (docs/07). No bypass/stealth/proxy libs.
fastapi==0.115.6
uvicorn[standard]==0.34.0
pydantic==2.10.4
httpx==0.28.1
asyncpg==0.30.0
python-dotenv==1.0.1
pyyaml==6.0.2
rapidfuzz==3.11.0
tenacity==9.0.0
+1
View File
@@ -0,0 +1 @@
# 0pt data engine — sources package (connectors + registry).
+319
View File
@@ -0,0 +1,319 @@
"""
0pt data engine — COMPLIANT source connectors.
Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market:
1. official API / bulk 2. authorized public web (robots + rate limits)
3. licensed partner data 4. otherwise -> level 0 "ej tillgänglig / licens".
EXPLICITLY NOT PRESENT (and must never be added here):
WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint
rotation to evade bans, scraping behind login walls, fake-account creation.
Every connector returns a normalized company dict + per-field provenance rows
so nothing ships unsourced. `Provenance` (below) carries the full field set
BEFORE_STATE.md §3 found missing from the actual dataflow — this repo's
5-field bridge_provenance was: source, confidence, level, and a DATE-only
freshness; the concept "bridge_provenance" as a named thing did not exist
anywhere in the repo at all. This adds, per field: retrieval TIMESTAMP
(distinct from the legacy date-only `freshness`), the transform applied,
connector/schema version, the producing system, and two slots
(`quality`/`entity_resolution`) that downstream stages (quality bridge,
resolve.py) fill in as the record moves through the pipeline — they start
`None` here and are never backfilled with a guess.
Terms change — VERIFY each source's current access/licence before relying.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from datetime import datetime, timezone
from typing import Any
import httpx
# A polite client that identifies itself and obeys timeouts/rate limits.
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)
# Bumped when a connector's parsing/field-mapping logic changes in a way that
# could shift output shape — recorded per-field as `Provenance.connector_version`
# so a downstream consumer can tell which parsing logic produced a value.
CONNECTOR_VERSIONS: dict[str, str] = {
"gleif": "gleif-connector@1",
"brreg": "brreg-connector@1",
"companies_house": "companies-house-connector@1",
"prh": "prh-connector@1",
"cvr": "cvr-connector@1",
"rdw": "rdw-connector@1",
}
PRODUCER = "0pt-engine"
# P014: statuses a compliant source uses to signal "stop asking me this way"
# (rate-limited, forbidden, or a legal/geo block) rather than "this record
# doesn't exist". Collapsing these into an ordinary None (as every connector
# did before this pass) is not itself a compliance violation -- nothing here
# ever escalates to a bypass -- but it silently discarded a real signal:
# adaptive_router.py had no way to know a source had just told it to back
# off, so nothing could avoid hammering a rate-limited endpoint on the very
# next record. See SourceBlockedError and adaptive_router.record_blocked.
BLOCKED_STATUSES = frozenset({401, 403, 429, 451})
class SourceBlockedError(Exception):
"""
Raised by a connector when the source responds with one of
BLOCKED_STATUSES. This is informational, never an instruction to retry
harder or escalate -- callers (pipeline.py's _fetch loop) catch it only
to record the block via adaptive_router.record_blocked and move on to
the next compliant source in the plan, exactly as they already would for
a plain None. Compliance stance (CLAUDE.md §1) is unchanged: a block is
always a stop signal for THAT source, never a cue to bypass it.
"""
def __init__(self, source_id: str, status_code: int):
self.source_id = source_id
self.status_code = status_code
super().__init__(f"{source_id} signaled a block (HTTP {status_code})")
def _raise_if_blocked(source_id: str, response: httpx.Response) -> None:
if response.status_code in BLOCKED_STATUSES:
raise SourceBlockedError(source_id, response.status_code)
def _client() -> httpx.AsyncClient:
return httpx.AsyncClient(
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
timeout=_TIMEOUT,
follow_redirects=True,
)
@dataclass
class Provenance:
field_name: str
source_id: str
confidence: float
level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
freshness: str | None = None # legacy: retrieval DATE only (kept for back-compat)
retrieved_at: str | None = None # full ISO-8601 UTC timestamp of the actual fetch
transform: str | None = None # e.g. "raw_fetch", "normalized", "entity_resolution_merge"
connector_version: str | None = None # e.g. "brreg-connector@1" — see CONNECTOR_VERSIONS
producer: str = PRODUCER # which system produced this field's value
quality: dict[str, Any] | None = None # filled by the quality bridge, if/when it runs (never guessed)
entity_resolution: dict[str, Any] | None = None # filled by resolve.py on merge, never at fetch time
artifact_ref: str | None = None # delivery artifact reference (ST0RE), set at delivery time — not yet wired
def make_provenance(
field_name: str,
source_id: str,
confidence: float,
level: int,
*,
transform: str = "raw_fetch",
) -> Provenance:
"""Factory used by every connector below so the new provenance fields
(retrieved_at, connector_version, producer) are populated consistently
instead of each call site re-deriving them ad hoc."""
now = datetime.now(timezone.utc)
return Provenance(
field_name=field_name,
source_id=source_id,
confidence=confidence,
level=level,
freshness=now.date().isoformat(),
retrieved_at=now.isoformat(),
transform=transform,
connector_version=CONNECTOR_VERSIONS.get(source_id),
producer=PRODUCER,
)
@dataclass
class CompanyResult:
org_nr: str | None = None
market: str | None = None
lei: str | None = None
name: str | None = None
status: str | None = None
address: dict[str, Any] | None = None
website: str | None = None
provenance: list[Provenance] = field(default_factory=list)
source: str = ""
# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
async def gleif_by_lei(lei: str) -> CompanyResult | None:
url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
async with _client() as c:
r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
_raise_if_blocked("gleif", r)
if r.status_code != 200:
return None
a = (r.json().get("data") or {}).get("attributes") or {}
entity = a.get("entity") or {}
legal = entity.get("legalAddress") or {}
res = CompanyResult(
lei=lei,
name=(entity.get("legalName") or {}).get("name"),
status=entity.get("status"),
address={"lines": legal.get("addressLines"), "city": legal.get("city"),
"country": legal.get("country"), "postal": legal.get("postalCode")},
source="gleif",
)
res.provenance += [
make_provenance("lei", "gleif", 0.99, 3),
make_provenance("name", "gleif", 0.97, 3),
]
return res
# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ─────────────
async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
async with _client() as c:
r = await c.get(url)
_raise_if_blocked("brreg", r)
if r.status_code != 200:
return None
d = r.json()
res = CompanyResult(
org_nr=orgnr, market="NO",
name=d.get("navn"),
status="dissolved" if d.get("konkurs") else "active",
address={"raw": d.get("forretningsadresse")},
source="brreg",
)
res.provenance += [
make_provenance("name", "brreg", 0.97, 3),
make_provenance("status", "brreg", 0.95, 3),
make_provenance("address", "brreg", 0.95, 3),
]
return res
# ── Companies House (UK) — free API, key required. VERIFY terms. ────────────
async def companies_house_by_number(number: str) -> CompanyResult | None:
key = os.getenv("COMPANIES_HOUSE_API_KEY")
if not key:
# No key -> deliver honestly as unavailable, never fabricate.
return CompanyResult(org_nr=number, market="GB", source="companies_house",
provenance=[make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")])
url = f"https://api.company-information.service.gov.uk/company/{number}"
async with _client() as c:
r = await c.get(url, auth=(key, ""))
_raise_if_blocked("companies_house", r)
if r.status_code != 200:
return None
d = r.json()
res = CompanyResult(
org_nr=number, market="GB",
name=d.get("company_name"),
status=d.get("company_status"),
address={"raw": d.get("registered_office_address")},
source="companies_house",
)
res.provenance += [
make_provenance("name", "companies_house", 0.97, 3),
make_provenance("status", "companies_house", 0.95, 3),
]
return res
# ── PRH avoindata (FI) — open API (CC-BY). VERIFY terms. ────────────────────
async def prh_by_businessid(business_id: str) -> CompanyResult | None:
url = f"https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId={business_id}"
async with _client() as c:
r = await c.get(url)
_raise_if_blocked("prh", r)
if r.status_code != 200:
return None
items = r.json().get("companies") or r.json().get("results") or []
if not items:
return None
d = items[0]
names = d.get("names") or []
name = names[0].get("name") if names else d.get("name")
res = CompanyResult(org_nr=business_id, market="FI", name=name, status=d.get("status"), source="prh")
if name:
res.provenance.append(make_provenance("name", "prh", 0.95, 3))
return res
# ── CVR (DK) via cvrapi.dk — free for low volume (descriptive UA). VERIFY. ───
async def cvr_by_number(cvr: str) -> CompanyResult | None:
url = f"https://cvrapi.dk/api?search={cvr}&country=dk"
async with _client() as c:
r = await c.get(url)
_raise_if_blocked("cvr", r)
if r.status_code != 200:
return None
d = r.json()
if not d.get("name"):
return None
res = CompanyResult(
org_nr=cvr, market="DK", name=d.get("name"),
status="ceased" if d.get("enddate") else "active",
address={"raw": d.get("address")}, source="cvr",
)
res.provenance += [make_provenance("name", "cvr", 0.93, 3), make_provenance("address", "cvr", 0.9, 2)]
return res
# ── RDW (NL) open vehicle data — Socrata. Technical/fleet only, no owner PII. ─
async def rdw_vehicle(kenteken: str) -> dict[str, Any] | None:
plate = kenteken.upper().replace("-", "").replace(" ", "")
url = f"https://opendata.rdw.nl/resource/m9d7-ebf2.json?kenteken={plate}"
async with _client() as c:
r = await c.get(url)
r.raise_for_status()
arr = r.json()
if not isinstance(arr, list) or any(not isinstance(item, dict) for item in arr):
raise ValueError("invalid RDW response")
if not arr:
return None
d = arr[0]
p = make_provenance("make", "rdw", 0.9, 3)
return {
"reg_plate": kenteken, "market": "NL",
"make": d.get("merk"), "model": d.get("handelsbenaming"),
"year": (d.get("datum_eerste_toelating") or "")[:4] or None,
"source": "rdw",
# Dict shape (not the Provenance dataclass) — this endpoint predates it
# and is returned directly as an API response body, not merged through
# resolve.py. Field names kept aligned with Provenance's for consistency.
"provenance": [{
"field": p.field_name, "source_id": p.source_id, "confidence": p.confidence, "level": p.level,
"freshness": p.freshness, "retrieved_at": p.retrieved_at, "transform": p.transform,
"connector_version": p.connector_version, "producer": p.producer,
}],
}
# ── OpenSanctions screening — search API. VERIFY commercial terms / key. ─────
async def opensanctions_screen(name: str) -> dict[str, Any]:
url = "https://api.opensanctions.org/search/default"
async with _client() as c:
r = await c.get(url, params={"q": name, "limit": 5})
r.raise_for_status()
payload = r.json()
if not isinstance(payload, dict) or not isinstance(payload.get("results"), list) or any(
not isinstance(item, dict) for item in payload["results"]
):
raise ValueError("invalid OpenSanctions response")
results = payload["results"]
return {
"source": "opensanctions",
"matches": [{"name": x.get("caption"), "score": x.get("score"), "schema": x.get("schema")} for x in results],
}
CONNECTORS = {
"gleif": gleif_by_lei,
"brreg": brreg_by_orgnr,
"companies_house": companies_house_by_number,
"prh": prh_by_businessid,
"cvr": cvr_by_number,
}
+34
View File
@@ -0,0 +1,34 @@
"""
Structured financials (docs/06). XBRL/iXBRL is the fast, exact lane (not OCR).
Companies House publishes accounts as iXBRL; this fetches the accounts filing
history (real, key-gated) and documents the XBRL extraction step.
"""
from __future__ import annotations
import os
from typing import Any
import httpx
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX)"
async def companies_house_filings(number: str) -> dict[str, Any]:
key = os.getenv("COMPANIES_HOUSE_API_KEY")
if not key:
return {"company": number, "filings": [], "note": "level0: kräver COMPANIES_HOUSE_API_KEY"}
url = f"https://api.company-information.service.gov.uk/company/{number}/filing-history?category=accounts"
async with httpx.AsyncClient(headers={"User-Agent": USER_AGENT}, timeout=15) as c:
r = await c.get(url, auth=(key, ""))
if r.status_code != 200:
return {"company": number, "filings": []}
items = r.json().get("items", [])
return {
"company": number,
"source": "companies_house",
"filings": [{"date": i.get("date"), "type": i.get("type"), "description": i.get("description")} for i in items[:10]],
}
# XBRL extraction (extension): with `arelle` installed, download each accounts
# filing's iXBRL document and pull tagged facts (turnover, profit, employees)
# into fact_financials with provenance. Structured, exact — never OCR for XBRL.
+99
View File
@@ -0,0 +1,99 @@
# 0pt data engine — sources registry (docs/06). Drives the router, /sources API,
# and the public sourcing-map. access: api | bulk | authorized_web | licensed.
# VERIFY current access/licence/robots per source before relying — terms change.
sources:
- source_id: gleif
name: GLEIF (LEI)
market: global
access: api
cadence: daily
licence: "open (CC0) — VERIFY"
entities: [company]
fields: [lei, legal_name, address, status]
base_url: https://api.gleif.org
robots_ok: true
notes: "cross-border key"
- source_id: brreg
name: Brønnøysundregistrene (Enhetsregisteret)
market: NO
access: api
cadence: daily
licence: "NLOD open — VERIFY"
entities: [company]
fields: [name, status, address, nace]
base_url: https://data.brreg.no
robots_ok: true
notes: "open API + bulk download"
- source_id: companies_house
name: Companies House
market: GB
access: api
cadence: daily
licence: "free, API key required — VERIFY"
entities: [company, officers, financials]
fields: [name, status, address, officers]
base_url: https://api.company-information.service.gov.uk
robots_ok: true
notes: "key via env COMPANIES_HOUSE_API_KEY"
- source_id: prh
name: PRH avoindata (Trade Register)
market: FI
access: api
cadence: daily
licence: "CC-BY — VERIFY"
entities: [company]
fields: [name, status]
base_url: https://avoindata.prh.fi
robots_ok: true
notes: "connector pending verification"
- source_id: cvr
name: CVR / Virk
market: DK
access: api
cadence: daily
licence: "terms apply (Virk) — VERIFY"
entities: [company]
fields: [name, status]
base_url: https://datacvr.virk.dk
robots_ok: true
notes: "connector pending verification"
- source_id: bolagsverket
name: Bolagsverket
market: SE
access: licensed
cadence: daily
licence: "restricted for bulk — partner/licensed route"
entities: [company]
fields: [name, status]
base_url: https://bolagsverket.se
robots_ok: true
notes: "no bulk scrape; licensed/partner only"
- source_id: rdw
name: RDW open vehicle data
market: NL
access: bulk
cadence: daily
licence: "open — VERIFY"
entities: [vehicle]
fields: [make, model, year, specs]
base_url: https://opendata.rdw.nl
robots_ok: true
notes: "technical/fleet vehicle data, not owner PII"
- source_id: opensanctions
name: OpenSanctions
market: global
access: bulk
cadence: daily
licence: "open — VERIFY commercial terms"
entities: [company, person]
fields: [sanction, pep]
base_url: https://www.opensanctions.org
robots_ok: true
notes: "screening/diligence enrichment"
@@ -0,0 +1,77 @@
"""
P014: adaptive_router.py previously had ZERO test coverage and a docstring
("all methods, no compliance restrictions. See b0ts for scraping") that
directly contradicted this product's own compliance stance and pointed at a
forbidden, separate, non-compliant repo. This file tests the corrected
version: plan_for_market's static per-market plan, and the new block-
tracking (record_blocked/is_blocked/cooldown) that makes the engine actually
back off a source that just told it to stop, instead of retrying it with no
memory at all.
"""
from __future__ import annotations
from workers.adaptive_router import AdaptiveRouter, MARKET_SOURCES, plan_for_market
class TestPlanForMarket:
def test_returns_the_full_static_plan_when_nothing_is_blocked(self):
router = AdaptiveRouter()
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
def test_unknown_market_falls_back_to_gleif_only(self):
router = AdaptiveRouter()
assert plan_for_market("ZZ", router=router) == ["gleif"]
def test_defaults_to_the_process_wide_router_when_none_is_passed(self):
# Just proves the call doesn't blow up and returns a real plan --
# not asserting anything about global state here (other tests use
# their own injected router precisely to avoid cross-test pollution
# of the shared default_router).
assert plan_for_market("FI") == MARKET_SOURCES["FI"]
class TestBlockTracking:
def test_a_freshly_blocked_source_is_excluded_from_the_plan(self):
router = AdaptiveRouter()
router.record_blocked("GB", "companies_house")
plan = plan_for_market("GB", router=router)
assert "companies_house" not in plan
assert "gleif" in plan # only the blocked source is removed, not the whole market
def test_is_blocked_is_false_before_any_block_is_recorded(self):
router = AdaptiveRouter()
assert router.is_blocked("GB", "companies_house") is False
def test_blocking_is_scoped_per_market_not_global_to_the_source_id(self):
# gleif appears in every market's plan -- blocking it for GB must not
# silently remove it from FI/SE/etc too.
router = AdaptiveRouter()
router.record_blocked("GB", "gleif")
assert "gleif" not in plan_for_market("GB", router=router)
assert "gleif" in plan_for_market("FI", router=router)
assert "gleif" in plan_for_market("SE", router=router)
def test_cooldown_expires_and_the_source_becomes_eligible_again(self):
router = AdaptiveRouter(cooldown_seconds=60.0)
router.record_blocked("GB", "companies_house", now=1000.0)
assert router.is_blocked("GB", "companies_house", now=1030.0) is True # still within cooldown
assert router.is_blocked("GB", "companies_house", now=1061.0) is False # cooldown elapsed
# And the plan reflects that:
assert "companies_house" in plan_for_market("GB", router=router)
def test_clear_forgets_every_recorded_block(self):
router = AdaptiveRouter()
router.record_blocked("GB", "companies_house")
router.record_blocked("NO", "brreg")
router.clear()
assert plan_for_market("GB", router=router) == MARKET_SOURCES["GB"]
assert plan_for_market("NO", router=router) == MARKET_SOURCES["NO"]
def test_two_router_instances_never_share_state(self):
# Regression guard for a shared-mutable-default bug class: each
# AdaptiveRouter() must own its own dict, not a class-level one.
a = AdaptiveRouter()
b = AdaptiveRouter()
a.record_blocked("GB", "companies_house")
assert a.is_blocked("GB", "companies_house") is True
assert b.is_blocked("GB", "companies_house") is False
@@ -0,0 +1,104 @@
"""
P014: every connector used by pipeline.py's adaptive plan (gleif, brreg,
companies_house, prh, cvr) used to collapse EVERY non-200 response into a
bare `None`, indistinguishable from "record genuinely doesn't exist". This
tests that the block-status subset (401/403/429/451) now raises
SourceBlockedError instead, while every other non-200 status (404, 500, ...)
still returns None exactly as before -- this is an ADDITIVE signal, not a
behavior change for the non-block case.
"""
from __future__ import annotations
import httpx
import pytest
import respx
from sources.connectors import (
SourceBlockedError,
brreg_by_orgnr,
companies_house_by_number,
cvr_by_number,
gleif_by_lei,
prh_by_businessid,
)
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/5493001KJTIIGC8Y1R12"
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
CVR_URL = "https://cvrapi.dk/api?search=10150817&country=dk"
@pytest.mark.asyncio
@respx.mock
@pytest.mark.parametrize("status", [401, 403, 429, 451])
async def test_gleif_raises_source_blocked_on_block_statuses(status):
respx.get(GLEIF_URL).mock(return_value=httpx.Response(status, json={}))
with pytest.raises(SourceBlockedError) as exc_info:
await gleif_by_lei("5493001KJTIIGC8Y1R12")
assert exc_info.value.source_id == "gleif"
assert exc_info.value.status_code == status
@pytest.mark.asyncio
@respx.mock
async def test_gleif_a_404_still_returns_none_not_blocked(monkeypatch):
respx.get(GLEIF_URL).mock(return_value=httpx.Response(404, json={}))
result = await gleif_by_lei("5493001KJTIIGC8Y1R12")
assert result is None # unchanged pre-P014 behavior for a genuine "not found"
@pytest.mark.asyncio
@respx.mock
async def test_brreg_raises_source_blocked_on_429():
respx.get(BRREG_URL).mock(return_value=httpx.Response(429, json={}))
with pytest.raises(SourceBlockedError) as exc_info:
await brreg_by_orgnr("923609016")
assert exc_info.value.source_id == "brreg"
@pytest.mark.asyncio
@respx.mock
async def test_companies_house_raises_source_blocked_on_403(monkeypatch):
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(403, json={}))
with pytest.raises(SourceBlockedError) as exc_info:
await companies_house_by_number("00000006")
assert exc_info.value.source_id == "companies_house"
@pytest.mark.asyncio
async def test_companies_house_missing_key_still_delivers_honest_level_0_not_a_block():
# No COMPANIES_HOUSE_API_KEY set -- this is a config gap, not a source
# block, and must keep returning the honest "unavailable" result it
# already did (never raise SourceBlockedError for a problem on OUR side).
result = await companies_house_by_number("00000006")
assert result is not None
assert result.provenance[0].level == 0
assert result.provenance[0].transform == "unavailable_no_key"
@pytest.mark.asyncio
@respx.mock
async def test_prh_raises_source_blocked_on_401():
respx.get(PRH_URL).mock(return_value=httpx.Response(401, json={}))
with pytest.raises(SourceBlockedError) as exc_info:
await prh_by_businessid("2077474-0")
assert exc_info.value.source_id == "prh"
@pytest.mark.asyncio
@respx.mock
async def test_cvr_raises_source_blocked_on_451():
respx.get(CVR_URL).mock(return_value=httpx.Response(451, json={}))
with pytest.raises(SourceBlockedError) as exc_info:
await cvr_by_number("10150817")
assert exc_info.value.source_id == "cvr"
@pytest.mark.asyncio
@respx.mock
async def test_cvr_a_500_still_returns_none_not_blocked():
respx.get(CVR_URL).mock(return_value=httpx.Response(500, json={}))
result = await cvr_by_number("10150817")
assert result is None
@@ -0,0 +1,58 @@
"""
Verifies the P010 provenance additions (BEFORE_STATE.md §3: source,
retrieval timestamp, transform, version, producer were absent from the real
dataflow) land correctly on every real connector's output — mocked HTTP,
real parsing code.
"""
from __future__ import annotations
import httpx
import pytest
import respx
from sources.connectors import brreg_by_orgnr, make_provenance, CONNECTOR_VERSIONS
class TestMakeProvenanceFactory:
def test_populates_all_new_fields_honestly_and_leaves_the_rest_none(self):
p = make_provenance("name", "brreg", 0.97, 3)
assert p.field_name == "name"
assert p.source_id == "brreg"
assert p.retrieved_at is not None
assert p.freshness == p.retrieved_at[:10] # date prefix of the same instant
assert p.transform == "raw_fetch"
assert p.connector_version == CONNECTOR_VERSIONS["brreg"]
assert p.producer == "0pt-engine"
# Never guessed — these are filled by later pipeline stages, not at fetch time.
assert p.quality is None
assert p.entity_resolution is None
assert p.artifact_ref is None
def test_transform_override_is_honored(self):
p = make_provenance("name", "companies_house", 0.0, 0, transform="unavailable_no_key")
assert p.transform == "unavailable_no_key"
def test_unknown_source_id_has_no_fabricated_version(self):
p = make_provenance("x", "some_future_source", 0.5, 1)
assert p.connector_version is None # honestly unknown, not guessed
class TestBrregConnectorProvenance:
@pytest.mark.asyncio
@respx.mock
async def test_real_parsing_path_produces_fully_populated_provenance(self):
respx.get("https://data.brreg.no/enhetsregisteret/api/enheter/923609016").mock(
return_value=httpx.Response(200, json={
"navn": "Volvo Personvagnar Norge AS", "konkurs": False,
"forretningsadresse": {"adresse": ["Storgata 1"]},
})
)
res = await brreg_by_orgnr("923609016")
assert res is not None
assert len(res.provenance) == 3 # name, status, address
for p in res.provenance:
assert p.source_id == "brreg"
assert p.connector_version == "brreg-connector@1"
assert p.producer == "0pt-engine"
assert p.retrieved_at is not None
assert p.transform == "raw_fetch"
@@ -0,0 +1,93 @@
from workers.entity_resolution import (
EntityCandidate,
ResolutionDecision,
rank_resolution_candidates,
resolve_entity_pair,
)
def organization(entity_id: str, **overrides):
values = {
"entity_id": entity_id,
"entity_type": "organization",
"name": "Example Technology AB",
"source_ref": f"source:{entity_id}",
"registration_country": "SE",
"registration_number": None,
"website": None,
"city": "Stockholm",
}
values.update(overrides)
return EntityCandidate(**values)
def person(entity_id: str, **overrides):
values = {
"entity_id": entity_id,
"entity_type": "person",
"name": "Anna Andersson",
"source_ref": f"source:{entity_id}",
"email": None,
"organization_id": None,
}
values.update(overrides)
return EntityCandidate(**values)
def test_registration_number_is_a_strong_organization_identifier():
result = resolve_entity_pair(
organization("left", registration_number="556123-4567"),
organization("right", registration_number="5561234567"),
)
assert result.decision is ResolutionDecision.EXACT_LINK
assert result.score == 1.0
assert result.requires_human_review is False
def test_conflicting_registration_numbers_fail_distinct_even_when_names_match():
result = resolve_entity_pair(
organization("left", registration_number="5561234567"),
organization("right", registration_number="5569999999"),
)
assert result.decision is ResolutionDecision.DISTINCT
assert result.score == 0.0
def test_same_person_email_is_exact_link():
result = resolve_entity_pair(
person("left", email="Anna@example.com"),
person("right", email=" anna@EXAMPLE.com "),
)
assert result.decision is ResolutionDecision.EXACT_LINK
assert result.requires_human_review is False
def test_fuzzy_organization_similarity_never_auto_merges():
result = resolve_entity_pair(
organization("left", name="Example Technology AB", website="example.se"),
organization("right", name="Example Technologies AB", website="https://www.example.se"),
)
assert result.score > 0.72
assert result.decision is ResolutionDecision.REVIEW
assert result.requires_human_review is True
def test_different_entity_types_are_distinct():
result = resolve_entity_pair(
organization("org"),
person("person"),
)
assert result.decision is ResolutionDecision.DISTINCT
def test_candidate_ranking_prefers_exact_then_review_and_is_stable():
query = organization("query", registration_number="5561234567", website="example.se")
candidates = [
organization("weak", name="Completely Different Company", registration_country="NO", city="Oslo"),
organization("review", name="Example Technologies AB", website="example.se"),
organization("exact", registration_number="556123-4567"),
]
ranked = rank_resolution_candidates(query, candidates)
assert [result.right_id for result in ranked] == ["exact", "review", "weak"]
assert ranked[0].decision is ResolutionDecision.EXACT_LINK
assert ranked[1].decision is ResolutionDecision.REVIEW
@@ -0,0 +1,70 @@
"""The engine must not accept anonymous traffic when INTERNAL_TOKEN is set.
Background (2026-09-15): lib/engine.ts on the edge has always SENT
`X-Internal-Token`, but the engine never verified it — a deployed engine would
have accepted anyone who could reach the port. The gate is fail-closed:
* INTERNAL_TOKEN set -> every route except /health requires the EXACT
token; wrong/missing = 401.
* INTERNAL_TOKEN unset -> gate inert (local dev); production compose always
sets it.
`/health` stays public on purpose: it is the liveness probe that
`lib/health-checks.ts` calls from the edge, and it leaks nothing but booleans.
The token is read at module import (the same value the running process serves
with), so each test reloads the module with the env it wants — and the
teardown reloads it one more time with the env CLEARED so later test files in
the same session never inherit a baked-in token.
"""
from __future__ import annotations
import importlib
import os
import pytest
from fastapi.testclient import TestClient
@pytest.fixture()
def engine(monkeypatch):
monkeypatch.setenv("INTERNAL_TOKEN", "s3cret")
import orchestrator.main as main
importlib.reload(main)
yield TestClient(main.app)
os.environ.pop("INTERNAL_TOKEN", None)
importlib.reload(main)
@pytest.fixture()
def engine_no_token(monkeypatch):
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
import orchestrator.main as main
importlib.reload(main)
yield TestClient(main.app)
os.environ.pop("INTERNAL_TOKEN", None)
importlib.reload(main)
def test_gate_off_when_internal_token_unset(engine_no_token):
assert engine_no_token.get("/health").status_code == 200
assert engine_no_token.get("/sources").status_code == 200
def test_health_stays_public_but_sources_require_token(engine):
# liveness probe is public by design
assert engine.get("/health").status_code == 200
# anonymous call to a data route -> 401 (was 200 before 2026-09-15)
r = engine.get("/sources")
assert r.status_code == 401
assert r.json()["error"]["code"] == "unauthorized"
def test_wrong_token_is_401_and_correct_token_passes(engine):
wrong = engine.get("/estimates?markets=SE", headers={"X-Internal-Token": "nope"})
assert wrong.status_code == 401
right = engine.get("/sources", headers={"X-Internal-Token": "s3cret"})
assert right.status_code == 200
+53
View File
@@ -0,0 +1,53 @@
"""
Honest documentation of two of the seven requested failure-test categories
that are NOT implemented as real fault-injection tests in this pass, and
why — matching this repo's established convention of stating a gap in code
(BEFORE_STATE.md, docs/P0_DATASET_QUALITY_FLOW.md) rather than silently
dropping it or faking coverage. These are `pytest.skip` markers, not
disguised passes, so they show up in the test report as explicitly
unaddressed rather than vanishing from view.
* CDC-interruption: this repo has NO change-data-capture pipeline
anywhere — no Debezium/WAL-slot consumer, no CDC producer, no
replication-slot code, confirmed by a full-repo search
(`grep -rn "debezium\\|wal_level\\|replication.slot\\|cdc" services/ src/`
finds nothing on this topic). A "CDC-interruption" fault-injection
test needs a CDC stream to interrupt; fabricating one to have
something to test would be building fictional infrastructure and
testing the fiction, not this codebase — the exact failure mode this
project's CLAUDE.md and BEFORE_STATE.md explicitly warn against
("never fabricate").
* True backfill idempotency (dedup against a delivery store): the
pipeline has no persistence layer at all (see workers/pipeline.py —
every run_job() call recomputes from scratch, nothing is written to a
database). "Backfill doesn't create duplicate deliveries" requires a
delivery store to check duplicates against, which doesn't exist.
test_pipeline_fault_injection.py::TestBackfillReplay covers the
narrower, real property that DOES hold today (replaying an identical
order config is compute-idempotent — byte-identical output, no drift)
and documents this exact distinction inline.
"""
import pytest
@pytest.mark.skip(
reason=(
"No CDC pipeline exists anywhere in this repo (verified by full-repo "
"search) — nothing to interrupt without inventing fictional "
"infrastructure. See this file's module docstring."
)
)
def test_cdc_interruption_not_implementable_against_real_code():
pass
@pytest.mark.skip(
reason=(
"No delivery-store persistence layer exists to dedupe a backfill "
"against — the compute-idempotency property that DOES exist is "
"covered for real in test_pipeline_fault_injection.py::"
"TestBackfillReplay. See this file's module docstring."
)
)
def test_true_backfill_idempotency_not_implementable_against_real_code():
pass
@@ -0,0 +1,112 @@
"""
P014: services/api/orchestrator/main.py had ZERO test coverage before this
pass, and `POST /orders/{oid}/run` had no idempotency protection at all --
calling it twice for the same order re-ran the full compliant sourcing
pipeline a second time, silently overwriting `_JOBS[oid]` with a second,
independently-produced delivery. This is the actual "duplicate delivery"
gap P014's recon flagged NOT MET (nothing downstream of order creation was
idempotent). These tests exercise the real ASGI app end to end (httpx
ASGITransport, no mocked HTTP layer) and prove: a sequential retry returns
the cached job without re-running the pipeline, a TRUE CONCURRENT race
(two requests in flight at once) is also closed by the per-order asyncio
lock, and an unknown order still 404s exactly as before.
"""
from __future__ import annotations
import asyncio
import httpx
import pytest
from orchestrator.main import app
def _fake_job_result(calls: list[str], order_id: str) -> dict:
calls.append(order_id)
return {
"stage": "qa",
"pct": 100,
"state": "done",
"metrics": {"rows": 1, "dropped_below_threshold": 0, "per_market": {"NO": 1}, "avg_confidence": 0.95},
"records": [{"market": "NO", "name": "Test AS", "confidence": 0.95}],
}
@pytest.mark.asyncio
async def test_sequential_retry_returns_cached_job_without_rerunning_pipeline(monkeypatch):
calls: list[str] = []
async def fake_run_job(order_id: str, config: dict) -> dict:
return _fake_job_result(calls, order_id)
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
oid = created.json()["order_id"]
r1 = await client.post(f"/orders/{oid}/run")
r2 = await client.post(f"/orders/{oid}/run")
r3 = await client.post(f"/orders/{oid}/run")
assert r1.status_code == 200
assert r1.json() == r2.json() == r3.json()
assert calls == [oid] # run_job invoked exactly once across three /run calls
@pytest.mark.asyncio
async def test_true_concurrent_race_still_invokes_run_job_exactly_once(monkeypatch):
calls: list[str] = []
async def fake_run_job(order_id: str, config: dict) -> dict:
# Widen the race window: both requests reach this await before
# either has written to _JOBS, which is exactly the race a bare
# `if oid in _JOBS` check (no lock) would lose.
await asyncio.sleep(0.05)
return _fake_job_result(calls, order_id)
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
created = await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})
oid = created.json()["order_id"]
r1, r2 = await asyncio.gather(
client.post(f"/orders/{oid}/run"),
client.post(f"/orders/{oid}/run"),
)
assert r1.status_code == 200 and r2.status_code == 200
assert r1.json() == r2.json()
assert calls == [oid] # THE regression this test guards: exactly one real pipeline run
@pytest.mark.asyncio
async def test_run_on_unknown_order_still_404s():
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
res = await client.post("/orders/does-not-exist/run")
assert res.status_code == 404
@pytest.mark.asyncio
async def test_different_orders_each_get_their_own_real_run(monkeypatch):
calls: list[str] = []
async def fake_run_job(order_id: str, config: dict) -> dict:
return _fake_job_result(calls, order_id)
monkeypatch.setattr("orchestrator.main.run_job", fake_run_job)
transport = httpx.ASGITransport(app=app)
async with httpx.AsyncClient(transport=transport, base_url="http://test") as client:
o1 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["NO"]}})).json()["order_id"]
o2 = (await client.post("/orders", json={"flow": "custom", "config": {"markets": ["GB"]}})).json()["order_id"]
await client.post(f"/orders/{o1}/run")
await client.post(f"/orders/{o2}/run")
# Idempotency is per-order, not global -- two distinct orders both really run.
assert sorted(calls) == sorted([o1, o2])
@@ -0,0 +1,162 @@
"""
Fault-injection tests for workers/pipeline.py's run_job — the REAL order
pipeline (not a mock of it). BEFORE_STATE.md §4 found schema-evolution,
CDC-interruption, backfill, and restart entirely untested. This file closes
what can be closed against code that actually exists, and says explicitly
why the rest can't be, instead of fabricating a harness for infrastructure
this repo doesn't have.
All external HTTP is mocked with respx (never hits real registries) so this
suite is deterministic and offline.
"""
from __future__ import annotations
import httpx
import pytest
import respx
from workers.pipeline import run_job
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
PRH_URL = "https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId=2077474-0"
VALID_BRREG_BODY = {
"navn": "Volvo Personvagnar Norge AS",
"konkurs": False,
"forretningsadresse": {"adresse": ["Storgata 1"], "poststed": "OSLO"},
}
class TestRestartMidFetch:
"""A worker crash/restart mid-job is not literally simulate-able (no
checkpointing exists to resume from — the pipeline is a single async
function with no persisted intermediate state). What this DOES test,
honestly: a connector raising mid-job (network reset, upstream 5xx,
connection drop — the kind of fault a real restart would also surface
as) never crashes the whole order; the job completes and delivers
whatever it validly could."""
@pytest.mark.asyncio
@respx.mock
async def test_a_connector_exception_does_not_crash_the_order(self):
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("connection reset"))
respx.get(PRH_URL).mock(return_value=httpx.Response(200, json={
"companies": [{"names": [{"name": "Suomi Oy"}], "status": "active"}],
}))
result = await run_job(
"order_restart_1",
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
)
assert result["state"] == "done"
# NO failed entirely (connector raised); FI still delivered.
assert result["metrics"]["per_market"]["NO"] == 0
assert result["metrics"]["per_market"]["FI"] == 1
assert len(result["records"]) == 1
assert result["records"][0]["market"] == "FI"
@pytest.mark.asyncio
@respx.mock
async def test_every_connector_failing_delivers_zero_rows_not_a_crash(self):
respx.get(BRREG_URL).mock(side_effect=httpx.ConnectError("down"))
respx.get(PRH_URL).mock(side_effect=httpx.ConnectError("down"))
result = await run_job(
"order_restart_2",
{"markets": ["NO", "FI"], "confidence": 50, "seeds": {"NO": ["923609016"], "FI": ["2077474-0"]}},
)
assert result["state"] == "done"
assert result["records"] == []
assert result["metrics"]["rows"] == 0
class TestSchemaEvolution:
"""Upstream registries adding/removing/renaming fields without notice is
a real risk (BEFORE_STATE.md flagged this as wholly untested). Verified
against the actual HTTP-parsing boundary in sources/connectors.py, via
respx-mocked responses shaped like plausible upstream drift — not a
fabricated schema-registry harness (this repo has no schema
versioning/CDC system to test against; see test_known_gaps.py)."""
@pytest.mark.asyncio
@respx.mock
async def test_upstream_adding_unexpected_new_fields_does_not_break_parsing(self):
drifted_body = {
**VALID_BRREG_BODY,
# Simulated schema drift: brreg adds fields this connector never asked for.
"stiftelsesdato": "2020-01-01",
"naeringskode1": {"kode": "45.11", "beskrivelse": "Handel"},
"nested_new_thing": {"a": {"b": [1, 2, 3]}},
}
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=drifted_body))
result = await run_job(
"order_schema_1",
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
)
assert result["state"] == "done"
assert len(result["records"]) == 1
assert result["records"][0]["name"] == "Volvo Personvagnar Norge AS"
@pytest.mark.asyncio
@respx.mock
async def test_upstream_removing_an_expected_field_degrades_honestly_not_a_crash(self):
# Real drift case: upstream drops "navn" (renamed/moved). The
# connector must not crash, and must not fabricate a name — the
# record is simply not produced (pipeline's existing `if res and
# res.name` gate), which is the correct "level 0 / not available"
# behavior, not a bug to work around.
stripped_body = {k: v for k, v in VALID_BRREG_BODY.items() if k != "navn"}
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=stripped_body))
result = await run_job(
"order_schema_2",
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
)
assert result["state"] == "done"
assert result["records"] == [] # honestly nothing delivered, not a fabricated name
@pytest.mark.asyncio
@respx.mock
async def test_upstream_returning_a_wholly_different_shape_does_not_crash(self):
# Worst case: upstream returns something structurally alien (e.g. a
# maintenance-page JSON blob instead of a company record).
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json={"error": "maintenance", "retry_after": 60}))
result = await run_job(
"order_schema_3",
{"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}},
)
assert result["state"] == "done"
assert result["records"] == []
class TestBackfillReplay:
"""This pipeline has NO persistence layer (BEFORE_STATE.md — every
run_job call recomputes from scratch; there is no delivery store to
dedupe a backfill against). A true "backfill doesn't create duplicate
deliveries" test would need that store and is NOT fabricated here. What
IS real and testable: replaying the identical order config (the exact
scenario a backfill re-run performs) is idempotent AT THE COMPUTE LEVEL
— same input, byte-identical output — so a backfill re-run at least
doesn't silently drift or become order-dependent across replays."""
@pytest.mark.asyncio
@respx.mock
async def test_replaying_the_same_order_config_yields_byte_identical_delivery(self):
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
config = {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}}
first = await run_job("order_backfill", config)
second = await run_job("order_backfill", config)
# Strip the per-run artifact_ref/timestamps (legitimately differ —
# every replay IS a new delivery event) before comparing the
# substantive content.
def strip_run_identity(result):
recs = []
for r in result["records"]:
rec = {k: v for k, v in r.items()}
rec["provenance"] = [
{k: v for k, v in p.items() if k not in ("retrieved_at", "artifact_ref")} for p in rec["provenance"]
]
recs.append(rec)
return recs
assert strip_run_identity(first) == strip_run_identity(second)
assert first["metrics"]["rows"] == second["metrics"]["rows"] == 1
@@ -0,0 +1,114 @@
"""
P014: integration proof that a source signaling a block (HTTP 401/403/429/451)
during a real run_job() call (a) never crashes or fabricates a result, (b) is
recorded on the process-wide adaptive_router so the engine backs off that
source, and (c) the pipeline still delivers from the next compliant source in
the plan -- exactly the "source-racing several authorized sources" behavior
the siax-0pt skill documents, now with an honest memory of blocks instead of
silently re-trying a rate-limited source with none.
Uses the real default_router (module-level singleton pipeline.py actually
uses), cleared before and after every test in this file so this suite never
leaks state into -- or picks up state from -- any other test module that
imports workers.pipeline.
"""
from __future__ import annotations
import httpx
import pytest
import respx
from workers.adaptive_router import default_router
from workers.pipeline import run_job
COMPANIES_HOUSE_URL = "https://api.company-information.service.gov.uk/company/00000006"
GLEIF_URL = "https://api.gleif.org/api/v1/lei-records/00000006"
@pytest.fixture(autouse=True)
def _clean_router():
default_router.clear()
yield
default_router.clear()
class TestSourceBlockedDuringARealRun:
@pytest.mark.asyncio
@respx.mock
async def test_a_429_from_one_source_is_recorded_as_blocked_and_the_run_still_delivers_from_the_next_source(
self, monkeypatch
):
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={"error": "rate limited"}))
respx.get(GLEIF_URL).mock(
return_value=httpx.Response(
200,
json={"data": {"attributes": {"entity": {"legalName": {"name": "Fallback Ltd"}, "status": "ACTIVE", "legalAddress": {}}}}},
)
)
assert default_router.is_blocked("GB", "companies_house") is False # sanity: clean slate
result = await run_job("order_block_1", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
assert result["state"] == "done"
assert result["metrics"]["rows"] == 1
assert result["records"][0]["name"] == "Fallback Ltd"
assert result["records"][0]["source"] == "gleif" # companies_house was blocked, gleif answered
# THE regression this test guards: the block was actually recorded,
# not silently discarded the way a bare `except Exception: res = None`
# would (that's still the fallback for genuine errors, but a block is
# now distinguishable from one).
assert default_router.is_blocked("GB", "companies_house") is True
@pytest.mark.asyncio
@respx.mock
async def test_a_blocked_source_is_excluded_from_the_plan_on_the_very_next_key_in_the_same_run(self, monkeypatch):
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
gb_second_url = "https://api.company-information.service.gov.uk/company/11111111"
gleif_second_url = "https://api.gleif.org/api/v1/lei-records/11111111"
companies_house_calls = respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(429, json={}))
# No mock registered for gb_second_url at all -- if the pipeline tried
# companies_house AGAIN for the second key after it was just blocked,
# respx would raise on the unmocked request and this test would fail
# with a connection error, not a clean assertion. That absence IS the
# assertion: the plan excluded the just-blocked source.
respx.get(GLEIF_URL).mock(
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "First"}, "legalAddress": {}}}}})
)
respx.get(gleif_second_url).mock(
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "Second"}, "legalAddress": {}}}}})
)
result = await run_job(
"order_block_2",
{"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006", "11111111"]}},
)
assert result["state"] == "done"
assert result["metrics"]["rows"] == 2
names = {r["name"] for r in result["records"]}
assert names == {"First", "Second"}
# companies_house was hit exactly once (for the first key, where it
# got blocked) -- never retried for the second key in this run.
assert companies_house_calls.call_count == 1
@pytest.mark.asyncio
@respx.mock
async def test_a_plain_5xx_is_not_treated_as_a_block(self, monkeypatch):
# A genuine server error is a different signal from a compliance
# block -- it should NOT be recorded via record_blocked (that would
# incorrectly suppress a source that's just having a bad moment, not
# one that told the engine to stop).
monkeypatch.setenv("COMPANIES_HOUSE_API_KEY", "test-key")
respx.get(COMPANIES_HOUSE_URL).mock(return_value=httpx.Response(500, json={}))
respx.get(GLEIF_URL).mock(
return_value=httpx.Response(200, json={"data": {"attributes": {"entity": {"legalName": {"name": "X"}, "legalAddress": {}}}}})
)
result = await run_job("order_block_3", {"markets": ["GB"], "confidence": 50, "seeds": {"GB": ["00000006"]}})
assert result["state"] == "done"
assert default_router.is_blocked("GB", "companies_house") is False
+164
View File
@@ -0,0 +1,164 @@
"""
Tests for workers/quality_bridge.py (the Python half of the P010 TS/Python
bridge) and its wiring into workers/pipeline.run_job. Verifies the
fail-open contract end to end: disabled, not configured, timeout,
non-2xx, malformed body, and success — and, critically, that `delivered`
rows from run_job are IDENTICAL regardless of which of those the bridge
call resolves to (the bridge is advisory, never a second gate).
"""
from __future__ import annotations
import httpx
import pytest
import respx
from workers.pipeline import run_job
from workers.quality_bridge import run_quality_bridge
BRIDGE_URL = "http://edge.local/api/0pt/quality/bridge-run"
BRREG_URL = "https://data.brreg.no/enhetsregisteret/api/enheter/923609016"
VALID_BRREG_BODY = {
"navn": "Volvo Personvagnar Norge AS",
"konkurs": False,
"forretningsadresse": {"adresse": ["Storgata 1"]},
}
@pytest.fixture(autouse=True)
def _clean_env(monkeypatch):
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
monkeypatch.delenv("OPT_QUALITY_BRIDGE_URL", raising=False)
monkeypatch.delenv("EDGE_URL", raising=False)
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
class TestBridgeDirectly:
@pytest.mark.asyncio
async def test_disabled_by_default_makes_zero_network_calls(self, monkeypatch):
# No respx mock registered at all — if this made an HTTP call it would raise.
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "disabled"
assert result.score is None
@pytest.mark.asyncio
async def test_enabled_but_no_url_configured(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "not_configured"
@pytest.mark.asyncio
@respx.mock
async def test_successful_run_is_parsed(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
"run": {"run_id": "dqrun_1", "status": "succeeded", "score": 100},
"issues": [],
}))
result = await run_quality_bridge(
tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[{"name": "X", "confidence": 0.9}]
)
assert result.status == "ok"
assert result.score == 100
assert result.run_id == "dqrun_1"
assert result.issue_count == 0
@pytest.mark.asyncio
@respx.mock
async def test_timeout_fails_open(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(side_effect=httpx.TimeoutException("timed out"))
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "unavailable"
assert result.detail == "timeout"
@pytest.mark.asyncio
@respx.mock
async def test_non_2xx_fails_open(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(500, json={"error": "boom"}))
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "unavailable"
assert "500" in result.detail
@pytest.mark.asyncio
@respx.mock
async def test_malformed_body_fails_open_not_raises(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, content=b"not json"))
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "unavailable"
@pytest.mark.asyncio
@respx.mock
async def test_connection_refused_fails_open(self, monkeypatch):
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("refused"))
result = await run_quality_bridge(tenant_id="t_a", project_id="p_1", dataset_id="ds_1", rows=[])
assert result.status == "unavailable"
class TestBridgeWiredIntoThePipeline:
"""The part that matters most: whatever the bridge does, `delivered`
(the rows the customer actually receives) must not change."""
@pytest.mark.asyncio
@respx.mock
async def test_missing_tenant_id_skips_the_bridge_honestly(self):
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
result = await run_job("order_notenant", {"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]}})
assert result["metrics"]["quality_bridge"]["status"] == "not_configured"
assert len(result["records"]) == 1 # delivery unaffected
@pytest.mark.asyncio
@respx.mock
async def test_bridge_disabled_delivers_identically_to_bridge_enabled_and_failing(self, monkeypatch):
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
config = {
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
"tenant_id": "t_bridge_pipeline", "project_id": "p_1",
}
monkeypatch.delenv("OPT_QUALITY_BRIDGE_ENABLED", raising=False)
disabled_result = await run_job("order_bridge_off", config)
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(side_effect=httpx.ConnectError("edge unreachable"))
failing_result = await run_job("order_bridge_fail", config)
assert disabled_result["metrics"]["rows"] == failing_result["metrics"]["rows"] == 1
assert [r["name"] for r in disabled_result["records"]] == [r["name"] for r in failing_result["records"]]
assert disabled_result["metrics"]["quality_bridge"]["status"] == "disabled"
assert failing_result["metrics"]["quality_bridge"]["status"] == "unavailable"
@pytest.mark.asyncio
@respx.mock
async def test_bridge_success_stamps_quality_provenance_without_dropping_or_adding_rows(self, monkeypatch):
respx.get(BRREG_URL).mock(return_value=httpx.Response(200, json=VALID_BRREG_BODY))
monkeypatch.setenv("OPT_QUALITY_BRIDGE_ENABLED", "true")
monkeypatch.setenv("OPT_QUALITY_BRIDGE_URL", "http://edge.local")
respx.post(BRIDGE_URL).mock(return_value=httpx.Response(201, json={
"run": {"run_id": "dqrun_9", "status": "succeeded", "score": 100},
"issues": [],
}))
result = await run_job(
"order_bridge_ok",
{
"markets": ["NO"], "confidence": 50, "seeds": {"NO": ["923609016"]},
"tenant_id": "t_bridge_ok", "project_id": "p_1",
},
)
assert result["metrics"]["rows"] == 1
assert result["metrics"]["quality_bridge"]["status"] == "ok"
assert result["metrics"]["quality_bridge"]["score"] == 100
for p in result["records"][0]["provenance"]:
assert p["quality"] == {
"score": 100, "run_id": "dqrun_9", "run_status": "succeeded", "checked_by": "0pt-quality-bridge",
}
assert p["artifact_ref"] == "0pt:pipeline_delivery:order_bridge_ok"
@@ -0,0 +1,166 @@
"""Regression tests for the Python adapter to canonical TypeScript ER.
The removed Python scorer is deliberately not tested because it no longer
exists. These tests pin the load-bearing adapter semantics instead:
canonical match edges merge; review/non-match edges do not; hard negative
identity evidence constrains the whole cluster; bridge failure leaves records
unmerged.
"""
from __future__ import annotations
import copy
import httpx
import pytest
import respx
from workers.resolve import _apply_matches, resolve
def _prov(field_name: str, source_id: str, confidence: float) -> dict:
return {
"field_name": field_name,
"source_id": source_id,
"confidence": confidence,
"entity_resolution": None,
}
def _rec(name: str, source: str, confidence: float = 0.9) -> dict:
return {
"market": "SE",
"name": name,
"status": "active",
"lei": None,
"org_nr": None,
"source": source,
"confidence": confidence,
"provenance": [_prov("name", source, confidence)],
}
def _pair(a: str, b: str, status: str = "match", decided_by: str = "score", score: float = 0.97) -> dict:
return {
"candidate_id": f"c_{a}_{b}",
"record_a": a,
"record_b": b,
"score": score,
"status": status,
"decided_by": decided_by,
}
def test_apply_matches_is_deterministic_and_uses_canonical_edges_only():
records = [_rec("Acme AB", "source-a", 0.91), _rec("ACME Aktiebolag", "source-b", 0.96)]
assert len(_apply_matches(copy.deepcopy(records), [])) == 2
@pytest.mark.asyncio
@respx.mock
async def test_canonical_match_merges_and_stamps_provenance(monkeypatch):
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
records = [_rec("Acme AB", "source-a", 0.91), _rec("Acme AB", "source-b", 0.96)]
route = respx.post("https://edge.example.test/api/0pt/internal/er/candidates")
def responder(request: httpx.Request) -> httpx.Response:
body = __import__("json").loads(request.content)
assert request.headers["X-Internal-Token"] == "test-internal-token"
assert body["tenant_id"] == "tenant-a"
assert body["blocking_fields"] == ["market", "block_initial"]
a, b = body["records"][0]["record_id"], body["records"][1]["record_id"]
# Endast hard-rule-beslut (decided_by != score/None) far skapa en
# canonical identity-lank - consumenten ignorerar medvetet score/"match".
return httpx.Response(
200,
json={"pairs": [_pair(a, b, "match", "0pt.company.stable-identifiers.v3", 0.99)], "review_queue": []},
)
route.side_effect = responder
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
assert len(result) == 1
assert result[0]["agreed_sources"] == 2
assert result[0]["confidence"] == pytest.approx(0.99)
assert set(result[0]["merged_from"]) <= {"source-a", "source-b"}
for provenance in result[0]["provenance"]:
er = provenance["entity_resolution"]
assert er["method"] == "canonical_ts_entity_resolution"
assert set(er["merged_sources"]) == {"source-a", "source-b"}
@pytest.mark.asyncio
@respx.mock
async def test_pending_review_never_auto_merges(monkeypatch):
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
records = [_rec("Acme AB", "source-a"), _rec("Acme Group AB", "source-b")]
def responder(request: httpx.Request) -> httpx.Response:
body = __import__("json").loads(request.content)
a, b = body["records"][0]["record_id"], body["records"][1]["record_id"]
pair = _pair(a, b, "pending_review")
return httpx.Response(200, json={"pairs": [pair], "review_queue": [pair]})
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").side_effect = responder
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
assert len(result) == 2
assert all(not r.get("merged_from") for r in result)
@pytest.mark.asyncio
@respx.mock
async def test_hard_non_match_blocks_transitive_cluster_merge(monkeypatch):
"""A-B and B-C matches may never imply A=C when canonical hard evidence says A!=C."""
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
records = [
_rec("Acme AB", "source-a", 0.95),
_rec("Acme AB", "source-b", 0.94),
_rec("Acme AB", "source-c", 0.93),
]
def responder(request: httpx.Request) -> httpx.Response:
body = __import__("json").loads(request.content)
a, b, c = [item["record_id"] for item in body["records"]]
return httpx.Response(
200,
json={
"pairs": [
_pair(a, b, "match", "0pt.company.stable-identifiers.v3", 0.98),
_pair(b, c, "match", "0pt.company.stable-identifiers.v3", 0.97),
_pair(a, c, "non_match", "0pt.company.stable-identifiers.v2", 0.0),
],
"review_queue": [],
},
)
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").side_effect = responder
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
assert len(result) == 2
assert max(r.get("agreed_sources", 1) for r in result) == 2
assert not any(set(r.get("merged_from", [])) >= {"source-b", "source-c"} for r in result)
@pytest.mark.asyncio
@respx.mock
async def test_bridge_outage_fails_closed_to_unmerged_records(monkeypatch):
monkeypatch.setenv("OPT_ER_BRIDGE_URL", "https://edge.example.test")
monkeypatch.setenv("INTERNAL_TOKEN", "test-internal-token")
records = [_rec("Acme AB", "source-a"), _rec("Acme AB", "source-b")]
respx.post("https://edge.example.test/api/0pt/internal/er/candidates").mock(side_effect=httpx.ConnectError("offline"))
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
assert len(result) == 2
assert all(not r.get("merged_from") for r in result)
@pytest.mark.asyncio
async def test_missing_internal_configuration_makes_zero_network_calls(monkeypatch):
monkeypatch.delenv("OPT_ER_BRIDGE_URL", raising=False)
monkeypatch.delenv("EDGE_URL", raising=False)
monkeypatch.delenv("INTERNAL_TOKEN", raising=False)
records = [_rec("Acme AB", "source-a"), _rec("Acme AB", "source-b")]
result = await resolve(records, tenant_id="tenant-a", project_id="project-a")
assert len(result) == 2
@@ -0,0 +1,53 @@
"""A failed or malformed source must never become a verified empty result."""
import httpx
import pytest
import respx
from fastapi.testclient import TestClient
from orchestrator import main
from sources.connectors import opensanctions_screen, rdw_vehicle
@pytest.mark.asyncio
@pytest.mark.parametrize("status", [403, 429, 500])
async def test_source_http_failure_raises(status):
with respx.mock:
respx.get("https://api.opensanctions.org/search/default").respond(status)
respx.get("https://opendata.rdw.nl/resource/m9d7-ebf2.json").respond(status)
with pytest.raises(httpx.HTTPStatusError):
await opensanctions_screen("example")
with pytest.raises(httpx.HTTPStatusError):
await rdw_vehicle("AB123")
@pytest.mark.asyncio
async def test_malformed_and_valid_empty_are_distinct():
with respx.mock:
screen = respx.get("https://api.opensanctions.org/search/default").respond(200, json={})
vehicle = respx.get("https://opendata.rdw.nl/resource/m9d7-ebf2.json").respond(200, json={})
with pytest.raises(ValueError):
await opensanctions_screen("example")
with pytest.raises(ValueError):
await rdw_vehicle("AB123")
screen.respond(200, json={"results": []})
vehicle.respond(200, json=[])
assert (await opensanctions_screen("example"))["matches"] == []
assert await rdw_vehicle("AB123") is None
def test_routes_return_503_on_source_failure_and_mark_verified_empty(monkeypatch):
async def failed(*args):
raise RuntimeError("fixture failure")
async def empty_screen(*args):
return {"source": "opensanctions", "matches": []}
async def empty_vehicle(*args):
return None
monkeypatch.setattr(main, "_INTERNAL_TOKEN", "")
monkeypatch.setattr(main, "opensanctions_screen", failed)
monkeypatch.setattr(main, "rdw_vehicle", failed)
client = TestClient(main.app)
assert client.get('/screen?q=example').status_code == 503
assert client.get('/vehicles/AB123').status_code == 503
monkeypatch.setattr(main, "opensanctions_screen", empty_screen)
monkeypatch.setattr(main, "rdw_vehicle", empty_vehicle)
assert client.get('/screen?q=example').json() == {"source": "opensanctions", "source_status": "ok", "matches": []}
assert client.get('/vehicles/AB123').json() == {"vehicle": None, "source_status": "ok"}
+1
View File
@@ -0,0 +1 @@
# 0pt data engine — workers package (router + pipeline).
+97
View File
@@ -0,0 +1,97 @@
"""
Adaptive router — per-market compliant-source planner.
P014 CORRECTION: this file's previous docstring read "all methods, no
compliance restrictions. See b0ts for scraping." That phrasing directly
contradicts this product's own compliance stance (CLAUDE.md §1: official
API/bulk -> authorized public web -> licensed partner data -> otherwise
level 0, never a bypass) and, worse, pointed a future reader at `b0ts` -- a
SEPARATE repo that does non-compliant scraping and is out of scope/forbidden
to touch from here. Nothing in this file ever called b0ts or any bypass
path; the line was simply wrong and is corrected, not extended.
What this file actually does: `plan_for_market(market)` returns the ordered
list of registered source_ids `pipeline.py._fetch` tries, in priority order,
for a given market. Every entry is a name pipeline.py may or may not have an
implemented connector for (see sources/connectors.py's CONNECTORS dict) --
an unimplemented source_id is simply skipped by `_fetch`, it is never a
license to build a bypass in its place.
P014 ADDITION: `record_blocked` / `is_blocked` give this planner a memory of
which (market, source_id) pairs recently told the engine to stop (HTTP
401/403/429/451 -- see sources/connectors.py's SourceBlockedError). Before
this pass, a source that started rate-limiting or blocking the engine mid-run
was retried on every subsequent record with no memory at all -- not a
compliance bypass, but not "respecting the block as a stop signal" either
(the same principle 0pt-motor's AdaptiveRouter documents for its own,
separate method-tier selector). `plan_for_market` now filters a currently-
blocked source out of the plan for the rest of the block's cooldown window,
so the engine backs off instead of hammering it. This is process-local,
in-memory state (matches every other piece of in-memory state this engine
documents honestly elsewhere, e.g. orchestrator/main.py's _ORDERS/_JOBS) --
not a persistent, cross-process reputation system.
"""
from __future__ import annotations
import time
MARKET_SOURCES: dict[str, list[str]] = {
"NO": ["brreg", "brreg_under", "proff", "1881", "gulesider", "flowfirma", "selskapssok", "regnskapsbasen", "okonomi", "visverdi", "gleif"],
"GB": ["companies_house", "gleif"],
"FI": ["prh", "gleif"],
"DK": ["cvr", "krak", "proff_dk", "118", "degulesider", "companydata", "grundigt", "datafordeler", "virk", "gleif"],
"DE": ["northdata", "handelsregister", "unternehmensregister", "moneyhouse", "creditreform", "bundesanzeiger", "registercheck", "personensuche", "whitepages", "companyhouse", "gleif"],
"SE": ["gleif"],
}
# How long a (market, source_id) pair is excluded from plan_for_market after
# being reported blocked. A fixed cooldown, not exponential backoff -- this
# is a planner, not a retry/backoff controller; a source is free to be tried
# again on the next run once the window elapses.
DEFAULT_BLOCK_COOLDOWN_SECONDS = 300.0
class AdaptiveRouter:
"""Tracks which (market, source_id) pairs are currently in a block cooldown."""
def __init__(self, cooldown_seconds: float = DEFAULT_BLOCK_COOLDOWN_SECONDS):
self._cooldown = cooldown_seconds
self._blocked_until: dict[tuple[str, str], float] = {}
def record_blocked(self, market: str, source_id: str, *, now: float | None = None) -> None:
"""Mark (market, source_id) as blocked starting now."""
self._blocked_until[(market, source_id)] = (now if now is not None else time.monotonic()) + self._cooldown
def is_blocked(self, market: str, source_id: str, *, now: float | None = None) -> bool:
deadline = self._blocked_until.get((market, source_id))
if deadline is None:
return False
current = now if now is not None else time.monotonic()
if current >= deadline:
# Cooldown elapsed -- clear it so the map doesn't grow forever.
del self._blocked_until[(market, source_id)]
return False
return True
def clear(self) -> None:
"""Test/process-reset hook -- forgets every recorded block."""
self._blocked_until.clear()
# Process-wide singleton, mirroring the in-memory-state convention already
# used elsewhere in this engine (e.g. orchestrator/main.py's _ORDERS/_JOBS).
# Injectable in `plan_for_market` for tests rather than only reachable via
# global mutable state.
default_router = AdaptiveRouter()
def plan_for_market(market: str, *, router: AdaptiveRouter | None = None) -> list[str]:
"""
Return the ordered source plan for a market, minus any source_id
currently in a block cooldown for that market. `router` defaults to the
process-wide `default_router`; tests inject their own instance so they
never share state with each other or with production.
"""
r = router if router is not None else default_router
plan = MARKET_SOURCES.get(market, ["gleif"])
return [source_id for source_id in plan if not r.is_blocked(market, source_id)]
+217
View File
@@ -0,0 +1,217 @@
from __future__ import annotations
from dataclasses import dataclass, field
from enum import Enum
from typing import Literal
from urllib.parse import urlparse
import re
import unicodedata
from rapidfuzz.fuzz import ratio
EntityType = Literal["organization", "person"]
class ResolutionDecision(str, Enum):
EXACT_LINK = "exact_link"
REVIEW = "review"
DISTINCT = "distinct"
@dataclass(frozen=True)
class EntityCandidate:
entity_id: str
entity_type: EntityType
name: str
source_ref: str
registration_country: str | None = None
registration_number: str | None = None
email: str | None = None
website: str | None = None
organization_id: str | None = None
city: str | None = None
metadata: dict[str, object] = field(default_factory=dict)
@dataclass(frozen=True)
class ResolutionResult:
left_id: str
right_id: str
decision: ResolutionDecision
score: float
reasons: tuple[str, ...]
requires_human_review: bool
algorithm_version: str = "entity-resolution-v1"
def _norm(value: str | None) -> str:
if not value:
return ""
normalized = unicodedata.normalize("NFKD", value)
ascii_text = "".join(char for char in normalized if not unicodedata.combining(char))
return re.sub(r"[^a-z0-9]+", " ", ascii_text.lower()).strip()
def _norm_registration_number(value: str | None) -> str:
return re.sub(r"[^0-9a-z]+", "", (value or "").lower())
def _norm_email(value: str | None) -> str:
return (value or "").strip().lower()
def _domain(value: str | None) -> str:
if not value:
return ""
raw = value.strip().lower()
if "@" in raw and "://" not in raw:
raw = raw.rsplit("@", 1)[-1]
if "://" not in raw:
raw = f"https://{raw}"
host = (urlparse(raw).hostname or "").lower()
return host.removeprefix("www.")
def _exact_organization(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult | None:
left_number = _norm_registration_number(left.registration_number)
right_number = _norm_registration_number(right.registration_number)
left_country = _norm(left.registration_country)
right_country = _norm(right.registration_country)
if left_number and right_number and left_country and right_country:
if left_country == right_country and left_number == right_number:
return ResolutionResult(
left.entity_id,
right.entity_id,
ResolutionDecision.EXACT_LINK,
1.0,
("same registration country and registration number",),
False,
)
if left_country == right_country and left_number != right_number:
return ResolutionResult(
left.entity_id,
right.entity_id,
ResolutionDecision.DISTINCT,
0.0,
("different registration numbers in the same country",),
False,
)
return None
def _exact_person(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult | None:
left_email = _norm_email(left.email)
right_email = _norm_email(right.email)
if left_email and right_email:
if left_email == right_email:
return ResolutionResult(
left.entity_id,
right.entity_id,
ResolutionDecision.EXACT_LINK,
1.0,
("same normalized email address",),
False,
)
return None
def resolve_entity_pair(left: EntityCandidate, right: EntityCandidate) -> ResolutionResult:
if left.entity_type != right.entity_type:
return ResolutionResult(
left.entity_id,
right.entity_id,
ResolutionDecision.DISTINCT,
0.0,
("different entity types",),
False,
)
exact = _exact_organization(left, right) if left.entity_type == "organization" else _exact_person(left, right)
if exact is not None:
return exact
left_name = _norm(left.name)
right_name = _norm(right.name)
if not left_name or not right_name:
return ResolutionResult(
left.entity_id,
right.entity_id,
ResolutionDecision.REVIEW,
0.0,
("missing normalized name prevents deterministic comparison",),
True,
)
name_similarity = ratio(left_name, right_name) / 100.0
reasons = [f"name similarity {name_similarity:.3f}"]
score = name_similarity * 0.65
if left.entity_type == "organization":
left_domain = _domain(left.website or left.email)
right_domain = _domain(right.website or right.email)
if left_domain and right_domain:
if left_domain == right_domain:
score += 0.25
reasons.append("same normalized domain")
else:
score -= 0.1
reasons.append("different normalized domains")
if _norm(left.registration_country) and _norm(left.registration_country) == _norm(right.registration_country):
score += 0.05
reasons.append("same registration country")
if _norm(left.city) and _norm(left.city) == _norm(right.city):
score += 0.05
reasons.append("same city")
else:
if left.organization_id and right.organization_id:
if left.organization_id == right.organization_id:
score += 0.25
reasons.append("same known organization")
else:
score -= 0.1
reasons.append("different known organizations")
left_domain = _domain(left.email)
right_domain = _domain(right.email)
if left_domain and right_domain and left_domain == right_domain:
score += 0.1
reasons.append("same email domain")
score = max(0.0, min(1.0, score))
# Only strong identifiers auto-link. Similarity-based matches remain operator-review candidates.
if score >= 0.72:
decision = ResolutionDecision.REVIEW
review = True
elif score <= 0.3:
decision = ResolutionDecision.DISTINCT
review = False
else:
decision = ResolutionDecision.REVIEW
review = True
return ResolutionResult(
left.entity_id,
right.entity_id,
decision,
round(score, 6),
tuple(reasons),
review,
)
def rank_resolution_candidates(
query: EntityCandidate,
candidates: list[EntityCandidate],
*,
limit: int = 20,
) -> list[ResolutionResult]:
results = [resolve_entity_pair(query, candidate) for candidate in candidates if candidate.entity_id != query.entity_id]
decision_order = {
ResolutionDecision.EXACT_LINK: 0,
ResolutionDecision.REVIEW: 1,
ResolutionDecision.DISTINCT: 2,
}
return sorted(
results,
key=lambda result: (decision_order[result.decision], -result.score, result.right_id),
)[: max(0, limit)]
+151
View File
@@ -0,0 +1,151 @@
"""Production pipeline: sourcing -> extraction -> canonical ER -> QA -> quality.
Identity decisions are owned by src/er/entity-resolution-service.ts. The
Python worker only applies canonical `match` decisions returned by its
internal bridge. If tenant context or the canonical bridge is unavailable,
records remain unmerged: false identity merges are never used as fallback.
"""
from __future__ import annotations
import logging
from typing import Any
from workers.resolve import resolve
from workers.quality_bridge import run_quality_bridge
from sources.connectors import (
CompanyResult,
SourceBlockedError,
brreg_by_orgnr,
companies_house_by_number,
cvr_by_number,
gleif_by_lei,
prh_by_businessid,
)
from workers.adaptive_router import default_router, plan_for_market
logger = logging.getLogger(__name__)
PIPELINE_VERSION = "0pt-pipeline@2"
DEMO_SEEDS: dict[str, list[str]] = {
"NO": ["923609016"],
"GB": ["00000006"],
}
async def _fetch(source_id: str, key: str) -> CompanyResult | None:
if source_id == "gleif": return await gleif_by_lei(key)
if source_id == "brreg": return await brreg_by_orgnr(key)
if source_id == "companies_house": return await companies_house_by_number(key)
if source_id == "prh": return await prh_by_businessid(key)
if source_id == "cvr": return await cvr_by_number(key)
return None
async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]:
markets: list[str] = config.get("markets", []) or ["NO"]
threshold = float(config.get("confidence", 92)) / 100.0
seeds: dict[str, list[str]] = config["seeds"] if config.get("seeds") is not None else DEMO_SEEDS
tenant_id = config.get("tenant_id")
project_id = config.get("project_id", "p_default")
correlation_id = config.get("correlation_id")
records: list[dict[str, Any]] = []
per_market: dict[str, int] = {}
for market in markets:
keys = seeds.get(market, [])
got = 0
for key in keys:
plan = plan_for_market(market)
for source_id in plan:
try:
res = await _fetch(source_id, key)
except SourceBlockedError:
default_router.record_blocked(market, source_id)
res = None
except Exception:
res = None
if res and res.name:
records.append({
"market": market,
"name": res.name,
"status": res.status,
"lei": res.lei,
"org_nr": res.org_nr,
"source": res.source,
"provenance": [p.__dict__ for p in res.provenance],
"confidence": max((p.confidence for p in res.provenance), default=0.0),
})
got += 1
break
per_market[market] = got
# Canonical ER is tenant-bound. Missing tenant context must never be
# papered over with a synthetic tenant or the removed Python scorer.
if tenant_id:
deduped = await resolve(
records,
tenant_id=tenant_id,
project_id=project_id,
correlation_id=correlation_id,
)
er_mode = "canonical_ts"
else:
deduped = [dict(r) for r in records]
er_mode = "fail_closed_unmerged_missing_tenant"
delivered = [r for r in deduped if r["confidence"] >= threshold]
dropped = len(deduped) - len(delivered)
avg_conf = round(sum(r["confidence"] for r in delivered) / len(delivered), 3) if delivered else 0.0
artifact_ref = f"0pt:pipeline_delivery:{order_id}"
for r in delivered:
for p in r.get("provenance", []):
p["artifact_ref"] = artifact_ref
quality_bridge_metrics: dict[str, Any]
if not tenant_id:
quality_bridge_metrics = {
"status": "not_configured",
"detail": "config.tenant_id missing — quality bridge not called",
}
else:
bridge_result = await run_quality_bridge(
tenant_id=tenant_id,
project_id=project_id,
dataset_id=f"0pt-order-{order_id}",
rows=delivered,
correlation_id=correlation_id,
)
quality_bridge_metrics = bridge_result.as_metrics()
if bridge_result.status == "ok":
quality_info = {
"score": bridge_result.score,
"run_id": bridge_result.run_id,
"run_status": bridge_result.run_status,
"checked_by": "0pt-quality-bridge",
}
for r in delivered:
for p in r.get("provenance", []):
p["quality"] = quality_info
elif bridge_result.status != "disabled":
logger.warning(
"0pt.quality_bridge did not complete for order %s: %s (%s)",
order_id, bridge_result.status, bridge_result.detail,
)
return {
"stage": "qa",
"pct": 100,
"state": "done",
"metrics": {
"rows": len(delivered),
"dropped_below_threshold": dropped,
"per_market": per_market,
"avg_confidence": avg_conf,
"pipeline_version": PIPELINE_VERSION,
"entity_resolution": er_mode,
"quality_bridge": quality_bridge_metrics,
},
"records": delivered,
}
+154
View File
@@ -0,0 +1,154 @@
"""
services/api/workers/quality_bridge.py
=========================================
P010: the TS/Python split-brain bridge (BEFORE_STATE.md §2). The production
order pipeline (workers/pipeline.py) has always run its own one-line QA
filter (`[r for r in deduped if r["confidence"] >= threshold]`) and never
touched this repo's sophisticated, versioned TS QualityService (393 lines:
rule registry, GE delegation, OpenLineage) — "noll bryggor mellan sig", per
BEFORE_STATE.md.
This module is the ADDITIVE bridge, called optionally from pipeline.py. It
is NOT a replacement for the Python filter and cannot become a silent one:
* Feature-flagged OFF by default (OPT_QUALITY_BRIDGE_ENABLED unset/"false").
A disabled bridge makes zero network calls and returns immediately.
* FAILS OPEN. Any failure mode (bridge URL unset, connection refused,
timeout, non-2xx, malformed JSON) is caught here and reported back as
`{"status": "unavailable", ...}` — it NEVER raises into pipeline.run_job,
and it never changes which rows get delivered. `run_job` remains the
single source of truth for delivery; this bridge is read-only/advisory.
* ADVISORY ONLY, not a second gate — and this is a real limitation, not a
choice made lightly: ExpectationEngine's execute() returns per-rule
PASS/FAIL COUNTS and up to 20 unexpected-value SAMPLES
(src/quality/expectation-engine.ts), not a per-row boolean array. There
is no way, from a dataset-level score + a handful of samples, to know
*which* of the N rows in this batch failed which rule — so per-row
gating against the TS engine's result cannot be implemented honestly
without first extending that engine's output contract (out of scope for
this pass; flagged, not worked around with a guess). A future
`OPT_QUALITY_BRIDGE_GATE_ENABLED` flag is deliberately NOT implemented
here for that reason — see the docstring on `BridgeResult` below.
Transport: a direct HTTP call to the Next.js edge's
POST /api/0pt/quality/bridge-run (app/api/0pt/quality/bridge-run/route.ts),
reusing the same `X-Internal-Token` shared-secret convention lib/engine.ts
already uses for the (opposite-direction) edge -> engine call, for
consistency. This was chosen over an N0D job dispatch (the other option the
brief raised) because: the TS QualityService instance the pipeline needs to
reach lives in-process inside the Next.js app (src/services/quality.ts, an
in-memory singleton, same pattern as the Source Registry before its Postgres
persistence landed) — there is no durable store an N0D job could read/write
through today, and dispatching a job whose only effect is "call this
in-process singleton" would add a queue hop for no isolation benefit. A
synchronous HTTP call, wrapped in a short timeout and a hard fail-open, is
the simpler and more honest choice for a first cut; nothing here blocks a
later move to N0D once the quality state has real (Postgres) persistence.
"""
from __future__ import annotations
import os
from dataclasses import dataclass
from typing import Any
import httpx
_TIMEOUT = httpx.Timeout(4.0, connect=2.0) # short — this must never meaningfully slow delivery
def _enabled() -> bool:
return os.getenv("OPT_QUALITY_BRIDGE_ENABLED", "false").strip().lower() in ("1", "true", "yes", "on")
def _bridge_url() -> str | None:
base = os.getenv("OPT_QUALITY_BRIDGE_URL") or os.getenv("EDGE_URL")
if not base:
return None
return f"{base.rstrip('/')}/api/0pt/quality/bridge-run"
@dataclass
class BridgeResult:
"""
Outcome of one bridge-run attempt. `status`:
- "disabled": OPT_QUALITY_BRIDGE_ENABLED is not set — no call made.
- "not_configured": enabled, but no OPT_QUALITY_BRIDGE_URL/EDGE_URL set.
- "unavailable": enabled + configured, but the call failed (timeout,
connection error, non-2xx, malformed body). Never raises.
- "ok": a real run completed; `score`/`run_id`/`issue_count` are populated.
There is deliberately no `gated`/`dropped_rows` field — see the module
docstring's "ADVISORY ONLY" section for why per-row gating isn't
implementable against the current TS engine contract.
"""
status: str
score: float | None = None
run_id: str | None = None
run_status: str | None = None
issue_count: int | None = None
detail: str | None = None
def as_metrics(self) -> dict[str, Any]:
return {
"status": self.status,
"score": self.score,
"run_id": self.run_id,
"run_status": self.run_status,
"issue_count": self.issue_count,
**({"detail": self.detail} if self.detail else {}),
}
async def run_quality_bridge(
*,
tenant_id: str,
project_id: str,
dataset_id: str,
rows: list[dict[str, Any]],
correlation_id: str | None = None,
) -> BridgeResult:
"""Ask the TS QualityService to grade `rows`. Never raises — see module docstring."""
if not _enabled():
return BridgeResult(status="disabled")
url = _bridge_url()
if not url:
return BridgeResult(status="not_configured", detail="OPT_QUALITY_BRIDGE_URL/EDGE_URL not set")
payload: dict[str, Any] = {
"tenant_id": tenant_id,
"project_id": project_id,
"dataset_id": dataset_id,
"rows": rows,
}
if correlation_id:
payload["correlation_id"] = correlation_id
headers = {
"Content-Type": "application/json",
"X-Internal-Token": os.getenv("INTERNAL_TOKEN", ""),
}
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
r = await client.post(url, json=payload, headers=headers)
if r.status_code not in (200, 201):
return BridgeResult(status="unavailable", detail=f"HTTP {r.status_code}")
body = r.json()
run = body.get("run") or {}
issues = body.get("issues") or []
return BridgeResult(
status="ok",
score=run.get("score"),
run_id=run.get("run_id"),
run_status=run.get("status"),
issue_count=len(issues),
)
except httpx.TimeoutException:
return BridgeResult(status="unavailable", detail="timeout")
except httpx.HTTPError as exc:
return BridgeResult(status="unavailable", detail=f"{type(exc).__name__}: {exc}")
except (ValueError, KeyError, TypeError) as exc:
# Malformed/unexpected response body — treat exactly like network
# unavailability, never propagate into the pipeline.
return BridgeResult(status="unavailable", detail=f"malformed response: {type(exc).__name__}")
+31
View File
@@ -0,0 +1,31 @@
"""
Lightweight in-process job queue (docs/02). Adequate for single-worker dev; swap
the backend for Redis/Celery at scale without changing callers.
"""
from __future__ import annotations
import asyncio
from typing import Any, Awaitable, Callable
Job = Callable[..., Awaitable[Any]]
class JobQueue:
def __init__(self) -> None:
self._q: "asyncio.Queue[tuple[Job, tuple[Any, ...]]]" = asyncio.Queue()
async def enqueue(self, fn: Job, *args: Any) -> None:
await self._q.put((fn, args))
async def worker(self) -> None:
while True:
fn, args = await self._q.get()
try:
await fn(*args)
except Exception as e: # keep the worker alive on a failed job
print("job failed:", repr(e))
finally:
self._q.task_done()
queue = JobQueue()
+235
View File
@@ -0,0 +1,235 @@
"""Thin adapter from the Python pipeline to 0pt's canonical TS ER engine.
Python no longer scores identity. It sends opaque records to
/api/0pt/internal/er/candidates, then applies only explicit hard-rule `match`
decisions. Score/fuzzy decisions are review evidence only and can never merge
records in this adapter, even if an older or compromised bridge returns
`status=match, decided_by=score`.
If the canonical engine is unavailable, records are returned UNMERGED (fail
closed against false identity merges).
A canonical hard non-match is also a cluster-wide constraint: transitive
matches are never allowed to pull two records with conflicting stable identity
evidence into the same entity.
"""
from __future__ import annotations
import copy
import hashlib
import json
import os
from typing import Any
import httpx
_TIMEOUT = httpx.Timeout(6.0, connect=2.0)
def _url() -> str | None:
base = os.getenv("OPT_ER_BRIDGE_URL") or os.getenv("EDGE_URL")
return f"{base.rstrip('/')}/api/0pt/internal/er/candidates" if base else None
def _record_id(record: dict[str, Any]) -> str:
stable = {
"market": record.get("market"),
"name": record.get("name"),
"status": record.get("status"),
"lei": record.get("lei"),
"org_nr": record.get("org_nr"),
"source": record.get("source"),
"provenance": record.get("provenance", []),
}
raw = json.dumps(stable, sort_keys=True, separators=(",", ":"), default=str).encode()
return "erp_" + hashlib.sha256(raw).hexdigest()[:32]
def _unique_record_ids(records: list[dict[str, Any]]) -> list[str]:
seen: dict[str, int] = {}
unique_ids: list[str] = []
for record in records:
rid = _record_id(record)
n = seen.get(rid, 0)
seen[rid] = n + 1
unique_ids.append(rid if n == 0 else f"{rid}_{n}")
return unique_ids
def _apply_matches(records: list[dict[str, Any]], pairs: list[dict[str, Any]]) -> list[dict[str, Any]]:
if not records:
return []
unique_ids = _unique_record_ids(records)
known_ids = set(unique_ids)
parent = {rid: rid for rid in unique_ids}
members = {rid: {rid} for rid in unique_ids}
hard_non_matches: set[frozenset[str]] = set()
for pair in pairs:
a, b = pair.get("record_a"), pair.get("record_b")
if a not in known_ids or b not in known_ids:
continue
if pair.get("status") == "non_match" and pair.get("decided_by") not in (None, "score"):
hard_non_matches.add(frozenset((a, b)))
def find(x: str) -> str:
while parent[x] != x:
parent[x] = parent[parent[x]]
x = parent[x]
return x
def components_conflict(left: set[str], right: set[str]) -> bool:
return any(frozenset((a, b)) in hard_non_matches for a in left for b in right)
def union(a: str, b: str) -> bool:
ra, rb = find(a), find(b)
if ra == rb:
return True
left, right = members[ra], members[rb]
if components_conflict(left, right):
return False
keep, drop = (ra, rb) if ra < rb else (rb, ra)
parent[drop] = keep
members[keep] = members[ra] | members[rb]
members.pop(drop, None)
return True
# Defence in depth: only stable-identifier hard rules may create canonical
# identity links. Score/fuzzy "match" responses are ignored at the consumer
# boundary even if an upstream/legacy route emits one.
match_pairs = [
p for p in pairs
if p.get("status") == "match"
and p.get("decided_by") not in (None, "score")
and p.get("record_a") in known_ids
and p.get("record_b") in known_ids
]
match_pairs.sort(
key=lambda p: (
-float(p.get("score") or 0.0),
str(p.get("candidate_id") or ""),
)
)
accepted_edges: list[dict[str, Any]] = []
for pair in match_pairs:
a, b = str(pair["record_a"]), str(pair["record_b"])
if union(a, b):
accepted_edges.append(pair)
groups: dict[str, list[tuple[str, dict[str, Any]]]] = {}
for rid, record in zip(unique_ids, records):
groups.setdefault(find(rid), []).append((rid, copy.deepcopy(record)))
out: list[dict[str, Any]] = []
for group in groups.values():
group.sort(key=lambda item: (-float(item[1].get("confidence", 0.0)), item[0]))
_, anchor = group[0]
sources = sorted({str(r.get("source")) for _, r in group if r.get("source")})
anchor["agreed_sources"] = len(sources) or 1
anchor["confidence"] = min(
0.99,
float(anchor.get("confidence", 0.0)) + 0.1 * max(0, len(sources) - 1),
)
anchor["merged_from"] = [r.get("source") for _, r in group[1:] if r.get("source")]
provenance = list(anchor.get("provenance", []))
keys = {(p.get("field_name"), p.get("source_id")) for p in provenance}
for _, record in group[1:]:
for p in record.get("provenance", []):
key = (p.get("field_name"), p.get("source_id"))
if key not in keys:
provenance.append(copy.deepcopy(p))
keys.add(key)
member_ids = {rid for rid, _ in group}
edges = [
p for p in accepted_edges
if p.get("record_a") in member_ids and p.get("record_b") in member_ids
]
er_info = {
"method": "canonical_ts_entity_resolution",
"merged_sources": sources,
"matched_edges": [
{
"candidate_id": p.get("candidate_id"),
"score": p.get("score"),
"decided_by": p.get("decided_by"),
}
for p in edges
],
}
if len(group) > 1:
for p in provenance:
p["entity_resolution"] = er_info
anchor["provenance"] = provenance
out.append(anchor)
return sorted(out, key=lambda r: (_record_id(r), str(r.get("source") or "")))
async def resolve(
records: list[dict[str, Any]],
*,
tenant_id: str,
project_id: str,
correlation_id: str | None = None,
) -> list[dict[str, Any]]:
"""Return canonically resolved records; never falls back to local scoring."""
if len(records) < 2:
return [copy.deepcopy(r) for r in records]
url = _url()
token = os.getenv("INTERNAL_TOKEN", "")
if not url or not token or not tenant_id or not project_id:
return [copy.deepcopy(r) for r in records]
unique_ids = _unique_record_ids(records)
er_records = []
for rid, record in zip(unique_ids, records):
name = str(record.get("name") or "")
market = record.get("market")
er_records.append({
"record_id": rid,
"fields": {
"market": market,
"registration_country": market,
"block_initial": name[:1].upper(),
"name": name,
"status": record.get("status"),
"lei": record.get("lei"),
"org_nr": record.get("org_nr"),
},
})
payload: dict[str, Any] = {
"tenant_id": tenant_id,
"project_id": project_id,
"records": er_records,
"blocking_fields": ["market", "block_initial"],
"compare_fields": [{"field": "name", "weight": 1}],
"auto_match_threshold": 0.92,
"review_threshold": 0.72,
}
if correlation_id:
payload["correlation_id"] = correlation_id
try:
async with httpx.AsyncClient(timeout=_TIMEOUT) as client:
response = await client.post(
url,
json=payload,
headers={"Content-Type": "application/json", "X-Internal-Token": token},
)
if response.status_code not in (200, 201):
return [copy.deepcopy(r) for r in records]
body = response.json()
pairs = body.get("pairs")
if not isinstance(pairs, list):
return [copy.deepcopy(r) for r in records]
return _apply_matches(records, pairs)
except (httpx.HTTPError, ValueError, TypeError, KeyError):
return [copy.deepcopy(r) for r in records]
__all__ = ["resolve", "_apply_matches"]
+21
View File
@@ -0,0 +1,21 @@
"""
Worker entrypoint. In production this drains a queue of order jobs and runs the
compliant pipeline. For now it exposes a CLI to run a single job locally.
python -m workers.run <order_id>
"""
from __future__ import annotations
import asyncio
import sys
from workers.pipeline import run_job
async def _main() -> None:
order_id = sys.argv[1] if len(sys.argv) > 1 else "demo"
result = await run_job(order_id, {"markets": ["NO", "GB"], "confidence": 90})
print(result["metrics"])
if __name__ == "__main__":
asyncio.run(_main())