FastAPI orchestrator (health/sources/estimates/orders/jobs per docs/04), adaptive router (compliant methods only, no bypass tiers), pipeline (sourcing->extraction->structuring/dedup->fusion->QA with per-field provenance, drops < confidence target), live connectors GLEIF/Brreg/Companies House (PRH/CVR/Bolagsverket gated to level-0 until terms verified). sources.yaml registry. infra/docker-compose + Caddyfile for the VPS. FlareSolverr/stealth/proxy/fake-account paths intentionally NOT carried over (CLAUDE.md §1). py_compile clean. Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
162 lines
5.8 KiB
Python
162 lines
5.8 KiB
Python
"""
|
|
0pt data engine — COMPLIANT source connectors.
|
|
|
|
Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market:
|
|
1. official API / bulk 2. authorized public web (robots + rate limits)
|
|
3. licensed partner data 4. otherwise -> level 0 "ej tillgänglig / licens".
|
|
|
|
EXPLICITLY NOT PRESENT (and must never be added here):
|
|
WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint
|
|
rotation to evade bans, scraping behind login walls, fake-account creation.
|
|
|
|
Every connector returns a normalized company dict + per-field provenance rows
|
|
(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced.
|
|
Terms change — VERIFY each source's current access/licence before relying.
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
from dataclasses import dataclass, field
|
|
from datetime import date
|
|
from typing import Any
|
|
|
|
import httpx
|
|
|
|
# A polite client that identifies itself and obeys timeouts/rate limits.
|
|
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
|
|
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)
|
|
|
|
|
|
def _client() -> httpx.AsyncClient:
|
|
return httpx.AsyncClient(
|
|
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
|
|
timeout=_TIMEOUT,
|
|
follow_redirects=True,
|
|
)
|
|
|
|
|
|
@dataclass
|
|
class Provenance:
|
|
field_name: str
|
|
source_id: str
|
|
confidence: float
|
|
level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
|
|
freshness: str | None = None
|
|
|
|
|
|
@dataclass
|
|
class CompanyResult:
|
|
org_nr: str | None = None
|
|
market: str | None = None
|
|
lei: str | None = None
|
|
name: str | None = None
|
|
status: str | None = None
|
|
address: dict[str, Any] | None = None
|
|
website: str | None = None
|
|
provenance: list[Provenance] = field(default_factory=list)
|
|
source: str = ""
|
|
|
|
|
|
def _today() -> str:
|
|
return date.today().isoformat()
|
|
|
|
|
|
# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
|
|
async def gleif_by_lei(lei: str) -> CompanyResult | None:
|
|
url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
|
|
async with _client() as c:
|
|
r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
|
|
if r.status_code != 200:
|
|
return None
|
|
a = (r.json().get("data") or {}).get("attributes") or {}
|
|
entity = a.get("entity") or {}
|
|
legal = entity.get("legalAddress") or {}
|
|
res = CompanyResult(
|
|
lei=lei,
|
|
name=(entity.get("legalName") or {}).get("name"),
|
|
status=entity.get("status"),
|
|
address={"lines": legal.get("addressLines"), "city": legal.get("city"),
|
|
"country": legal.get("country"), "postal": legal.get("postalCode")},
|
|
source="gleif",
|
|
)
|
|
f = _today()
|
|
res.provenance += [
|
|
Provenance("lei", "gleif", 0.99, 3, f),
|
|
Provenance("name", "gleif", 0.97, 3, f),
|
|
]
|
|
return res
|
|
|
|
|
|
# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ─────────────
|
|
async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
|
|
url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
|
|
async with _client() as c:
|
|
r = await c.get(url)
|
|
if r.status_code != 200:
|
|
return None
|
|
d = r.json()
|
|
res = CompanyResult(
|
|
org_nr=orgnr, market="NO",
|
|
name=d.get("navn"),
|
|
status="dissolved" if d.get("konkurs") else "active",
|
|
address={"raw": d.get("forretningsadresse")},
|
|
source="brreg",
|
|
)
|
|
f = _today()
|
|
res.provenance += [
|
|
Provenance("name", "brreg", 0.97, 3, f),
|
|
Provenance("status", "brreg", 0.95, 3, f),
|
|
Provenance("address", "brreg", 0.95, 3, f),
|
|
]
|
|
return res
|
|
|
|
|
|
# ── Companies House (UK) — free API, key required. VERIFY terms. ────────────
|
|
async def companies_house_by_number(number: str) -> CompanyResult | None:
|
|
key = os.getenv("COMPANIES_HOUSE_API_KEY")
|
|
if not key:
|
|
# No key -> deliver honestly as unavailable, never fabricate.
|
|
return CompanyResult(org_nr=number, market="GB", source="companies_house",
|
|
provenance=[Provenance("name", "companies_house", 0.0, 0, _today())])
|
|
url = f"https://api.company-information.service.gov.uk/company/{number}"
|
|
async with _client() as c:
|
|
r = await c.get(url, auth=(key, ""))
|
|
if r.status_code != 200:
|
|
return None
|
|
d = r.json()
|
|
res = CompanyResult(
|
|
org_nr=number, market="GB",
|
|
name=d.get("company_name"),
|
|
status=d.get("company_status"),
|
|
address={"raw": d.get("registered_office_address")},
|
|
source="companies_house",
|
|
)
|
|
f = _today()
|
|
res.provenance += [
|
|
Provenance("name", "companies_house", 0.97, 3, f),
|
|
Provenance("status", "companies_house", 0.95, 3, f),
|
|
]
|
|
return res
|
|
|
|
|
|
# Registries that require verified terms / keys before bulk use.
|
|
# Implement as official-API connectors only; never bypass. Until wired they
|
|
# return level-0 so the field is delivered honestly as "ej tillgänglig".
|
|
async def prh_by_businessid(business_id: str) -> CompanyResult | None: # FI — VERIFY
|
|
return CompanyResult(org_nr=business_id, market="FI", source="prh",
|
|
provenance=[Provenance("name", "prh", 0.0, 0, _today())])
|
|
|
|
|
|
async def cvr_by_number(cvr: str) -> CompanyResult | None: # DK — VERIFY (Virk terms)
|
|
return CompanyResult(org_nr=cvr, market="DK", source="cvr",
|
|
provenance=[Provenance("name", "cvr", 0.0, 0, _today())])
|
|
|
|
|
|
CONNECTORS = {
|
|
"gleif": gleif_by_lei,
|
|
"brreg": brreg_by_orgnr,
|
|
"companies_house": companies_house_by_number,
|
|
"prh": prh_by_businessid,
|
|
"cvr": cvr_by_number,
|
|
}
|