Files
0pt/services/api/sources/connectors.py
T
adminandClaude Opus 4.8 31128a71c8 feat(engine): import compliant data engine (services/api) + infra
FastAPI orchestrator (health/sources/estimates/orders/jobs per docs/04), adaptive router (compliant methods only, no bypass tiers), pipeline (sourcing->extraction->structuring/dedup->fusion->QA with per-field provenance, drops < confidence target), live connectors GLEIF/Brreg/Companies House (PRH/CVR/Bolagsverket gated to level-0 until terms verified). sources.yaml registry. infra/docker-compose + Caddyfile for the VPS. FlareSolverr/stealth/proxy/fake-account paths intentionally NOT carried over (CLAUDE.md §1). py_compile clean.

Co-Authored-By: Claude Opus 4.8 <noreply@anthropic.com>
2026-06-24 20:02:50 +02:00

162 lines
5.8 KiB
Python

"""
0pt data engine — COMPLIANT source connectors.
Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market:
1. official API / bulk 2. authorized public web (robots + rate limits)
3. licensed partner data 4. otherwise -> level 0 "ej tillgänglig / licens".
EXPLICITLY NOT PRESENT (and must never be added here):
WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint
rotation to evade bans, scraping behind login walls, fake-account creation.
Every connector returns a normalized company dict + per-field provenance rows
(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced.
Terms change — VERIFY each source's current access/licence before relying.
"""
from __future__ import annotations
import os
from dataclasses import dataclass, field
from datetime import date
from typing import Any
import httpx
# A polite client that identifies itself and obeys timeouts/rate limits.
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)
def _client() -> httpx.AsyncClient:
return httpx.AsyncClient(
headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
timeout=_TIMEOUT,
follow_redirects=True,
)
@dataclass
class Provenance:
field_name: str
source_id: str
confidence: float
level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
freshness: str | None = None
@dataclass
class CompanyResult:
org_nr: str | None = None
market: str | None = None
lei: str | None = None
name: str | None = None
status: str | None = None
address: dict[str, Any] | None = None
website: str | None = None
provenance: list[Provenance] = field(default_factory=list)
source: str = ""
def _today() -> str:
return date.today().isoformat()
# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
async def gleif_by_lei(lei: str) -> CompanyResult | None:
url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
async with _client() as c:
r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
if r.status_code != 200:
return None
a = (r.json().get("data") or {}).get("attributes") or {}
entity = a.get("entity") or {}
legal = entity.get("legalAddress") or {}
res = CompanyResult(
lei=lei,
name=(entity.get("legalName") or {}).get("name"),
status=entity.get("status"),
address={"lines": legal.get("addressLines"), "city": legal.get("city"),
"country": legal.get("country"), "postal": legal.get("postalCode")},
source="gleif",
)
f = _today()
res.provenance += [
Provenance("lei", "gleif", 0.99, 3, f),
Provenance("name", "gleif", 0.97, 3, f),
]
return res
# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ─────────────
async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
async with _client() as c:
r = await c.get(url)
if r.status_code != 200:
return None
d = r.json()
res = CompanyResult(
org_nr=orgnr, market="NO",
name=d.get("navn"),
status="dissolved" if d.get("konkurs") else "active",
address={"raw": d.get("forretningsadresse")},
source="brreg",
)
f = _today()
res.provenance += [
Provenance("name", "brreg", 0.97, 3, f),
Provenance("status", "brreg", 0.95, 3, f),
Provenance("address", "brreg", 0.95, 3, f),
]
return res
# ── Companies House (UK) — free API, key required. VERIFY terms. ────────────
async def companies_house_by_number(number: str) -> CompanyResult | None:
key = os.getenv("COMPANIES_HOUSE_API_KEY")
if not key:
# No key -> deliver honestly as unavailable, never fabricate.
return CompanyResult(org_nr=number, market="GB", source="companies_house",
provenance=[Provenance("name", "companies_house", 0.0, 0, _today())])
url = f"https://api.company-information.service.gov.uk/company/{number}"
async with _client() as c:
r = await c.get(url, auth=(key, ""))
if r.status_code != 200:
return None
d = r.json()
res = CompanyResult(
org_nr=number, market="GB",
name=d.get("company_name"),
status=d.get("company_status"),
address={"raw": d.get("registered_office_address")},
source="companies_house",
)
f = _today()
res.provenance += [
Provenance("name", "companies_house", 0.97, 3, f),
Provenance("status", "companies_house", 0.95, 3, f),
]
return res
# Registries that require verified terms / keys before bulk use.
# Implement as official-API connectors only; never bypass. Until wired they
# return level-0 so the field is delivered honestly as "ej tillgänglig".
async def prh_by_businessid(business_id: str) -> CompanyResult | None: # FI — VERIFY
return CompanyResult(org_nr=business_id, market="FI", source="prh",
provenance=[Provenance("name", "prh", 0.0, 0, _today())])
async def cvr_by_number(cvr: str) -> CompanyResult | None: # DK — VERIFY (Virk terms)
return CompanyResult(org_nr=cvr, market="DK", source="cvr",
provenance=[Provenance("name", "cvr", 0.0, 0, _today())])
CONNECTORS = {
"gleif": gleif_by_lei,
"brreg": brreg_by_orgnr,
"companies_house": companies_house_by_number,
"prh": prh_by_businessid,
"cvr": cvr_by_number,
}