0pt/services/api/sources/connectors.py

"""
0pt data engine — COMPLIANT source connectors.

Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market:
  1. official API / bulk   2. authorized public web (robots + rate limits)
  3. licensed partner data  4. otherwise -> level 0 "ej tillgänglig / licens".

EXPLICITLY NOT PRESENT (and must never be added here):
  WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint
  rotation to evade bans, scraping behind login walls, fake-account creation.

Every connector returns a normalized company dict + per-field provenance rows
(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced.
Terms change — VERIFY each source's current access/licence before relying.
"""
from __future__ import annotations

import os
from dataclasses import dataclass, field
from datetime import date
from typing import Any

import httpx

# A polite client that identifies itself and obeys timeouts/rate limits.
USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)"
_TIMEOUT = httpx.Timeout(15.0, connect=10.0)


def _client() -> httpx.AsyncClient:
    return httpx.AsyncClient(
        headers={"User-Agent": USER_AGENT, "Accept": "application/json"},
        timeout=_TIMEOUT,
        follow_redirects=True,
    )


@dataclass
class Provenance:
    field_name: str
    source_id: str
    confidence: float
    level: int          # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated
    freshness: str | None = None


@dataclass
class CompanyResult:
    org_nr: str | None = None
    market: str | None = None
    lei: str | None = None
    name: str | None = None
    status: str | None = None
    address: dict[str, Any] | None = None
    website: str | None = None
    provenance: list[Provenance] = field(default_factory=list)
    source: str = ""


def _today() -> str:
    return date.today().isoformat()


# ── GLEIF / LEI (global cross-border key) — open API ────────────────────────
async def gleif_by_lei(lei: str) -> CompanyResult | None:
    url = f"https://api.gleif.org/api/v1/lei-records/{lei}"
    async with _client() as c:
        r = await c.get(url, headers={"Accept": "application/vnd.api+json"})
        if r.status_code != 200:
            return None
        a = (r.json().get("data") or {}).get("attributes") or {}
        entity = a.get("entity") or {}
        legal = entity.get("legalAddress") or {}
        res = CompanyResult(
            lei=lei,
            name=(entity.get("legalName") or {}).get("name"),
            status=entity.get("status"),
            address={"lines": legal.get("addressLines"), "city": legal.get("city"),
                     "country": legal.get("country"), "postal": legal.get("postalCode")},
            source="gleif",
        )
        f = _today()
        res.provenance += [
            Provenance("lei", "gleif", 0.99, 3, f),
            Provenance("name", "gleif", 0.97, 3, f),
        ]
        return res


# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ─────────────
async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None:
    url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}"
    async with _client() as c:
        r = await c.get(url)
        if r.status_code != 200:
            return None
        d = r.json()
        res = CompanyResult(
            org_nr=orgnr, market="NO",
            name=d.get("navn"),
            status="dissolved" if d.get("konkurs") else "active",
            address={"raw": d.get("forretningsadresse")},
            source="brreg",
        )
        f = _today()
        res.provenance += [
            Provenance("name", "brreg", 0.97, 3, f),
            Provenance("status", "brreg", 0.95, 3, f),
            Provenance("address", "brreg", 0.95, 3, f),
        ]
        return res


# ── Companies House (UK) — free API, key required. VERIFY terms. ────────────
async def companies_house_by_number(number: str) -> CompanyResult | None:
    key = os.getenv("COMPANIES_HOUSE_API_KEY")
    if not key:
        # No key -> deliver honestly as unavailable, never fabricate.
        return CompanyResult(org_nr=number, market="GB", source="companies_house",
                             provenance=[Provenance("name", "companies_house", 0.0, 0, _today())])
    url = f"https://api.company-information.service.gov.uk/company/{number}"
    async with _client() as c:
        r = await c.get(url, auth=(key, ""))
        if r.status_code != 200:
            return None
        d = r.json()
        res = CompanyResult(
            org_nr=number, market="GB",
            name=d.get("company_name"),
            status=d.get("company_status"),
            address={"raw": d.get("registered_office_address")},
            source="companies_house",
        )
        f = _today()
        res.provenance += [
            Provenance("name", "companies_house", 0.97, 3, f),
            Provenance("status", "companies_house", 0.95, 3, f),
        ]
        return res


# ── PRH avoindata (FI) — open API (CC-BY). VERIFY terms. ────────────────────
async def prh_by_businessid(business_id: str) -> CompanyResult | None:
    url = f"https://avoindata.prh.fi/opendata-ytj-api/v3/companies?businessId={business_id}"
    async with _client() as c:
        r = await c.get(url)
        if r.status_code != 200:
            return None
        items = r.json().get("companies") or r.json().get("results") or []
        if not items:
            return None
        d = items[0]
        names = d.get("names") or []
        name = names[0].get("name") if names else d.get("name")
        res = CompanyResult(org_nr=business_id, market="FI", name=name, status=d.get("status"), source="prh")
        if name:
            res.provenance.append(Provenance("name", "prh", 0.95, 3, _today()))
        return res


# ── CVR (DK) via cvrapi.dk — free for low volume (descriptive UA). VERIFY. ───
async def cvr_by_number(cvr: str) -> CompanyResult | None:
    url = f"https://cvrapi.dk/api?search={cvr}&country=dk"
    async with _client() as c:
        r = await c.get(url)
        if r.status_code != 200:
            return None
        d = r.json()
        if not d.get("name"):
            return None
        res = CompanyResult(
            org_nr=cvr, market="DK", name=d.get("name"),
            status="ceased" if d.get("enddate") else "active",
            address={"raw": d.get("address")}, source="cvr",
        )
        f = _today()
        res.provenance += [Provenance("name", "cvr", 0.93, 3, f), Provenance("address", "cvr", 0.9, 2, f)]
        return res


# ── RDW (NL) open vehicle data — Socrata. Technical/fleet only, no owner PII. ─
async def rdw_vehicle(kenteken: str) -> dict[str, Any] | None:
    plate = kenteken.upper().replace("-", "").replace(" ", "")
    url = f"https://opendata.rdw.nl/resource/m9d7-ebf2.json?kenteken={plate}"
    async with _client() as c:
        r = await c.get(url)
        if r.status_code != 200:
            return None
        arr = r.json()
        if not arr:
            return None
        d = arr[0]
        return {
            "reg_plate": kenteken, "market": "NL",
            "make": d.get("merk"), "model": d.get("handelsbenaming"),
            "year": (d.get("datum_eerste_toelating") or "")[:4] or None,
            "source": "rdw",
            "provenance": [{"field": "make", "source_id": "rdw", "confidence": 0.9, "level": 3, "freshness": _today()}],
        }


# ── OpenSanctions screening — search API. VERIFY commercial terms / key. ─────
async def opensanctions_screen(name: str) -> dict[str, Any]:
    url = "https://api.opensanctions.org/search/default"
    async with _client() as c:
        r = await c.get(url, params={"q": name, "limit": 5})
        if r.status_code != 200:
            return {"matches": [], "source": "opensanctions"}
        results = r.json().get("results", [])
        return {
            "source": "opensanctions",
            "matches": [{"name": x.get("caption"), "score": x.get("score"), "schema": x.get("schema")} for x in results],
        }


CONNECTORS = {
    "gleif": gleif_by_lei,
    "brreg": brreg_by_orgnr,
    "companies_house": companies_house_by_number,
    "prh": prh_by_businessid,
    "cvr": cvr_by_number,
}