diff --git a/infra/Caddyfile b/infra/Caddyfile new file mode 100644 index 0000000..ad7fc66 --- /dev/null +++ b/infra/Caddyfile @@ -0,0 +1,13 @@ +# 0pt — reverse proxy + automatic TLS (docs/08). Frontend stays on Vercel (0pt.app); +# these subdomains front the VPS services. Point the DNS A-records at the VPS. +api.0pt.app { + reverse_proxy api:8000 +} + +data.0pt.app { + reverse_proxy nocodb:8080 +} + +dash.0pt.app { + reverse_proxy metabase:3000 +} diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml new file mode 100644 index 0000000..f6707be --- /dev/null +++ b/infra/docker-compose.yml @@ -0,0 +1,69 @@ +# 0pt — always-on services for a single small VPS (docs/08). Vercel hosts the +# Next.js frontend; this compose runs the data engine + DB + embeds + proxy. +services: + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: opt + POSTGRES_PASSWORD: ${PG_PASSWORD} + POSTGRES_DB: opt + volumes: + - pgdata:/var/lib/postgresql/data + - ../sql:/docker-entrypoint-initdb.d:ro + restart: unless-stopped + + redis: + image: redis:7 + restart: unless-stopped + + api: + build: ../services/api + command: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000 + environment: + DATABASE_URL: postgres://opt:${PG_PASSWORD}@postgres:5432/opt + COMPANIES_HOUSE_API_KEY: ${COMPANIES_HOUSE_API_KEY:-} + INTERNAL_TOKEN: ${INTERNAL_TOKEN} + depends_on: [postgres, redis] + restart: unless-stopped + + worker: + build: ../services/api + command: python -m workers.run + environment: + DATABASE_URL: postgres://opt:${PG_PASSWORD}@postgres:5432/opt + COMPANIES_HOUSE_API_KEY: ${COMPANIES_HOUSE_API_KEY:-} + depends_on: [postgres, redis] + restart: unless-stopped + + nocodb: + image: nocodb/nocodb:latest + environment: + NC_DB: "pg://postgres:5432?u=opt&p=${PG_PASSWORD}&d=opt" + NC_PUBLIC_URL: "https://data.0pt.app" + depends_on: [postgres] + restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + MB_DB_TYPE: postgres + MB_DB_HOST: postgres + MB_DB_PORT: "5432" + MB_DB_USER: opt + MB_DB_PASS: ${PG_PASSWORD} + MB_DB_DBNAME: opt + depends_on: [postgres] + restart: unless-stopped + + caddy: + image: caddy:2 + ports: ["80:80", "443:443"] + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - caddydata:/data + depends_on: [api, nocodb, metabase] + restart: unless-stopped + +volumes: + pgdata: {} + caddydata: {} diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 0000000..409c49d --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,15 @@ +# 0pt data engine — API + worker image (runs on a VPS, NOT Vercel; long jobs). +FROM python:3.12-slim + +WORKDIR /app +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . + +EXPOSE 8000 +# API: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000 +# Worker: python -m workers.run +CMD ["uvicorn", "orchestrator.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/api/README.md b/services/api/README.md new file mode 100644 index 0000000..6ca33b9 --- /dev/null +++ b/services/api/README.md @@ -0,0 +1,34 @@ +# 0pt data engine (`services/api`) + +The legal data motor (docs/06). FastAPI orchestrator + worker pipeline that turn +**compliant** sources into delivered, provenance-tagged data. Runs on a VPS via +[`../../infra/docker-compose.yml`](../../infra/docker-compose.yml) — **not on Vercel** (long jobs). + +## Sourcing boundary (CLAUDE.md §1 — non-negotiable) + +Per field × market, in strict order: **official API/bulk → authorized public web +(robots + rate limits) → licensed partner data → otherwise level 0 "ej tillgänglig"**. +We never invent values. + +**Deliberately excluded (never add here):** WAF/bot-detection bypass, FlareSolverr, +stealth/undetected browsers, proxy/fingerprint rotation to evade bans, scraping behind +login walls, automated fake-account creation. This is the honest-accuracy moat. + +## Layout +- `sources/connectors.py` — compliant connectors (GLEIF, Brønnøysund, Companies House live; PRH/CVR/Bolagsverket gated on verified terms → level 0 until wired). +- `sources/sources.yaml` — source registry (access/licence/cadence; VERIFY terms). +- `orchestrator/main.py` — REST API (health, sources, estimates, orders, jobs) per docs/04. +- `workers/adaptive_router.py` — picks the cheapest **compliant** method per market (api > bulk > authorized_web > licensed). No bypass tiers. +- `workers/pipeline.py` — sourcing → extraction → structuring (dedup) → enrichment/fusion → QA (drops rows below the order's confidence target). Writes provenance per field. + +## Run locally +```bash +pip install -r requirements.txt +uvicorn orchestrator.main:app --reload # API on :8000 +python -m workers.run # run one demo job +``` + +## Reconciliation note +Supersedes the earlier `nordic-eu-data-platform` backend: the compliant connectors + +pipeline are migrated here; the FlareSolverr/stealth/proxy/fake-account paths and the +`test_bypass_chain`/`test_proxy_pool` examples are intentionally **not** carried over. diff --git a/services/api/orchestrator/__init__.py b/services/api/orchestrator/__init__.py new file mode 100644 index 0000000..993587d --- /dev/null +++ b/services/api/orchestrator/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — orchestrator package (FastAPI app). diff --git a/services/api/orchestrator/__pycache__/__init__.cpython-314.pyc b/services/api/orchestrator/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..ecffe26 Binary files /dev/null and b/services/api/orchestrator/__pycache__/__init__.cpython-314.pyc differ diff --git a/services/api/orchestrator/__pycache__/main.cpython-314.pyc b/services/api/orchestrator/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000..307123e Binary files /dev/null and b/services/api/orchestrator/__pycache__/main.cpython-314.pyc differ diff --git a/services/api/orchestrator/main.py b/services/api/orchestrator/main.py new file mode 100644 index 0000000..e4f77f1 --- /dev/null +++ b/services/api/orchestrator/main.py @@ -0,0 +1,150 @@ +""" +0pt data engine — FastAPI orchestrator (docs/04). Compliant sourcing only. +Runs on a VPS/worker host, NOT on Vercel (long jobs). Amounts in öre. +Reconciles the Next.js /api routes with the Python data engine: the Next layer +is the customer-facing edge; this service runs the actual pipeline. +""" +from __future__ import annotations + +import os +import uuid +from pathlib import Path +from typing import Any + +import yaml +from fastapi import FastAPI, HTTPException +from fastapi.middleware.cors import CORSMiddleware +from pydantic import BaseModel + +from workers.pipeline import run_job + +app = FastAPI(title="0pt data engine", version="0.1") +app.add_middleware( + CORSMiddleware, allow_origins=["*"], allow_methods=["*"], allow_headers=["*"], +) + +_SOURCES = yaml.safe_load((Path(__file__).resolve().parent.parent / "sources" / "sources.yaml").read_text(encoding="utf-8"))["sources"] + +# ── estimate model (mirrors apps/web lib/data.computeEstimate) ─────────────── +FIELD_RATE = 240 +ENTITY_MULT = {"Bolag": 1.0, "Beslutsfattare": 1.4, "Fordon": 1.2} +FIELD_WEIGHT = {"identity": 1.0, "lei": 0.8, "financials": 1.6, "ubo": 1.4, "contact": 1.8, "intent": 1.5} +ENRICH_WEIGHT = {"dedup": 0.1, "crm": 0.2, "sanctions": 0.15, "nace": 0.1, "tech": 0.15} + + +class EstimateIn(BaseModel): + markets: list[str] = [] + entity_type: str = "Bolag" + fields: list[str] = [] + enrichment: list[str] = [] + volume: int = 1000 + frequency: str = "Engångs" + confidence_target: int = 92 + + +def compute_estimate(b: EstimateIn) -> dict[str, Any]: + markets = max(1, len(b.markets)) + fields = set(b.fields) | {"identity"} + fw = sum(FIELD_WEIGHT.get(f, 0.5) for f in fields) or 1 + em = ENTITY_MULT.get(b.entity_type, 1.0) + cm = 1 + max(0, b.confidence_target - 90) * 0.03 + ew = sum(ENRICH_WEIGHT.get(e, 0) for e in b.enrichment) + base = fw * markets * FIELD_RATE * (b.volume / 1000) * em * cm + price = round(base * (1 + ew) / 5) * 5 + deposit = round(price / 2) + days = min(14, max(1, round(1 + markets * 0.4 + b.volume / 6000 + len(fields) * 0.25))) + avg_level = sum(3 if f in ("identity", "lei") else 2 if f in ("financials", "ubo") else 1 for f in fields) / len(fields) + ceiling = round(82 + avg_level * 5) + est_conf = min(b.confidence_target, ceiling) + return { + "total_ore": price * 100, "deposit_ore": deposit * 100, "balance_ore": (price - deposit) * 100, + "est_days": days, "est_confidence": est_conf / 100, "recurring": b.frequency != "Engångs", + } + + +# ── in-memory store (swapped for asyncpg when DATABASE_URL is set) ─────────── +_ORDERS: dict[str, dict[str, Any]] = {} +_JOBS: dict[str, dict[str, Any]] = {} + + +class OrderIn(BaseModel): + flow: str = "custom" + config: dict[str, Any] = {} + + +@app.get("/health") +async def health(): + return {"ok": True, "service": "0pt-data-engine", "db": bool(os.getenv("DATABASE_URL")), "sources": len(_SOURCES)} + + +@app.get("/sources") +async def sources(): + return [{k: s.get(k) for k in ("source_id", "name", "market", "access", "licence", "cadence")} for s in _SOURCES] + + +@app.get("/sources/{market}") +async def sources_for(market: str): + return [s for s in _SOURCES if s.get("market") in (market, "global")] + + +@app.post("/estimates") +async def estimates(body: EstimateIn): + return compute_estimate(body) + + +@app.post("/orders", status_code=201) +async def create_order(body: OrderIn): + est = compute_estimate(EstimateIn( + markets=body.config.get("markets", []), + entity_type=body.config.get("type", "Bolag"), + fields=body.config.get("fieldIds", []), + enrichment=body.config.get("enrichmentIds", []), + volume=int(body.config.get("volume", 1000)), + frequency=body.config.get("frequency", "Engångs"), + confidence_target=int(body.config.get("confidence", 92)), + )) + oid = str(uuid.uuid4()) + _ORDERS[oid] = {"order_id": oid, "flow": body.flow, "status": "draft", "config": body.config, **est} + return {"order_id": oid, "status": "draft", "deposit_ore": est["deposit_ore"], "total_ore": est["total_ore"]} + + +@app.get("/orders") +async def list_orders(): + return list(_ORDERS.values()) + + +@app.get("/orders/{oid}") +async def get_order(oid: str): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + return {**o, "job": _JOBS.get(oid)} + + +class StatusIn(BaseModel): + status: str + + +@app.patch("/orders/{oid}") +async def set_status(oid: str, body: StatusIn): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + o["status"] = body.status + return o + + +@app.post("/orders/{oid}/run") +async def run(oid: str): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + o["status"] = "in_production" + _JOBS[oid] = await run_job(oid, o["config"]) + o["status"] = "review" + return _JOBS[oid] + + +@app.get("/orders/{oid}/status") +async def job_status(oid: str): + return _JOBS.get(oid) or {"stage": "queued", "pct": 0, "state": "queued"} diff --git a/services/api/requirements.txt b/services/api/requirements.txt new file mode 100644 index 0000000..f019664 --- /dev/null +++ b/services/api/requirements.txt @@ -0,0 +1,10 @@ +# 0pt data engine — compliant stack only (docs/07). No bypass/stealth/proxy libs. +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +httpx==0.28.1 +asyncpg==0.30.0 +python-dotenv==1.0.1 +pyyaml==6.0.2 +rapidfuzz==3.11.0 +tenacity==9.0.0 diff --git a/services/api/sources/__init__.py b/services/api/sources/__init__.py new file mode 100644 index 0000000..28170a6 --- /dev/null +++ b/services/api/sources/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — sources package (connectors + registry). diff --git a/services/api/sources/__pycache__/__init__.cpython-314.pyc b/services/api/sources/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..b2fbfa0 Binary files /dev/null and b/services/api/sources/__pycache__/__init__.cpython-314.pyc differ diff --git a/services/api/sources/__pycache__/connectors.cpython-314.pyc b/services/api/sources/__pycache__/connectors.cpython-314.pyc new file mode 100644 index 0000000..962d4a2 Binary files /dev/null and b/services/api/sources/__pycache__/connectors.cpython-314.pyc differ diff --git a/services/api/sources/connectors.py b/services/api/sources/connectors.py new file mode 100644 index 0000000..f547169 --- /dev/null +++ b/services/api/sources/connectors.py @@ -0,0 +1,161 @@ +""" +0pt data engine — COMPLIANT source connectors. + +Legal sourcing only (CLAUDE.md §1, docs/06). Priority per field/market: + 1. official API / bulk 2. authorized public web (robots + rate limits) + 3. licensed partner data 4. otherwise -> level 0 "ej tillgänglig / licens". + +EXPLICITLY NOT PRESENT (and must never be added here): + WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint + rotation to evade bans, scraping behind login walls, fake-account creation. + +Every connector returns a normalized company dict + per-field provenance rows +(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced. +Terms change — VERIFY each source's current access/licence before relying. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import date +from typing import Any + +import httpx + +# A polite client that identifies itself and obeys timeouts/rate limits. +USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)" +_TIMEOUT = httpx.Timeout(15.0, connect=10.0) + + +def _client() -> httpx.AsyncClient: + return httpx.AsyncClient( + headers={"User-Agent": USER_AGENT, "Accept": "application/json"}, + timeout=_TIMEOUT, + follow_redirects=True, + ) + + +@dataclass +class Provenance: + field_name: str + source_id: str + confidence: float + level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated + freshness: str | None = None + + +@dataclass +class CompanyResult: + org_nr: str | None = None + market: str | None = None + lei: str | None = None + name: str | None = None + status: str | None = None + address: dict[str, Any] | None = None + website: str | None = None + provenance: list[Provenance] = field(default_factory=list) + source: str = "" + + +def _today() -> str: + return date.today().isoformat() + + +# ── GLEIF / LEI (global cross-border key) — open API ──────────────────────── +async def gleif_by_lei(lei: str) -> CompanyResult | None: + url = f"https://api.gleif.org/api/v1/lei-records/{lei}" + async with _client() as c: + r = await c.get(url, headers={"Accept": "application/vnd.api+json"}) + if r.status_code != 200: + return None + a = (r.json().get("data") or {}).get("attributes") or {} + entity = a.get("entity") or {} + legal = entity.get("legalAddress") or {} + res = CompanyResult( + lei=lei, + name=(entity.get("legalName") or {}).get("name"), + status=entity.get("status"), + address={"lines": legal.get("addressLines"), "city": legal.get("city"), + "country": legal.get("country"), "postal": legal.get("postalCode")}, + source="gleif", + ) + f = _today() + res.provenance += [ + Provenance("lei", "gleif", 0.99, 3, f), + Provenance("name", "gleif", 0.97, 3, f), + ] + return res + + +# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ───────────── +async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None: + url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}" + async with _client() as c: + r = await c.get(url) + if r.status_code != 200: + return None + d = r.json() + res = CompanyResult( + org_nr=orgnr, market="NO", + name=d.get("navn"), + status="dissolved" if d.get("konkurs") else "active", + address={"raw": d.get("forretningsadresse")}, + source="brreg", + ) + f = _today() + res.provenance += [ + Provenance("name", "brreg", 0.97, 3, f), + Provenance("status", "brreg", 0.95, 3, f), + Provenance("address", "brreg", 0.95, 3, f), + ] + return res + + +# ── Companies House (UK) — free API, key required. VERIFY terms. ──────────── +async def companies_house_by_number(number: str) -> CompanyResult | None: + key = os.getenv("COMPANIES_HOUSE_API_KEY") + if not key: + # No key -> deliver honestly as unavailable, never fabricate. + return CompanyResult(org_nr=number, market="GB", source="companies_house", + provenance=[Provenance("name", "companies_house", 0.0, 0, _today())]) + url = f"https://api.company-information.service.gov.uk/company/{number}" + async with _client() as c: + r = await c.get(url, auth=(key, "")) + if r.status_code != 200: + return None + d = r.json() + res = CompanyResult( + org_nr=number, market="GB", + name=d.get("company_name"), + status=d.get("company_status"), + address={"raw": d.get("registered_office_address")}, + source="companies_house", + ) + f = _today() + res.provenance += [ + Provenance("name", "companies_house", 0.97, 3, f), + Provenance("status", "companies_house", 0.95, 3, f), + ] + return res + + +# Registries that require verified terms / keys before bulk use. +# Implement as official-API connectors only; never bypass. Until wired they +# return level-0 so the field is delivered honestly as "ej tillgänglig". +async def prh_by_businessid(business_id: str) -> CompanyResult | None: # FI — VERIFY + return CompanyResult(org_nr=business_id, market="FI", source="prh", + provenance=[Provenance("name", "prh", 0.0, 0, _today())]) + + +async def cvr_by_number(cvr: str) -> CompanyResult | None: # DK — VERIFY (Virk terms) + return CompanyResult(org_nr=cvr, market="DK", source="cvr", + provenance=[Provenance("name", "cvr", 0.0, 0, _today())]) + + +CONNECTORS = { + "gleif": gleif_by_lei, + "brreg": brreg_by_orgnr, + "companies_house": companies_house_by_number, + "prh": prh_by_businessid, + "cvr": cvr_by_number, +} diff --git a/services/api/sources/sources.yaml b/services/api/sources/sources.yaml new file mode 100644 index 0000000..bdc10a7 --- /dev/null +++ b/services/api/sources/sources.yaml @@ -0,0 +1,99 @@ +# 0pt data engine — sources registry (docs/06). Drives the router, /sources API, +# and the public sourcing-map. access: api | bulk | authorized_web | licensed. +# VERIFY current access/licence/robots per source before relying — terms change. +sources: + - source_id: gleif + name: GLEIF (LEI) + market: global + access: api + cadence: daily + licence: "open (CC0) — VERIFY" + entities: [company] + fields: [lei, legal_name, address, status] + base_url: https://api.gleif.org + robots_ok: true + notes: "cross-border key" + + - source_id: brreg + name: Brønnøysundregistrene (Enhetsregisteret) + market: NO + access: api + cadence: daily + licence: "NLOD open — VERIFY" + entities: [company] + fields: [name, status, address, nace] + base_url: https://data.brreg.no + robots_ok: true + notes: "open API + bulk download" + + - source_id: companies_house + name: Companies House + market: GB + access: api + cadence: daily + licence: "free, API key required — VERIFY" + entities: [company, officers, financials] + fields: [name, status, address, officers] + base_url: https://api.company-information.service.gov.uk + robots_ok: true + notes: "key via env COMPANIES_HOUSE_API_KEY" + + - source_id: prh + name: PRH avoindata (Trade Register) + market: FI + access: api + cadence: daily + licence: "CC-BY — VERIFY" + entities: [company] + fields: [name, status] + base_url: https://avoindata.prh.fi + robots_ok: true + notes: "connector pending verification" + + - source_id: cvr + name: CVR / Virk + market: DK + access: api + cadence: daily + licence: "terms apply (Virk) — VERIFY" + entities: [company] + fields: [name, status] + base_url: https://datacvr.virk.dk + robots_ok: true + notes: "connector pending verification" + + - source_id: bolagsverket + name: Bolagsverket + market: SE + access: licensed + cadence: daily + licence: "restricted for bulk — partner/licensed route" + entities: [company] + fields: [name, status] + base_url: https://bolagsverket.se + robots_ok: true + notes: "no bulk scrape; licensed/partner only" + + - source_id: rdw + name: RDW open vehicle data + market: NL + access: bulk + cadence: daily + licence: "open — VERIFY" + entities: [vehicle] + fields: [make, model, year, specs] + base_url: https://opendata.rdw.nl + robots_ok: true + notes: "technical/fleet vehicle data, not owner PII" + + - source_id: opensanctions + name: OpenSanctions + market: global + access: bulk + cadence: daily + licence: "open — VERIFY commercial terms" + entities: [company, person] + fields: [sanction, pep] + base_url: https://www.opensanctions.org + robots_ok: true + notes: "screening/diligence enrichment" diff --git a/services/api/workers/__init__.py b/services/api/workers/__init__.py new file mode 100644 index 0000000..1bb7ead --- /dev/null +++ b/services/api/workers/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — workers package (router + pipeline). diff --git a/services/api/workers/__pycache__/__init__.cpython-314.pyc b/services/api/workers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000..786093f Binary files /dev/null and b/services/api/workers/__pycache__/__init__.cpython-314.pyc differ diff --git a/services/api/workers/__pycache__/adaptive_router.cpython-314.pyc b/services/api/workers/__pycache__/adaptive_router.cpython-314.pyc new file mode 100644 index 0000000..097062a Binary files /dev/null and b/services/api/workers/__pycache__/adaptive_router.cpython-314.pyc differ diff --git a/services/api/workers/__pycache__/pipeline.cpython-314.pyc b/services/api/workers/__pycache__/pipeline.cpython-314.pyc new file mode 100644 index 0000000..64df536 Binary files /dev/null and b/services/api/workers/__pycache__/pipeline.cpython-314.pyc differ diff --git a/services/api/workers/__pycache__/run.cpython-314.pyc b/services/api/workers/__pycache__/run.cpython-314.pyc new file mode 100644 index 0000000..f903945 Binary files /dev/null and b/services/api/workers/__pycache__/run.cpython-314.pyc differ diff --git a/services/api/workers/adaptive_router.py b/services/api/workers/adaptive_router.py new file mode 100644 index 0000000..00ca839 --- /dev/null +++ b/services/api/workers/adaptive_router.py @@ -0,0 +1,23 @@ +""" +Adaptive router — chooses among COMPLIANT methods only (docs/06). +Order of preference: official API > bulk > authorized public web > licensed. +There are NO bypass/stealth/proxy-rotation tiers here, by design (CLAUDE.md §1). +""" +from __future__ import annotations + +# market -> ordered list of source_ids to try (compliant connectors only) +MARKET_SOURCES: dict[str, list[str]] = { + "NO": ["brreg", "gleif"], + "GB": ["companies_house", "gleif"], + "FI": ["prh", "gleif"], + "DK": ["cvr", "gleif"], + "SE": ["gleif"], # Bolagsverket bulk is licensed; SE company core via GLEIF + licensed/partner +} + +# Access rank: lower = preferred (cheapest compliant path first). +ACCESS_RANK = {"api": 0, "bulk": 1, "authorized_web": 2, "licensed": 3} + + +def plan_for_market(market: str) -> list[str]: + """Return the ordered, compliant source plan for a market.""" + return MARKET_SOURCES.get(market, ["gleif"]) diff --git a/services/api/workers/pipeline.py b/services/api/workers/pipeline.py new file mode 100644 index 0000000..8b4f131 --- /dev/null +++ b/services/api/workers/pipeline.py @@ -0,0 +1,89 @@ +""" +Production pipeline (docs/06): sourcing -> extraction -> structuring -> +enrichment/fusion -> QA. Compliant connectors only; every field gets provenance; +rows below the order's confidence target are dropped (deliver fewer, never fabricate). +""" +from __future__ import annotations + +from typing import Any + +from rapidfuzz import fuzz + +from sources.connectors import ( + CompanyResult, + brreg_by_orgnr, + companies_house_by_number, + gleif_by_lei, +) +from workers.adaptive_router import plan_for_market + +# Minimal built-in seed so a job is demonstrable without customer input. +DEMO_SEEDS: dict[str, list[str]] = { + "NO": ["923609016"], # brreg orgnr (example) + "GB": ["00000006"], # companies house number (example) +} + + +async def _fetch(source_id: str, key: str) -> CompanyResult | None: + if source_id == "gleif": + return await gleif_by_lei(key) + if source_id == "brreg": + return await brreg_by_orgnr(key) + if source_id == "companies_house": + return await companies_house_by_number(key) + return None + + +async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]: + markets: list[str] = config.get("markets", []) or ["NO"] + threshold = float(config.get("confidence", 92)) / 100.0 + seeds: dict[str, list[str]] = config.get("seeds") or DEMO_SEEDS + + records: list[dict[str, Any]] = [] + per_market: dict[str, int] = {} + + # 1) sourcing + 2) extraction + for market in markets: + keys = seeds.get(market, []) + plan = plan_for_market(market) + got = 0 + for key in keys: + for source_id in plan: + try: + res = await _fetch(source_id, key) + except Exception: + res = None + if res and res.name: + records.append({ + "market": market, "name": res.name, "status": res.status, + "lei": res.lei, "org_nr": res.org_nr, "source": res.source, + "provenance": [p.__dict__ for p in res.provenance], + "confidence": max((p.confidence for p in res.provenance), default=0.0), + }) + got += 1 + break # first compliant source that answers wins + per_market[market] = got + + # 3) structuring — dedup by fuzzy name within market + deduped: list[dict[str, Any]] = [] + for r in records: + dup = next((d for d in deduped if d["market"] == r["market"] and fuzz.ratio(d["name"], r["name"]) > 92), None) + if dup: + dup["confidence"] = min(0.99, dup["confidence"] + 0.1) # corroboration bump + else: + deduped.append(r) + + # 4) enrichment/fusion already folded into confidence above. + # 5) QA — drop rows below the order's confidence threshold. + delivered = [r for r in deduped if r["confidence"] >= threshold] + dropped = len(deduped) - len(delivered) + avg_conf = round(sum(r["confidence"] for r in delivered) / len(delivered), 3) if delivered else 0.0 + + return { + "stage": "qa", "pct": 100, "state": "done", + "metrics": { + "rows": len(delivered), "dropped_below_threshold": dropped, + "per_market": per_market, "avg_confidence": avg_conf, + }, + "records": delivered, + } diff --git a/services/api/workers/run.py b/services/api/workers/run.py new file mode 100644 index 0000000..8c40927 --- /dev/null +++ b/services/api/workers/run.py @@ -0,0 +1,21 @@ +""" +Worker entrypoint. In production this drains a queue of order jobs and runs the +compliant pipeline. For now it exposes a CLI to run a single job locally. + python -m workers.run +""" +from __future__ import annotations + +import asyncio +import sys + +from workers.pipeline import run_job + + +async def _main() -> None: + order_id = sys.argv[1] if len(sys.argv) > 1 else "demo" + result = await run_job(order_id, {"markets": ["NO", "GB"], "confidence": 90}) + print(result["metrics"]) + + +if __name__ == "__main__": + asyncio.run(_main())