From 31128a71c83731d145f6ea941a107d5c6d91212d Mon Sep 17 00:00:00 2001 From: Simon Axelsson Date: Wed, 24 Jun 2026 20:02:50 +0200 Subject: [PATCH] feat(engine): import compliant data engine (services/api) + infra MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FastAPI orchestrator (health/sources/estimates/orders/jobs per docs/04), adaptive router (compliant methods only, no bypass tiers), pipeline (sourcing->extraction->structuring/dedup->fusion->QA with per-field provenance, drops < confidence target), live connectors GLEIF/Brreg/Companies House (PRH/CVR/Bolagsverket gated to level-0 until terms verified). sources.yaml registry. infra/docker-compose + Caddyfile for the VPS. FlareSolverr/stealth/proxy/fake-account paths intentionally NOT carried over (CLAUDE.md §1). py_compile clean. Co-Authored-By: Claude Opus 4.8 --- infra/Caddyfile | 13 ++ infra/docker-compose.yml | 69 ++++++++ services/api/Dockerfile | 15 ++ services/api/README.md | 34 ++++ services/api/orchestrator/__init__.py | 1 + .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 169 bytes .../__pycache__/main.cpython-314.pyc | Bin 0 -> 11706 bytes services/api/orchestrator/main.py | 150 ++++++++++++++++ services/api/requirements.txt | 10 ++ services/api/sources/__init__.py | 1 + .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 164 bytes .../__pycache__/connectors.cpython-314.pyc | Bin 0 -> 9634 bytes services/api/sources/connectors.py | 161 ++++++++++++++++++ services/api/sources/sources.yaml | 99 +++++++++++ services/api/workers/__init__.py | 1 + .../__pycache__/__init__.cpython-314.pyc | Bin 0 -> 164 bytes .../adaptive_router.cpython-314.pyc | Bin 0 -> 1273 bytes .../__pycache__/pipeline.cpython-314.pyc | Bin 0 -> 5107 bytes .../workers/__pycache__/run.cpython-314.pyc | Bin 0 -> 1275 bytes services/api/workers/adaptive_router.py | 23 +++ services/api/workers/pipeline.py | 89 ++++++++++ services/api/workers/run.py | 21 +++ 22 files changed, 687 insertions(+) create mode 100644 infra/Caddyfile create mode 100644 infra/docker-compose.yml create mode 100644 services/api/Dockerfile create mode 100644 services/api/README.md create mode 100644 services/api/orchestrator/__init__.py create mode 100644 services/api/orchestrator/__pycache__/__init__.cpython-314.pyc create mode 100644 services/api/orchestrator/__pycache__/main.cpython-314.pyc create mode 100644 services/api/orchestrator/main.py create mode 100644 services/api/requirements.txt create mode 100644 services/api/sources/__init__.py create mode 100644 services/api/sources/__pycache__/__init__.cpython-314.pyc create mode 100644 services/api/sources/__pycache__/connectors.cpython-314.pyc create mode 100644 services/api/sources/connectors.py create mode 100644 services/api/sources/sources.yaml create mode 100644 services/api/workers/__init__.py create mode 100644 services/api/workers/__pycache__/__init__.cpython-314.pyc create mode 100644 services/api/workers/__pycache__/adaptive_router.cpython-314.pyc create mode 100644 services/api/workers/__pycache__/pipeline.cpython-314.pyc create mode 100644 services/api/workers/__pycache__/run.cpython-314.pyc create mode 100644 services/api/workers/adaptive_router.py create mode 100644 services/api/workers/pipeline.py create mode 100644 services/api/workers/run.py diff --git a/infra/Caddyfile b/infra/Caddyfile new file mode 100644 index 0000000..ad7fc66 --- /dev/null +++ b/infra/Caddyfile @@ -0,0 +1,13 @@ +# 0pt — reverse proxy + automatic TLS (docs/08). Frontend stays on Vercel (0pt.app); +# these subdomains front the VPS services. Point the DNS A-records at the VPS. +api.0pt.app { + reverse_proxy api:8000 +} + +data.0pt.app { + reverse_proxy nocodb:8080 +} + +dash.0pt.app { + reverse_proxy metabase:3000 +} diff --git a/infra/docker-compose.yml b/infra/docker-compose.yml new file mode 100644 index 0000000..f6707be --- /dev/null +++ b/infra/docker-compose.yml @@ -0,0 +1,69 @@ +# 0pt — always-on services for a single small VPS (docs/08). Vercel hosts the +# Next.js frontend; this compose runs the data engine + DB + embeds + proxy. +services: + postgres: + image: pgvector/pgvector:pg16 + environment: + POSTGRES_USER: opt + POSTGRES_PASSWORD: ${PG_PASSWORD} + POSTGRES_DB: opt + volumes: + - pgdata:/var/lib/postgresql/data + - ../sql:/docker-entrypoint-initdb.d:ro + restart: unless-stopped + + redis: + image: redis:7 + restart: unless-stopped + + api: + build: ../services/api + command: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000 + environment: + DATABASE_URL: postgres://opt:${PG_PASSWORD}@postgres:5432/opt + COMPANIES_HOUSE_API_KEY: ${COMPANIES_HOUSE_API_KEY:-} + INTERNAL_TOKEN: ${INTERNAL_TOKEN} + depends_on: [postgres, redis] + restart: unless-stopped + + worker: + build: ../services/api + command: python -m workers.run + environment: + DATABASE_URL: postgres://opt:${PG_PASSWORD}@postgres:5432/opt + COMPANIES_HOUSE_API_KEY: ${COMPANIES_HOUSE_API_KEY:-} + depends_on: [postgres, redis] + restart: unless-stopped + + nocodb: + image: nocodb/nocodb:latest + environment: + NC_DB: "pg://postgres:5432?u=opt&p=${PG_PASSWORD}&d=opt" + NC_PUBLIC_URL: "https://data.0pt.app" + depends_on: [postgres] + restart: unless-stopped + + metabase: + image: metabase/metabase:latest + environment: + MB_DB_TYPE: postgres + MB_DB_HOST: postgres + MB_DB_PORT: "5432" + MB_DB_USER: opt + MB_DB_PASS: ${PG_PASSWORD} + MB_DB_DBNAME: opt + depends_on: [postgres] + restart: unless-stopped + + caddy: + image: caddy:2 + ports: ["80:80", "443:443"] + volumes: + - ./Caddyfile:/etc/caddy/Caddyfile + - caddydata:/data + depends_on: [api, nocodb, metabase] + restart: unless-stopped + +volumes: + pgdata: {} + caddydata: {} diff --git a/services/api/Dockerfile b/services/api/Dockerfile new file mode 100644 index 0000000..409c49d --- /dev/null +++ b/services/api/Dockerfile @@ -0,0 +1,15 @@ +# 0pt data engine — API + worker image (runs on a VPS, NOT Vercel; long jobs). +FROM python:3.12-slim + +WORKDIR /app +ENV PYTHONUNBUFFERED=1 PIP_NO_CACHE_DIR=1 + +COPY requirements.txt . +RUN pip install -r requirements.txt + +COPY . . + +EXPOSE 8000 +# API: uvicorn orchestrator.main:app --host 0.0.0.0 --port 8000 +# Worker: python -m workers.run +CMD ["uvicorn", "orchestrator.main:app", "--host", "0.0.0.0", "--port", "8000"] diff --git a/services/api/README.md b/services/api/README.md new file mode 100644 index 0000000..6ca33b9 --- /dev/null +++ b/services/api/README.md @@ -0,0 +1,34 @@ +# 0pt data engine (`services/api`) + +The legal data motor (docs/06). FastAPI orchestrator + worker pipeline that turn +**compliant** sources into delivered, provenance-tagged data. Runs on a VPS via +[`../../infra/docker-compose.yml`](../../infra/docker-compose.yml) — **not on Vercel** (long jobs). + +## Sourcing boundary (CLAUDE.md §1 — non-negotiable) + +Per field × market, in strict order: **official API/bulk → authorized public web +(robots + rate limits) → licensed partner data → otherwise level 0 "ej tillgänglig"**. +We never invent values. + +**Deliberately excluded (never add here):** WAF/bot-detection bypass, FlareSolverr, +stealth/undetected browsers, proxy/fingerprint rotation to evade bans, scraping behind +login walls, automated fake-account creation. This is the honest-accuracy moat. + +## Layout +- `sources/connectors.py` — compliant connectors (GLEIF, Brønnøysund, Companies House live; PRH/CVR/Bolagsverket gated on verified terms → level 0 until wired). +- `sources/sources.yaml` — source registry (access/licence/cadence; VERIFY terms). +- `orchestrator/main.py` — REST API (health, sources, estimates, orders, jobs) per docs/04. +- `workers/adaptive_router.py` — picks the cheapest **compliant** method per market (api > bulk > authorized_web > licensed). No bypass tiers. +- `workers/pipeline.py` — sourcing → extraction → structuring (dedup) → enrichment/fusion → QA (drops rows below the order's confidence target). Writes provenance per field. + +## Run locally +```bash +pip install -r requirements.txt +uvicorn orchestrator.main:app --reload # API on :8000 +python -m workers.run # run one demo job +``` + +## Reconciliation note +Supersedes the earlier `nordic-eu-data-platform` backend: the compliant connectors + +pipeline are migrated here; the FlareSolverr/stealth/proxy/fake-account paths and the +`test_bypass_chain`/`test_proxy_pool` examples are intentionally **not** carried over. diff --git a/services/api/orchestrator/__init__.py b/services/api/orchestrator/__init__.py new file mode 100644 index 0000000..993587d --- /dev/null +++ b/services/api/orchestrator/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — orchestrator package (FastAPI app). diff --git a/services/api/orchestrator/__pycache__/__init__.cpython-314.pyc b/services/api/orchestrator/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..ecffe26ec0cf7bb419eb79dd3d48c0d195ec714b GIT binary patch literal 169 zcmdPq6%$&V zT2vfUoSB=S7n7b@l2MuzQ<|x3P*9?qSWplHlq}0kPA!f}EXa(>FG|ixEiNfaEXgm5 siI30B%PfhH*DI*J#bJ}1pHiBWYFESxG!|rWF^KVrnURsPh#ANN06_pMC;$Ke literal 0 HcmV?d00001 diff --git a/services/api/orchestrator/__pycache__/main.cpython-314.pyc b/services/api/orchestrator/__pycache__/main.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..307123e45278e772c89ccc1cb1cd832fb23d07de GIT binary patch literal 11706 zcmcIqdr%zrec!$Jo8uwz5^#V3p$8xdJuGCA10iujLU=(Z$v!wAcMF_!+?{s!kc8dj zsQn!E+xLQq(UnqXwIjpdYsAC@MfDsTNA0 zHd72G&_fhMdo;*@qBD7u?}Pu8th#yGZHvn`pt z++5$DY>k>?oJ{_{^!(~NV2m=go}z^Ep%q~hQ@|85Ma%|fBU9Y06)Kn#$d@u@OgXcO zsetlIriyVf)y)Q>lBr>8naxZcQ_nOoTOi%YG&JjlstpvJJ;Tb|N-U#9hbSj}^|^2`j4(C3u8itL*?km=I~eGabbv+XH;o*()Nb4cYm zrXBiyZdJeSPg&pdnYr9KyV>!S)*YGFoiM}URWme^Gw*`(BROZ@3?l`yds9ZE1-Mu24V-xY+1u zILf;ICHa=?NPrjIeI8eg3y!h8zy*XD=h_mA1^MQd*2X4RXKXwk2?wHri;pF^U^qJJ zibW%rn=DKs%0to>aC!Usn=i(=^DO5Yi}AvCSMSLItnsp3kd3&uL}JkCTx^(!;oRe~ zL{#8i;i&6_A8;%T!v!KxL87f@{3_%j8yhr7RF%Q zoGl#4baF)kms!pdR)z^Ccp)~WXWu3lx>5yP|dU4L^#O0I9xwY83+oAK*SXf z$Jq#+N|U9LmaTzkG$sUua4gFALSEMw5XRuGbw@8ZYGh;j)MR`2z(8Nur63!}rqICA zd6MZr5e|hS?8N}bLW#K}z_TY}AvOXTBh2cDqnONFLyJjmOVdsmSuccz2rC;WSdNDg zWqNC)R<;Ksk=R9lj0*$Vcr`oD!fAvu*)cW{f=;rL(23gNZFU#5p8y@}I4Gj0&Wy?r78UShrB7C~=al5Qd498?6P+5SJ9{u~DV)$jXA97F4AU-)U7! zKb(Y2k{X0d5-v*3P_3ZdOf^x0j-dzjj0Tk`qZJH{4sv?TL7kD*8PYiu$r(vYlVE1d z(9%M3R>4C0*)TWc6Rf1pPU;+jjmZPCbIO)3UI>o|1lALsG)Ka`@DhmFplloufVc=e zR{))5EtJVt%O<}+8W?B&e%a#pkH45BPF* zlW#l_jyAm1>m<%KvI95n-(cd^i;UM9H?Jm$I zgXB;^Dh(qs4)}z}xoWt**b@koK6D@x_dFEg;qr(?lGLxLeA@5;qm*R@U$dVlIkI%J zYG_7+9b&TT{jmNdwSXWxt=7rCx#ziORceQsUc=qp#p@EV6`uOW9Ixh;qEx#O!efzq5ah?{ao zGYx6h>k&8mD%2T=964uCXZylBXvUa3a_*|0vVT+VY`N`oxLBjEn=wCabhU*_W%V7m zuUhC@9%U!d&inuL%TI=(s*OpnKow0gNL!sQAF*dJxO^N;V z)m&~~Q|s_rm^`l?N;C2Zm!K2ou2iX(eKYy{N5SeafHo_*k5a@T!>-aU$Q`>;C-y?3 z1}wWIH9#E!o&#M0B}(Ya=oz(yqJRS^b!V5Tq`r#k0lqO)m0-=*QhT&vx={m@WB7S{ zFCZh^5l`3g!~WA0l%l1aNYwDbuJ0y7DfF?!(eW+uoZKbJcJ9mh2S^_KH+sws6T_C)(@2Gx7K# zD#+Jr>u3&ab82+H@(=#wh52u`|6iTP&~H7B`!Frs;~LrC)ysG~yVFO(bwF{WhC2)| znPzj(f;$3b+);>tM7g_cyBV3we{*ojUb$eeOkJ3*Ub1f%?VG<7dCYZVmn{&bq1QjQ zhvUVNgAxeh>ir^J3H7m8hWenrF61+s=b%iv8;7(fEI@BvSdBjM9~lo8;%md)#%{xRLSVdz?zYZvW6X(H9>gTi?VJQ>Bg(*aSV_~bJ4Y`t5wjy;zq@sRx)o-b}XAMZ}v~;PrH|j8WxHg=4uv;woCRM zw{(m4)2H}XY}p`K}@_NU3hiT`D)UG^K_(2XJ7b6#bQytm*)2CAXZ@)CV<5uzf?(bLJwu^nI7W@2SaAfi1$g-(m`mk8G zb*}ps|J~mCo;&&fQ2vv0@yzgoXLv=gwHa0@t=>T3;bdtRgJx5RM!_=ipJWbqG5xaM zf9zyOe-H?p327}(KSb|Nj#+gC%_@clvq}TzkrvGY@KoiPRg3}5DkIz?CfRtB11r@N zl?{rC&TWR)vTg+ITdob01|m+QM96y^Ow?2$!QM1%lfV zJagyO2vs1D*A`d@&XVNesai=_0$3mLe{TlrW9vy{Gnmj3VYMlVxbXM!e&ZmaXgCJ; zs^I)Ay78qEGN4q{C2InB`Sj|Iuf7>W7EAjwNLsCJ)}`-2qpXRY9{{6X@mdhS!47nQ z>`;6-vL-Yv+YY-2+#T-zF8?X!IER3^Q4ew-i$$O}FB{;lWTO)t`r*I`>_2&m>FnxX zb7HbV**$j&TJ^xg;{l@z{khEv5@Hr>TW^K_`tXkn{;u?IOYa<#+%HN8UlIpL?v9`nrBbKOOk0 z>GhYO;4TzLgb%F=QI~l9p>6oFWKBHgt6+@REN(shrn%klcG&^j`^I*|4|GuQSNe9t z-xwe2SD^T#zSf%M>^aB#7g39|zL)8Fn31ehMZ53kx6A-olpn4&HJr)2!nKt#jz&5)H z08vcI9@w52S;3WMj9f-g;3nA|NJ}IJw>_7HYRT(BWq)12)Gvz_}A)S_(h zsG$f^i>oB7XXes$^|$x@$)35AZ?)f=n7<_M?Ov$uzUH2C&smSz>R^1lP4)^L&2l`S8I9d zuw<C!TYFYHR^__4N1qXTuL%seDpl(V^r3D|HDuTA1^3ox=mpyNGxXAB zd-=@j`aZ>4^^8HCH7nN7yi+4%QhR@<+08>fwZA$$W7(IcgjxbNF5YG&=CEA*;A}(JoZ{yxSlpTe7B4qfs$IKG`vBB4P`dD}ivb zL6KeX{Bwa3K>>3oaf)7u8ZAU$a5Y#E#i#=#H0;u#F_8#|WIe{M3M3qZI=HA%zsOop zDzY{f4)Lg{z>HCJjV;Kr;Eo474)h22HK_8#!~YLNMBP}*-Z`-BEV^!;v0l%c$(uFL z?UI~L%S9#EduDpBpPV^4dt~mYRMfhhS9tA}tFK)9!qqRNCZSDU({f?)^|qO|>+Lh` zv+cJ!q{4lO;af9h`{xFvLTI=foX^*{&1}2gJkvb8ZEjL3IB>75A{Bb~{EhSP#%{#s zBJ&+mWxG`Nxkq`XO@@b*$!bh?J}RKBjyFTs&b@x_jfi9^Pj)UBRnBT>y>kQapS^i@ z-gLX+&VKRK*`-r2FPwT=EF2R5=&r+%ctUp}fFPkS$4{u1Auz|F%8POr zaIZrVeAUw7_Ye+g246G+kp#=Hg``$&p^%E_z6lLu@bKslP_Bl&w~W^vGY-k=T6IHI zy68!J`i5wJ1z6xSVxF66%yR=Kd^XIJ!1bv&sE*$9fPyGhNM29g3V34VWd8^fYmxAJ z04IB^#=(Qy0KG7bf`@+-<;$Eu+9#^+8l#&5_;iBa_aS*n_~enBp>^8!(B6LwF7)5n zh^)By!1-u^m$mTqvQbCKrU)dC%Y`8V1dJyo)3H@h$qd#%DS2=qhL8aXEFtBQ#_2qtPh)WBCiJp(2dm`SF7f*(9lg~D$^ZQAxUtZRD` zvlAJ64YP7Lu?!VMd@k#1B_C&EBKl{LvjjlW@H)K;_3)*ejXB^k~cM8&9m{Qm+s4OB}|hJaE}op zdEqM>0p?onPoawY7Dh;)C)tmbY6<%(Fz(NwnWFerx3C2~Z{(|Fg6IAMYLnOk{RPj! zhl;{Zx|3s9dX}xWWOo{hwmpvqmI6f;Mbix5_g7IF&y-LZED)G;{cnb7U~#k1fbi5O zL3C7|83kjcH$I>sn%+%bpP*nw==Ldz_{<81(V%M=r7G*tI8DRHGi14RP}R{PMNwvZt_lhf1{Os6!JyLPY zQhv)9yQXTd99u3ZOV?VR(}iC-0AHFP=&6F8k9mY~ueo*ZdiqBO+Pzu#BV#3`CyQ3I zl>(IhKRN4d+^TBR&~z^`*L&$yEt-G5^pnhml1nti03<#gbLD*L1uATUuSSYb4gXeT z=w@wN?sc*r_B{&D7x=Lu!GhO29u!EKz=CBShItit3r!?H*QB(-Eu~8PJhVdv!Eb>` z0S?Y<&s}|Py8n$sOV$&j^+alP_R_5l^Yr|_+udTt2~l?bu~P#FI{ zfsX^L{)7OBsULopQvRtJNx&z5{8*1t&mkD#{umB=yBH>{USRcR+pl$p@fGSqj z;m*?S-Ch})HI&?nW!lk<9H5l7TFZP9D;wc!a)w@M0P5_U-3=q zPfQ$gx2%OfDP%_=6!K>W!5?WxSbroI2;mPQBk<=HEZ)emd@M2nM;(Vhf8ZYz_>vI_ z`33kJOL1$V5O8NOa%0qkQ4L0;7>!{>Zbm|s^O)_x2%U+F=j%(5l6CM!MIc5#f!+ku zao@%^H?dj|=M)?x>Y0cihol2__(OOR>V8fE@uq{y-z$f>pF#~f9e4qb{MRd5nx-FV z^tA3{8%5Xtf-?UTW&MaM`H0&13(ER4YU@YTu8*k7Us9X@*gE!64SSN>WlFn5 z6)aE%%esQu-LrM`+FOY`{dc-nv=m+Vh|n6lLqy zBuy&@%8{S6Eaw3(7b|wm@wd9?gQBA&X`RKs#@T~aR^T@qFI$e^KJQ(@eiYsEro^j|K%Qan{E(S;8*G<}hNsHHSTDobP-lhkm?OsJwa!xMe`SY3mImtbv;(itlo zXA6@h5?%L5gMJijQNKcxE0yZ>>gq^;vbc1ATzZv6mp;+}*2b=7E0}uZplRpS dict[str, Any]: + markets = max(1, len(b.markets)) + fields = set(b.fields) | {"identity"} + fw = sum(FIELD_WEIGHT.get(f, 0.5) for f in fields) or 1 + em = ENTITY_MULT.get(b.entity_type, 1.0) + cm = 1 + max(0, b.confidence_target - 90) * 0.03 + ew = sum(ENRICH_WEIGHT.get(e, 0) for e in b.enrichment) + base = fw * markets * FIELD_RATE * (b.volume / 1000) * em * cm + price = round(base * (1 + ew) / 5) * 5 + deposit = round(price / 2) + days = min(14, max(1, round(1 + markets * 0.4 + b.volume / 6000 + len(fields) * 0.25))) + avg_level = sum(3 if f in ("identity", "lei") else 2 if f in ("financials", "ubo") else 1 for f in fields) / len(fields) + ceiling = round(82 + avg_level * 5) + est_conf = min(b.confidence_target, ceiling) + return { + "total_ore": price * 100, "deposit_ore": deposit * 100, "balance_ore": (price - deposit) * 100, + "est_days": days, "est_confidence": est_conf / 100, "recurring": b.frequency != "Engångs", + } + + +# ── in-memory store (swapped for asyncpg when DATABASE_URL is set) ─────────── +_ORDERS: dict[str, dict[str, Any]] = {} +_JOBS: dict[str, dict[str, Any]] = {} + + +class OrderIn(BaseModel): + flow: str = "custom" + config: dict[str, Any] = {} + + +@app.get("/health") +async def health(): + return {"ok": True, "service": "0pt-data-engine", "db": bool(os.getenv("DATABASE_URL")), "sources": len(_SOURCES)} + + +@app.get("/sources") +async def sources(): + return [{k: s.get(k) for k in ("source_id", "name", "market", "access", "licence", "cadence")} for s in _SOURCES] + + +@app.get("/sources/{market}") +async def sources_for(market: str): + return [s for s in _SOURCES if s.get("market") in (market, "global")] + + +@app.post("/estimates") +async def estimates(body: EstimateIn): + return compute_estimate(body) + + +@app.post("/orders", status_code=201) +async def create_order(body: OrderIn): + est = compute_estimate(EstimateIn( + markets=body.config.get("markets", []), + entity_type=body.config.get("type", "Bolag"), + fields=body.config.get("fieldIds", []), + enrichment=body.config.get("enrichmentIds", []), + volume=int(body.config.get("volume", 1000)), + frequency=body.config.get("frequency", "Engångs"), + confidence_target=int(body.config.get("confidence", 92)), + )) + oid = str(uuid.uuid4()) + _ORDERS[oid] = {"order_id": oid, "flow": body.flow, "status": "draft", "config": body.config, **est} + return {"order_id": oid, "status": "draft", "deposit_ore": est["deposit_ore"], "total_ore": est["total_ore"]} + + +@app.get("/orders") +async def list_orders(): + return list(_ORDERS.values()) + + +@app.get("/orders/{oid}") +async def get_order(oid: str): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + return {**o, "job": _JOBS.get(oid)} + + +class StatusIn(BaseModel): + status: str + + +@app.patch("/orders/{oid}") +async def set_status(oid: str, body: StatusIn): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + o["status"] = body.status + return o + + +@app.post("/orders/{oid}/run") +async def run(oid: str): + o = _ORDERS.get(oid) + if not o: + raise HTTPException(404, "order saknas") + o["status"] = "in_production" + _JOBS[oid] = await run_job(oid, o["config"]) + o["status"] = "review" + return _JOBS[oid] + + +@app.get("/orders/{oid}/status") +async def job_status(oid: str): + return _JOBS.get(oid) or {"stage": "queued", "pct": 0, "state": "queued"} diff --git a/services/api/requirements.txt b/services/api/requirements.txt new file mode 100644 index 0000000..f019664 --- /dev/null +++ b/services/api/requirements.txt @@ -0,0 +1,10 @@ +# 0pt data engine — compliant stack only (docs/07). No bypass/stealth/proxy libs. +fastapi==0.115.6 +uvicorn[standard]==0.34.0 +pydantic==2.10.4 +httpx==0.28.1 +asyncpg==0.30.0 +python-dotenv==1.0.1 +pyyaml==6.0.2 +rapidfuzz==3.11.0 +tenacity==9.0.0 diff --git a/services/api/sources/__init__.py b/services/api/sources/__init__.py new file mode 100644 index 0000000..28170a6 --- /dev/null +++ b/services/api/sources/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — sources package (connectors + registry). diff --git a/services/api/sources/__pycache__/__init__.cpython-314.pyc b/services/api/sources/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..b2fbfa0c1d9b4a7fc44581ae9678f3ab0f92f767 GIT binary patch literal 164 zcmdPquHLb zr$kDlu23R9K%X~hrds4p1B}MNmlF)!PUA>AJ776))vSa!57^H08lNd|*X$yv zImYe7RoQN~zIBvn5v`)F!(^msPLVICM0+_EacPC)W;X>T9FS7bo+;6usjRs&xlYI} zd@Q#}bU}I%X{T5$2EK&wC1NS?Zo-#}Wx$sc-Yr%DUrG2fu?qM-gfADX$4kVT@ltW` zxLd3pFB66Fa-UhN5KRq~r!M}_Ff6_;rU{{d77!$PDlALFzs~=)(0$>B;lY8fp%Foi z#+9HX1f#Mn1+}Q6dU<|OnhHdWoUlA4MCC|AXzm{Dy42I_oe2r=zk8@f2t|XcukDz} zD-0{)s1nu^LQGPG$*>d&`DOyjHAy?c3&J6<5S^S12gA@{*YJSg6DH!3Yl0vg_6mWx zHVrlBq>vDcPej5&;kq;-G%L}GsHO@B1tp+KLL@vB)>IEPb;Jvipd_nUKA>nabVueT zv_mJ_w4_`QtCG-qT8K!qQbcGI>ZPlK7LG)wKKPkD6$wv4??wytFv;GRKmy$ZBZDss zLl;JbVX^mOFUX@gAcusRxT*;IrAlvY{J4>)AL+1MQZ zaHs{_oD7E~%xi0f_0Pz-^#}|`k<@8fQdJM^8R$rZ@lFZqbU3C8aoK1isWkwZ( z(*c-0*@UCL;y~Zaf)of&XSU&f$d4 z6#ipCt-zqhss?7B8l|q7j(?Y;LXcKXy+}{fA}yMHG@MoDH&|BZj0Kx3!a?2ecBu(D z*c}OjK;EOE6Apwh6VOy_zh3X|J~4I)H`bUMo{7q1Q(E&&+Om(&=WO(C;4jbRHUh>XZ~KwB6#UL_W>H%u^5St_^$j zFYMLnqM9GhtsGAt?N7SSKX6r~D{H>AunumMV$B?BXq(J3bEF>*k!HpzjRrI`UD>BG zjeuz5tZ3Q?IfP>f$3mK!a2#+J!dW%5XoDC}Vs?!atspgr&JX9#vCgQPqVwc9_~pQi zq}y_58csuBo?=p@}t&i2snc&IDc6WMqyAS}|L->>t2|4cL# zkH7}P?ewSdKm@yjSfSrP8CFy+0=OebjfA{Y3mdKxrm4IAeiaXL&>zq=B|H(=Bo*?u zFM_fMiV}j;RqKHKCH1MRH4y)}Su_MsgIF=f5+T!?g~W`dpfxLrahW_Di5ZJSYrJTMdbUiS zo#c_F8h6xD;{~FHl^a)!d9rPneSV8F2&Zs;mdMm zRrB2oRVlVAUA||&XW>AK-IJHwTVokYK2EqHTy0Cgc=mlmJ0ZOw)~D zAbE=+pU0I-4s6bo8qZS1SyOe`48?bZY#!`9ddR~lSVT8#(NG|v+rnzpfT10$qH}&y zLg|FkWawbP!_Xht3l~qke&UT&8}_=7?R6=8Ly~PUhK~mza|kG0e9(*ZLxI%V9V-zb zd}!O64Ox<{gxiov^>?M)4oPOOKU2(IP+vtQfnqMBOTIv|)V+zHWaDuX-5M{HBcTZB zt2o||VAS|6l?^M@zH>d8AR;|WY|`d8D)l1emlaIkB(mlYoipcjXh8Y|JpM>^qaP~Q49GE z6trX6(FjDdBi?OvqPaaY*ON4!cnnj}M>R;qN9|gJds>YcJId}_%ktD(=27#mwX9FA z#f{;KBw>VlLJPL1wXkcSe743DTC`{LMaL+&YYPRr63!=-aOO%BKA}Wmu7v9eC5o~o ze5_b3mULtkLCi9S>XwA;%EzpumXq`_YN{P!46P&*#8PO*eT5mdwr5((%i|%hEI-d{ zYC+Q~6XGVB%km?ZIGmJc?bZg$LiN&vKP2=fp+O5Z=oF}f6xrKw9*mP^p?E5{GvNeP zQ(?F*sDb^7f4J*UGBs3~KFWk?Px+iD>koj-LUE0~i}C8YX> z6x0E875N6~tQ-O-B76`HQxB~aLjmH11ayAOw$V8-Xu)IF9TD`jyE0I0Be5Ycowm>s zj4`mCRNaxm>p>D@gIF6}4sidJgwDmHfc6nUXY%cih9uogh7Nj?sK^~?OMn$KrheZ4 zFLR|o|LvL2`Hi2(I{Dw8dCy9WT%DPcG~I3(5&j^y$l{y`>JklaKs2gNkBxMqGjS!N z(?Ok9bUL7PN1gsp4td2hKkUhH32}`lk?pV3dNX1YCkys3iyMxX zj~y*5V=2dp4aeD}5|IZCvKfs znogDMPrIv^q<6wU2(LI(dpp+3?zrz)u2+6^EY&5Z-FuhK@7jKBTk)iX_SNY-=KHpF z+ea;_vzIp++Wj*9X-Ud9zrBjZ> z7@u9bzQV2qR*O=OBb#Ps5C4c_Z2X%}5QnSeD-RLQo3IlHzhVf#nE0#2@73F|4mYd& z>**iSeJ8B%Fu?t!vfpg}2fDA*`j2J6try!NalMA_r`Ywq9P*7#kp4bjfwZ9t>1m`N zR0>E>oNI;8hx-el=!fleKf`@^6svvMNkjU5iU$5ZL#UbSueIH`)BR58eFu+xxd~~d z4a#gYz7X^PY`&4vpqt@5%Zg!w5o7EOwDL?V;T0&9vl2?tN&vFVY~m-`e2PRLw-Qjt zy@~_;E@+hNYJMh`*`XB&;1#zEmK8vr^9gx{JM+BE6LHS`b#ZR+_ke8w4*&Py z-~KHT&NX)=i*R@s=AD51wkfY1^-1!yq^S@A{|oLqH3E4gQBr(Ex@qVFx|XxDZn+kf zuf^fA#TE*yDqehsbPim`@T>%yK5$(x2LAvWkm06I4G=dI@QC#D9%nd%#1GsJlt|hO zcplL={Ctvx|2$(jiFjM$u_$;F6%olrBqKm{I;332$V)&}G+{F^V}Oj4bdDfn9Q|o1 zK+f}j17pCq)+UcbIwmz=osxsn}BBT3K}p4OX%;_b->ym5H@w^Q z;XLw+)vy+4d%b6dups}Wx%aF?U1-$N%z#U>s1``!ntP1xN9wd$Xz$xQ^(yc z!;HH%G^W?m$k!3t!1Z+6?t18+7U$gqJn~0PNIPs$=G$v>PL9t^z_okM16012Dpz6F zIaOYPsvO7)IDGI)wx1`_r>OEzbE^DVKDJ0@m064)-ExoMdZd~f%lX}-=$e_vaMzLm zPOeF0uTWd+G;4-um64Yg%^ewsuYnS|jN55?T-ojFc3K{D9`j7`>?khRiPj^gt;a8b zmF@M=QFCTXZ+&+-ofddPv*bCQ+u8v0+X7A}JB)_(j*y)*sPJu)0;g)1(Te#_*KYaw zPS@@&T4K}epHm>ZL2lmiGs>wZVrv5JKN6PE0%<>k4chobzv$K0VU$CM*5 z@eDblJ2dz?{QndD|7{*jSgw9uM+jCT&k-ut%kh!0YiOYNqW}B_xFz?)`@AKw5vcQ9`Rpu3{H7pyqN$>>b&AeB5 zr=qjo_%&aq2sm2DpgKP1;FF>5$fO$=+Rkju@KuxvD5_WHU@YV(?~jr{<3K4AzY94DAZin-pye(w{oI@ATa{p5zC2N4FKz zvIiZg7Vv|<3CLA0H)cWZJE@|5>5|ghM{gZXR(et;2lDW(GgW;$S$Za2()cudOV+ln zHm=cY0BD^A;F5KRS9{j#)?Q3G&OGk9G~^JiWgDXX1xW8^S8cDCUIz%WS=Z|sr9w@)n)yv;Phjo%l9 zu9R$&Z}L#Dk?<6NLyOArRkrRi0i*z?#=zM!0w^$Agx^|hDd((m%&I#8DRv8_$nA<* zH}wtZOiYW`@NL+5%iy#G{@{8;@zKWs1H7P8(0x!)O_L3% z;B level 0 "ej tillgänglig / licens". + +EXPLICITLY NOT PRESENT (and must never be added here): + WAF/bot bypass, FlareSolverr, stealth/undetected browsers, proxy/fingerprint + rotation to evade bans, scraping behind login walls, fake-account creation. + +Every connector returns a normalized company dict + per-field provenance rows +(source_id, confidence 0-1, level 0-3, freshness) so nothing ships unsourced. +Terms change — VERIFY each source's current access/licence before relying. +""" +from __future__ import annotations + +import os +from dataclasses import dataclass, field +from datetime import date +from typing import Any + +import httpx + +# A polite client that identifies itself and obeys timeouts/rate limits. +USER_AGENT = "0pt-data-engine/0.1 (+https://0pt.app; SIAX Technology AB; hej@siax.io)" +_TIMEOUT = httpx.Timeout(15.0, connect=10.0) + + +def _client() -> httpx.AsyncClient: + return httpx.AsyncClient( + headers={"User-Agent": USER_AGENT, "Accept": "application/json"}, + timeout=_TIMEOUT, + follow_redirects=True, + ) + + +@dataclass +class Provenance: + field_name: str + source_id: str + confidence: float + level: int # 0 unavailable/licensed, 1 single, 2 partial, 3 corroborated + freshness: str | None = None + + +@dataclass +class CompanyResult: + org_nr: str | None = None + market: str | None = None + lei: str | None = None + name: str | None = None + status: str | None = None + address: dict[str, Any] | None = None + website: str | None = None + provenance: list[Provenance] = field(default_factory=list) + source: str = "" + + +def _today() -> str: + return date.today().isoformat() + + +# ── GLEIF / LEI (global cross-border key) — open API ──────────────────────── +async def gleif_by_lei(lei: str) -> CompanyResult | None: + url = f"https://api.gleif.org/api/v1/lei-records/{lei}" + async with _client() as c: + r = await c.get(url, headers={"Accept": "application/vnd.api+json"}) + if r.status_code != 200: + return None + a = (r.json().get("data") or {}).get("attributes") or {} + entity = a.get("entity") or {} + legal = entity.get("legalAddress") or {} + res = CompanyResult( + lei=lei, + name=(entity.get("legalName") or {}).get("name"), + status=entity.get("status"), + address={"lines": legal.get("addressLines"), "city": legal.get("city"), + "country": legal.get("country"), "postal": legal.get("postalCode")}, + source="gleif", + ) + f = _today() + res.provenance += [ + Provenance("lei", "gleif", 0.99, 3, f), + Provenance("name", "gleif", 0.97, 3, f), + ] + return res + + +# ── Brønnøysundregistrene (NO) — open API (NLOD). VERIFY terms. ───────────── +async def brreg_by_orgnr(orgnr: str) -> CompanyResult | None: + url = f"https://data.brreg.no/enhetsregisteret/api/enheter/{orgnr}" + async with _client() as c: + r = await c.get(url) + if r.status_code != 200: + return None + d = r.json() + res = CompanyResult( + org_nr=orgnr, market="NO", + name=d.get("navn"), + status="dissolved" if d.get("konkurs") else "active", + address={"raw": d.get("forretningsadresse")}, + source="brreg", + ) + f = _today() + res.provenance += [ + Provenance("name", "brreg", 0.97, 3, f), + Provenance("status", "brreg", 0.95, 3, f), + Provenance("address", "brreg", 0.95, 3, f), + ] + return res + + +# ── Companies House (UK) — free API, key required. VERIFY terms. ──────────── +async def companies_house_by_number(number: str) -> CompanyResult | None: + key = os.getenv("COMPANIES_HOUSE_API_KEY") + if not key: + # No key -> deliver honestly as unavailable, never fabricate. + return CompanyResult(org_nr=number, market="GB", source="companies_house", + provenance=[Provenance("name", "companies_house", 0.0, 0, _today())]) + url = f"https://api.company-information.service.gov.uk/company/{number}" + async with _client() as c: + r = await c.get(url, auth=(key, "")) + if r.status_code != 200: + return None + d = r.json() + res = CompanyResult( + org_nr=number, market="GB", + name=d.get("company_name"), + status=d.get("company_status"), + address={"raw": d.get("registered_office_address")}, + source="companies_house", + ) + f = _today() + res.provenance += [ + Provenance("name", "companies_house", 0.97, 3, f), + Provenance("status", "companies_house", 0.95, 3, f), + ] + return res + + +# Registries that require verified terms / keys before bulk use. +# Implement as official-API connectors only; never bypass. Until wired they +# return level-0 so the field is delivered honestly as "ej tillgänglig". +async def prh_by_businessid(business_id: str) -> CompanyResult | None: # FI — VERIFY + return CompanyResult(org_nr=business_id, market="FI", source="prh", + provenance=[Provenance("name", "prh", 0.0, 0, _today())]) + + +async def cvr_by_number(cvr: str) -> CompanyResult | None: # DK — VERIFY (Virk terms) + return CompanyResult(org_nr=cvr, market="DK", source="cvr", + provenance=[Provenance("name", "cvr", 0.0, 0, _today())]) + + +CONNECTORS = { + "gleif": gleif_by_lei, + "brreg": brreg_by_orgnr, + "companies_house": companies_house_by_number, + "prh": prh_by_businessid, + "cvr": cvr_by_number, +} diff --git a/services/api/sources/sources.yaml b/services/api/sources/sources.yaml new file mode 100644 index 0000000..bdc10a7 --- /dev/null +++ b/services/api/sources/sources.yaml @@ -0,0 +1,99 @@ +# 0pt data engine — sources registry (docs/06). Drives the router, /sources API, +# and the public sourcing-map. access: api | bulk | authorized_web | licensed. +# VERIFY current access/licence/robots per source before relying — terms change. +sources: + - source_id: gleif + name: GLEIF (LEI) + market: global + access: api + cadence: daily + licence: "open (CC0) — VERIFY" + entities: [company] + fields: [lei, legal_name, address, status] + base_url: https://api.gleif.org + robots_ok: true + notes: "cross-border key" + + - source_id: brreg + name: Brønnøysundregistrene (Enhetsregisteret) + market: NO + access: api + cadence: daily + licence: "NLOD open — VERIFY" + entities: [company] + fields: [name, status, address, nace] + base_url: https://data.brreg.no + robots_ok: true + notes: "open API + bulk download" + + - source_id: companies_house + name: Companies House + market: GB + access: api + cadence: daily + licence: "free, API key required — VERIFY" + entities: [company, officers, financials] + fields: [name, status, address, officers] + base_url: https://api.company-information.service.gov.uk + robots_ok: true + notes: "key via env COMPANIES_HOUSE_API_KEY" + + - source_id: prh + name: PRH avoindata (Trade Register) + market: FI + access: api + cadence: daily + licence: "CC-BY — VERIFY" + entities: [company] + fields: [name, status] + base_url: https://avoindata.prh.fi + robots_ok: true + notes: "connector pending verification" + + - source_id: cvr + name: CVR / Virk + market: DK + access: api + cadence: daily + licence: "terms apply (Virk) — VERIFY" + entities: [company] + fields: [name, status] + base_url: https://datacvr.virk.dk + robots_ok: true + notes: "connector pending verification" + + - source_id: bolagsverket + name: Bolagsverket + market: SE + access: licensed + cadence: daily + licence: "restricted for bulk — partner/licensed route" + entities: [company] + fields: [name, status] + base_url: https://bolagsverket.se + robots_ok: true + notes: "no bulk scrape; licensed/partner only" + + - source_id: rdw + name: RDW open vehicle data + market: NL + access: bulk + cadence: daily + licence: "open — VERIFY" + entities: [vehicle] + fields: [make, model, year, specs] + base_url: https://opendata.rdw.nl + robots_ok: true + notes: "technical/fleet vehicle data, not owner PII" + + - source_id: opensanctions + name: OpenSanctions + market: global + access: bulk + cadence: daily + licence: "open — VERIFY commercial terms" + entities: [company, person] + fields: [sanction, pep] + base_url: https://www.opensanctions.org + robots_ok: true + notes: "screening/diligence enrichment" diff --git a/services/api/workers/__init__.py b/services/api/workers/__init__.py new file mode 100644 index 0000000..1bb7ead --- /dev/null +++ b/services/api/workers/__init__.py @@ -0,0 +1 @@ +# 0pt data engine — workers package (router + pipeline). diff --git a/services/api/workers/__pycache__/__init__.cpython-314.pyc b/services/api/workers/__pycache__/__init__.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..786093f98784523af560ed5b46184f681a58451a GIT binary patch literal 164 zcmdPqH<# literal 0 HcmV?d00001 diff --git a/services/api/workers/__pycache__/adaptive_router.cpython-314.pyc b/services/api/workers/__pycache__/adaptive_router.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..097062a75459d40535cecb16e59b5c59aeb71c81 GIT binary patch literal 1273 zcmZ8g%}*Og6rbI-jj@9vDHH`rGfEJ}D%g&cmMV&(x(+1>0%U+(YFLeT$M!bs-PO!2 z6iz<$z>z}_Tq<#^dh4;rs(*p29$L+@$)z_}v`UVBvxZddlV;wV_qFf$-rElfBRK@L zKK<~EZ!Cmf>c9}#2Kd}2xKHX19Z{}) zR4nB-xeJkD6Gxmi8E1jR9>BBdIUeNQ0B&_N;f!XEtPKh%xfxhxuBat&y2P~jKL<9O_*K!4-?!} zrbGD)CY1%{s0?Q^=E3ps+p@kxfIQO-d521&KXgYusXeIg3g}v0cre_0%aiT6QIEaF zN+cI)6xG3a0J9KvKu|vnAwM|N%O$Y~OHzuC)HU0_>?yNtJ_Xq);0P^r(Q9NH_x{w^ zS*@?x`D*gI04qvxYeCZ`?({_k9$i#Vl=|8N-f<_^P=PXi2%z!VQ!R^n;MP&D}nFlDuvEE;?qM$F^ z>_HHJ53~u&Nf5f|RsQC$`CC8dZ=E&I56`_{@|9ky!l%KuQOv4o+jhdh_4KAv|EgK= z5Voy`Y#V+b5OTKNjA3rNi5l0SpwYL@z4XslYp_<^HmPo`C1pUlx4}Ir_me@n*@vCb zjeYir&w@qITZC}E>05?j{AH$$)ZaN|-0f#kdiZ4NbfuS@`7M?0tm&0e64VfJsJv58Z9_U`%h-t7JF=?_ysKI;`$y7|``m~?{w EACFCB_y7O^ literal 0 HcmV?d00001 diff --git a/services/api/workers/__pycache__/pipeline.cpython-314.pyc b/services/api/workers/__pycache__/pipeline.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..64df536c00cbc6c528b2e59e94543bbd86c491c1 GIT binary patch literal 5107 zcmbUlTWlN0agWEhC_W@oZ;59~rbJn$WXn+$OHoYAvcxE1Ou1rA^V;3F^2l5E`^=_97q&ihSg!m175vfuiW_ zk*612wCDmiJG(nOyE8j8JA1pVrDg0gxz~L!C%M&4@)3 zHHKI!pc$qm+Km9$1hm6CNvGg+KtF7d3<|Cd7>7-gY1k~8hfAcAVT)u@`gMWQVXI_y zBUaBEd=zVBO+KnaBiUH<2|HWjf%RBRC1Oh}QOYhkSS#Q*1utXmO~~t*y=5N7>3Ak4 zCDS}3Bn2*&Zpmn1S?^kV-~*iHW6oo{LFoEHY_6HPg#*r#U>sj3>ENoJnw!$Ot%n zn&YE&t)MI=ndlW`767%7Tj78AuB6US*m;Nq}zkN~(c zfd+ZmoL!Hh>w!sen9=k=Mrlw1 zDM_}T#F$G&PR>NqIKgANI;IRKIWaPk&WIct;WJYwIV@WeDK0rqI-mxQHX#+|Bjaft znTq0PI7!w7pO8%t?d|IBe5i9zx2zdDC>uKqrF+&HPsXHgfL7a?R8my16DMTzp~1(W zh#VapJajZh!FiI86g_-2G;cQpGKWHFOw;ongyPUwhmKGa6icxhmWEHu>N;pRZ2jjN zAeBwZiE@!-T&7_$nSO?wk@c8^ljLWeTV!Xx$pq6p_frJ2=s}FBs3em08)g5%{_t^; z!(v!WPNn&9A}LK|PKGnd-JOE8J1Pia7(AVfabh?sB*W8b5FdzD1M>+pvLzBJ9s?JN z;3{BS3!m5sU=H0y2FiYmfW?6|QZ2=lD(gF_^xg(46eIGe1>{E05qZYb0|dZE8;`oev>fs3^hkTrZb}^&?BqaG>33i%72+dZ6EdL3 z_QTf*pZNhgNrCO@6blNqQ@+csPC(%1;xF*$r}0N~YN2f@n$h(sM853Ynt)!{prPnB1jy&0l)S;dFm&BeyXxAp)Rc31ZrVIqqi3VHX=xzmYQ1S|%^F*wH|yMa z-LP}jTh~2t53qi=!e31-QvP!7`D(y^N%>vc zw+MF0LisDTmr4x;cTogZ68L`^Gng`~Ef_Pl6ETfSQ~+DXWu&QBf z0KzT+UJ6%0Rj^`3Q;Ov&U?&;a0f3m_K1zR!0AhYnEjK9DN>J&<)j+ky`bdXjegD!z zh<6n2j~ZRKIT$Y%7WvS=qUH8dEydmh%&P;dWhoyeXu_N9?!G)UV_}ttYRfF$UZSpn zkU)p-zq(eX-?TPZ5?MP8dngn^>qCOB0}1+&#*PN5!$2VzLfZT7C1lu;M=*woRVpQ9 zP*)O6V@NQEjCOQb20kSr6JXZ>v(%!nNu_`CPpU-VmAcJdDU^mv*YyONsY4@hOrg?I zi#m6s-bVKTLdSHYwvBNWWA(=jtl|DVSR?ck^V(Ke6Z8{1+g4aJ^b-@?R@f5gC+4=T zuombi0dHHeqvO_$#)mSR!&Zb&{e84%8v!(fni1HU2{izfU}ZJodXs%QOAaSBS#B6Ci^+=DC!vEf#3)^wyqYUd)`4PQLr+Isv!qx93m=_ zS11!4;U;x6A)Cs7Q(u|d*RG_41zMpzRJJbfZbe=aYm^dJ2M}8h=U29&FO}1Mxo`6H zFW9a^>_RMw#*_t7Br*b0Nf z>wDLuxSPsaa{M7?x9N>Uk(Td~0;Km~Iok#dWRAHL_z4M z4#+x@eFn`zf;|F$OwN>rJ73u~6@x-JDWPzmGKKrp z)R*9R?u>x@Y(yGS;~x$!6@nOo!`m%Ad*&j9J8)+GuMiCDDRH;!tywn-LJv|YH7!Z5#3 zMvFeLwn@DM@*tMl#%)VmwX3mk&)IdpgWwZXQHLW%9cqd?q%mGRWzJC=ukzH7y?((}98Yi)Jv(dzj$V4uX%%=Ej0sfr-d;V9a?}>2y=(!5Nllox` zfJ#ET8F5(UF${Z;Z{*B2r@avf;5Wx z1PghFMv)hH1FOw;gx94drYVw3LlL&B?@jOWk;T3X?pNGzd5gy4s-(bAq-cVZHK(F7 zEyN_HEpf6o4tY8F87INJMwGSWtt3|#A3+i2(G-y;Fegr=Q*rPdZv6>jjh;?KHat=` zU@itpl<0NJG(4tc-FPY;l`vdD2$Q>6$slFZ;F%aFkOz};y0RWpe2$OfRzl$=ppAfC z1dwno8zT{tgGC}TJrzA8Yr%IUK$AT2Z62&5o#Dv?Bs1l8VTMd2R{$o_RW_~f9FsVS z4-#;I0AgOU79K1hGI?9UmeZ1w{3>a%OoPO@gAjpd#WV0Alxe7xV~|b7ry+D2ikgy* zsy0ND9eO;>*77to3;u5?Iz0vtLu6NC5PV9p)O@x3Le2S_rNA=IRqwr7-IuNIyC_{8 zpBq}WyB5aJO};d_R0m@fdvo@#i`)%+?_40?*!p_gkJ^@>%r!o6vvDBXIPgx*+~6n8 zp1Gm?j;6Us@90r^&AdKubG|t9{LEs-Qd!Q{Jg;4~H7uUa**x>wywx^Oe^g#MKk!lg zjs?@2hIY7D8PC#8j_JNx)4ecoSC5>PFLf<;UFf;c^GZ+NS$i({QZQe|EP0ka7n^?F z`ey6O@Vl`qlb0s*RrQPFh3WIt%SISzd$Vn2;sg3a^Hp=cs(GdB_x-==&v_nwyZ>VJ z_1V{E-|oNWc@zd-i(P9QeYgBd!{vrt+i3RdC$4+O)=awQiaSW>s$8=mSJmR4?+-5w z$j<89wWy)-7LA-$YYnJj$3pjYW6iA=kZ`RX?QEGF`l!@7Ppw+Zt`ncnm8Y|(GCAw% zdHUlL>nhWK zoUC|%FkH5X*z1A{t&avP)KQtG}+7GQ-9SberP5t2LQuV82H>|tnhCV(% z_Cn_D(QCv3b5(sgSKmA0yAS>LTQ^*fe{kx$?MODBm>XWxBa?l>xsX|mEj_q=IB#_1 zEoEm17ELb)v&|2$Oy@0Ci)~p?_lj{%M?1}HMr0~mcxbKEV9~>?Rj2==7u7!a)oLYk z2T=}pzNX>jfi)Vie|}*>EhE&uFPf2Kkh&+P!LFY3R|X!YF4gV_c)7wFaO*EOQ-M13 z<(*oBdv&D0%Led0gTD%@_ubAwSL^#-2OFSq#bFQhXs>wm1h)G-q3dcL1rx8jO9P#z ztL-%D>7WSSSqga2TQ1i|NI-}uiF-#=>Z_dm90B(=fFq}4fS4nZ@eKSVAg^_DoydA= zMu2}mn8Z!lgyEeZCx1{1VZW?!X^TGKT>R&QbwIfBbX8uma%J(?fG--4LVP*RMIbUr z94^Ej5`|>j`j3LbmXwI3_#xcbPXUGUnmbVNQ)Ay0{LV^o{WuCE6d4Ty_zLt*QFk<2 zO8cc5QB9wriqBBZU9FDN+(rPtv><99oSV)x-#foMuQkr^pKpi5Y}U+~vliwyq6rQB Nhq&v6*P!zL`!_k2a#;WX literal 0 HcmV?d00001 diff --git a/services/api/workers/__pycache__/run.cpython-314.pyc b/services/api/workers/__pycache__/run.cpython-314.pyc new file mode 100644 index 0000000000000000000000000000000000000000..f903945b62147ea01f97c34a15a01bd80c321182 GIT binary patch literal 1275 zcmaJ=U1%It6h3!mc4s^JF&MDIA4)>_8A6(91}5Q_Nbxzio%lLzj--~Bn~e&^ip zzBD{!0ot|6#hn*4fCp-loADTsqmK~uzy=8out}XPYf_t1SgY$&&qhNUIO_Gna#0qi zz&4J9T{sTy5ovCg?Bc>Z0L=-2GHmK=Pv;jr8K+^>oQC=S!aR8lFpgHqTGhPrll6AY z-(#GzNb+tH2a&ANdPEZ*H#@!z;)u#tAZU|&K_sY0-|w&vqwyAvc@y1s;)}Q%H7W1l zwrnxWkHe%Lc#)(@kg#?Tu^K%Wa~j2$X&@=vO=7`Rq_Zn^Dr4n{6A?t)ZKlG}cIi2B)ak|SYBy+}v8p5;@S-S|o}wyNac<%f7vuOU-G0h|j{rJO zkID#oumKHi;S($#u9ab(w1`b?ZH`o+svoGcX^}J8;ZbU=#u2*(7`M0;^U#w*g*=%q zo;~fnB^Vcu2*NmWwgcJfTy#3Y>??_!^^(NF%}asLgySWF)2Htsw_4Io2VJ+%nz=3? zz=*@x#3*(>_zQIM?4jyzyqOJ0xD3APax)QptkSuU;3Q0;my@Roj!Gc2H6uq)##{vr zvhXz)U`!R@|2-PyWz<-1uVbkaa1?{gHc&~nZW}etE;Qbq)8_lPp9j0RgXP^NJ$C zWRcHSahFLR_@Y`!^)`!AU35iiczpX3M-Gq{5{}BQYH2Cgj(TCyX-lE_Qp(kbO$%;_ zGUt;eEz}SMsIn4bmHd1x53`2 z+s0n`_J=>)zdX10>*)ILL+ibB`^K^B&)*pT&6>Ptln-?nw#nZ@ebuW=LrX7`J5Q0N z8RO2#IO4SAX4kuJ)k;m*ZN|Rqrk3k&bx?{*D48-%Pjn+ci1{;$@kstL^3Rs#n`NNl zMP7)GsgAniB|OguRhQ*jS`6`vx7it9!8KKBaT>daIw9mxGl=oX0#bPZ!+(JFC!Bgz kL_t0r1Y_{}mAm@*ePiI-g{v37EPQ3&GsZA6Vn+a literal 0 HcmV?d00001 diff --git a/services/api/workers/adaptive_router.py b/services/api/workers/adaptive_router.py new file mode 100644 index 0000000..00ca839 --- /dev/null +++ b/services/api/workers/adaptive_router.py @@ -0,0 +1,23 @@ +""" +Adaptive router — chooses among COMPLIANT methods only (docs/06). +Order of preference: official API > bulk > authorized public web > licensed. +There are NO bypass/stealth/proxy-rotation tiers here, by design (CLAUDE.md §1). +""" +from __future__ import annotations + +# market -> ordered list of source_ids to try (compliant connectors only) +MARKET_SOURCES: dict[str, list[str]] = { + "NO": ["brreg", "gleif"], + "GB": ["companies_house", "gleif"], + "FI": ["prh", "gleif"], + "DK": ["cvr", "gleif"], + "SE": ["gleif"], # Bolagsverket bulk is licensed; SE company core via GLEIF + licensed/partner +} + +# Access rank: lower = preferred (cheapest compliant path first). +ACCESS_RANK = {"api": 0, "bulk": 1, "authorized_web": 2, "licensed": 3} + + +def plan_for_market(market: str) -> list[str]: + """Return the ordered, compliant source plan for a market.""" + return MARKET_SOURCES.get(market, ["gleif"]) diff --git a/services/api/workers/pipeline.py b/services/api/workers/pipeline.py new file mode 100644 index 0000000..8b4f131 --- /dev/null +++ b/services/api/workers/pipeline.py @@ -0,0 +1,89 @@ +""" +Production pipeline (docs/06): sourcing -> extraction -> structuring -> +enrichment/fusion -> QA. Compliant connectors only; every field gets provenance; +rows below the order's confidence target are dropped (deliver fewer, never fabricate). +""" +from __future__ import annotations + +from typing import Any + +from rapidfuzz import fuzz + +from sources.connectors import ( + CompanyResult, + brreg_by_orgnr, + companies_house_by_number, + gleif_by_lei, +) +from workers.adaptive_router import plan_for_market + +# Minimal built-in seed so a job is demonstrable without customer input. +DEMO_SEEDS: dict[str, list[str]] = { + "NO": ["923609016"], # brreg orgnr (example) + "GB": ["00000006"], # companies house number (example) +} + + +async def _fetch(source_id: str, key: str) -> CompanyResult | None: + if source_id == "gleif": + return await gleif_by_lei(key) + if source_id == "brreg": + return await brreg_by_orgnr(key) + if source_id == "companies_house": + return await companies_house_by_number(key) + return None + + +async def run_job(order_id: str, config: dict[str, Any]) -> dict[str, Any]: + markets: list[str] = config.get("markets", []) or ["NO"] + threshold = float(config.get("confidence", 92)) / 100.0 + seeds: dict[str, list[str]] = config.get("seeds") or DEMO_SEEDS + + records: list[dict[str, Any]] = [] + per_market: dict[str, int] = {} + + # 1) sourcing + 2) extraction + for market in markets: + keys = seeds.get(market, []) + plan = plan_for_market(market) + got = 0 + for key in keys: + for source_id in plan: + try: + res = await _fetch(source_id, key) + except Exception: + res = None + if res and res.name: + records.append({ + "market": market, "name": res.name, "status": res.status, + "lei": res.lei, "org_nr": res.org_nr, "source": res.source, + "provenance": [p.__dict__ for p in res.provenance], + "confidence": max((p.confidence for p in res.provenance), default=0.0), + }) + got += 1 + break # first compliant source that answers wins + per_market[market] = got + + # 3) structuring — dedup by fuzzy name within market + deduped: list[dict[str, Any]] = [] + for r in records: + dup = next((d for d in deduped if d["market"] == r["market"] and fuzz.ratio(d["name"], r["name"]) > 92), None) + if dup: + dup["confidence"] = min(0.99, dup["confidence"] + 0.1) # corroboration bump + else: + deduped.append(r) + + # 4) enrichment/fusion already folded into confidence above. + # 5) QA — drop rows below the order's confidence threshold. + delivered = [r for r in deduped if r["confidence"] >= threshold] + dropped = len(deduped) - len(delivered) + avg_conf = round(sum(r["confidence"] for r in delivered) / len(delivered), 3) if delivered else 0.0 + + return { + "stage": "qa", "pct": 100, "state": "done", + "metrics": { + "rows": len(delivered), "dropped_below_threshold": dropped, + "per_market": per_market, "avg_confidence": avg_conf, + }, + "records": delivered, + } diff --git a/services/api/workers/run.py b/services/api/workers/run.py new file mode 100644 index 0000000..8c40927 --- /dev/null +++ b/services/api/workers/run.py @@ -0,0 +1,21 @@ +""" +Worker entrypoint. In production this drains a queue of order jobs and runs the +compliant pipeline. For now it exposes a CLI to run a single job locally. + python -m workers.run +""" +from __future__ import annotations + +import asyncio +import sys + +from workers.pipeline import run_job + + +async def _main() -> None: + order_id = sys.argv[1] if len(sys.argv) > 1 else "demo" + result = await run_job(order_id, {"markets": ["NO", "GB"], "confidence": 90}) + print(result["metrics"]) + + +if __name__ == "__main__": + asyncio.run(_main())