workers: ixbrl_lite — lxml-baserad iXBRL-extraktor (60x snabbare an Arelle)

- 2.3M bolag, 6.3M fakta pa ~2 timmar (vs Arelle ~5 dagar) - HTMLParser med lowercase-attribut (ix:nonfraction/nonnumeric) - contexts + periods + entity ur XBRL-strukturen - taxonomi-generisk: 733 unika concept over hela bastaxonomin
2026-08-30 23:09:06 +02:00
parent f27613a6cd
commit 6a6a96e1c2
1 changed files with 111 additions and 0 deletions
@@ -0,0 +1,111 @@
+#!/usr/bin/env python3
+"""Slutgiltig iXBRL-extraktor: lxml-baserad, taxonomi-generisk, ~6 sek/zip, ~2.4h för 1421 zips."""
+import sys, os, zipfile, io, json, time, lxml.etree as ET
+
+def _tag(el):
+    try: return str(el.tag).lower()
+    except: return ""
+
+def parse_one(tree):
+    facts = []
+    for el in tree.iter():
+        tag = _tag(el)
+        if tag in ("ix:nonfraction", "ix:nonnumeric"):
+            name = el.get("name", "")
+            val = (el.text or "").strip()
+            dec = el.get("decimals")
+            sc = el.get("scale")
+            facts.append({
+                "concept_qname": name,
+                "value": val,
+                "numeric": float(val) if val and dec is not None else None,
+                "decimals": int(dec) if dec else None,
+                "scale": int(sc) if sc else None,
+                "sign": el.get("sign"),
+                "contextRef": el.get("contextref", "") or el.get("contextRef", "") or "",
+                "unitRef": el.get("unitref", "") or el.get("unitRef", "") or "",
+                "precision": el.get("precision"),
+            })
+
+    contexts = {}
+    for el in tree.iter():
+        tag = _tag(el)
+        if tag == "xbrli:context":
+            cid = el.get("id", "")
+            period = entity_el = None
+            for child in el:
+                ct = _tag(child)
+                if ct == "xbrli:period": period = child
+                elif ct == "xbrli:entity": entity_el = child
+            start = end = inst = None
+            if period is not None:
+                for pc in period:
+                    pt = _tag(pc)
+                    if pt == "xbrli:startdate": start = (pc.text or "").strip()
+                    elif pt == "xbrli:enddate": end = (pc.text or "").strip()
+                    elif pt == "xbrli:instant": inst = (pc.text or "").strip()
+            eid = None
+            if entity_el is not None:
+                for ec in entity_el:
+                    if _tag(ec) == "xbrli:identifier":
+                        eid = (ec.text or "").strip()
+                        break
+            contexts[cid] = {"start": start, "end": end, "instant": inst, "entity": eid}
+
+    orgnr = next((c["entity"] for c in contexts.values() if c["entity"]), None)
+    for f in facts:
+        c = contexts.get(f.pop("contextRef"), {})
+        f["period_start"] = c.get("start")
+        f["period_end"] = c.get("end")
+        f["instant"] = c.get("instant")
+        f["entity_identifier"] = c.get("entity") or orgnr
+    return {"orgnr": orgnr, "facts": facts, "fact_count": len(facts), "error": None}
+
+def process_zip(outer):
+    recs = []
+    z = zipfile.ZipFile(outer)
+    for inner in sorted(z.namelist()):
+        if not inner.lower().endswith(".zip"): continue
+        ib = z.read(inner)
+        iz = zipfile.ZipFile(io.BytesIO(ib))
+        for xn in sorted(iz.namelist()):
+            if not xn.lower().endswith((".xhtml",".xhtm",".html",".htm")): continue
+            try:
+                tree = ET.fromstring(iz.read(xn), parser=ET.HTMLParser())
+                recs.append(parse_one(tree))
+            except Exception as ex:
+                recs.append({"orgnr": None, "facts": [], "fact_count": 0, "error": str(ex)})
+    z.close()
+    return recs
+
+BASE = sys.argv[1] if len(sys.argv)>1 else "/data/0pt/landing/bolagsverket/arsredovisningar"
+OUT = sys.argv[2] if len(sys.argv)>2 else "/data/0pt/output/chunks/merged.jsonl"
+DONE = os.path.join(os.path.dirname(OUT), "done_zips.txt")
+
+done = set()
+if os.path.exists(DONE): done = set(open(DONE).read().splitlines())
+outer_zips = sorted(os.path.join(r,f) for r,_,fs in os.walk(BASE) for f in fs if f.lower().endswith(".zip"))
+remaining = [(z, i) for i, z in enumerate(outer_zips) if z not in done]
+
+sys.stderr.write(f"# total={len(outer_zips)} klara={len(done)} kvar={len(remaining)}\n")
+
+out = open(OUT, "a", encoding="utf-8")
+ntot, nftot, t0 = 0, 0, time.time()
+for zi, (outer, _) in enumerate(remaining):
+    tz = time.time()
+    recs = process_zip(outer)
+    nz = len(recs); nzf = sum(r.get("fact_count", 0) for r in recs)
+    for rec in recs:
+        out.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n")
+    ntot += nz; nftot += nzf
+    done.add(outer)
+    with open(DONE, "w") as d: d.write("\n".join(sorted(done)))
+    dt = max(time.time() - t0, 0.1)
+    sys.stderr.write(
+        f"[{len(done)}/{len(outer_zips)}] {os.path.basename(outer)}: {nz} bolag {nzf} fakta "
+        f"({nz/max(time.time()-tz,0.1):.1f} b/s, totalt {ntot/max(dt,1):.1f} b/s)\n")
+    sys.stderr.flush(); out.flush()
+
+out.close()
+dt = time.time() - t0
+sys.stderr.write(f"# KLAR bolag={ntot} fakta={nftot} {ntot/max(dt,1):.1f} b/s\n")