#!/usr/bin/env python3 """Slutgiltig iXBRL-extraktor: lxml-baserad, taxonomi-generisk, ~6 sek/zip, ~2.4h för 1421 zips.""" import sys, os, zipfile, io, json, time, lxml.etree as ET def _tag(el): try: return str(el.tag).lower() except: return "" def parse_one(tree): facts = [] for el in tree.iter(): tag = _tag(el) if tag in ("ix:nonfraction", "ix:nonnumeric"): name = el.get("name", "") val = (el.text or "").strip() dec = el.get("decimals") sc = el.get("scale") facts.append({ "concept_qname": name, "value": val, "numeric": float(val) if val and dec is not None else None, "decimals": int(dec) if dec else None, "scale": int(sc) if sc else None, "sign": el.get("sign"), "contextRef": el.get("contextref", "") or el.get("contextRef", "") or "", "unitRef": el.get("unitref", "") or el.get("unitRef", "") or "", "precision": el.get("precision"), }) contexts = {} for el in tree.iter(): tag = _tag(el) if tag == "xbrli:context": cid = el.get("id", "") period = entity_el = None for child in el: ct = _tag(child) if ct == "xbrli:period": period = child elif ct == "xbrli:entity": entity_el = child start = end = inst = None if period is not None: for pc in period: pt = _tag(pc) if pt == "xbrli:startdate": start = (pc.text or "").strip() elif pt == "xbrli:enddate": end = (pc.text or "").strip() elif pt == "xbrli:instant": inst = (pc.text or "").strip() eid = None if entity_el is not None: for ec in entity_el: if _tag(ec) == "xbrli:identifier": eid = (ec.text or "").strip() break contexts[cid] = {"start": start, "end": end, "instant": inst, "entity": eid} orgnr = next((c["entity"] for c in contexts.values() if c["entity"]), None) for f in facts: c = contexts.get(f.pop("contextRef"), {}) f["period_start"] = c.get("start") f["period_end"] = c.get("end") f["instant"] = c.get("instant") f["entity_identifier"] = c.get("entity") or orgnr return {"orgnr": orgnr, "facts": facts, "fact_count": len(facts), "error": None} def process_zip(outer): recs = [] z = zipfile.ZipFile(outer) for inner in sorted(z.namelist()): if not inner.lower().endswith(".zip"): continue ib = z.read(inner) iz = zipfile.ZipFile(io.BytesIO(ib)) for xn in sorted(iz.namelist()): if not xn.lower().endswith((".xhtml",".xhtm",".html",".htm")): continue try: tree = ET.fromstring(iz.read(xn), parser=ET.HTMLParser()) recs.append(parse_one(tree)) except Exception as ex: recs.append({"orgnr": None, "facts": [], "fact_count": 0, "error": str(ex)}) z.close() return recs BASE = sys.argv[1] if len(sys.argv)>1 else "/data/0pt/landing/bolagsverket/arsredovisningar" OUT = sys.argv[2] if len(sys.argv)>2 else "/data/0pt/output/chunks/merged.jsonl" DONE = os.path.join(os.path.dirname(OUT), "done_zips.txt") done = set() if os.path.exists(DONE): done = set(open(DONE).read().splitlines()) outer_zips = sorted(os.path.join(r,f) for r,_,fs in os.walk(BASE) for f in fs if f.lower().endswith(".zip")) remaining = [(z, i) for i, z in enumerate(outer_zips) if z not in done] sys.stderr.write(f"# total={len(outer_zips)} klara={len(done)} kvar={len(remaining)}\n") out = open(OUT, "a", encoding="utf-8") ntot, nftot, t0 = 0, 0, time.time() for zi, (outer, _) in enumerate(remaining): tz = time.time() recs = process_zip(outer) nz = len(recs); nzf = sum(r.get("fact_count", 0) for r in recs) for rec in recs: out.write(json.dumps(rec, ensure_ascii=False, default=str) + "\n") ntot += nz; nftot += nzf done.add(outer) with open(DONE, "w") as d: d.write("\n".join(sorted(done))) dt = max(time.time() - t0, 0.1) sys.stderr.write( f"[{len(done)}/{len(outer_zips)}] {os.path.basename(outer)}: {nz} bolag {nzf} fakta " f"({nz/max(time.time()-tz,0.1):.1f} b/s, totalt {ntot/max(dt,1):.1f} b/s)\n") sys.stderr.flush(); out.flush() out.close() dt = time.time() - t0 sys.stderr.write(f"# KLAR bolag={ntot} fakta={nftot} {ntot/max(dt,1):.1f} b/s\n")