diff --git a/services/api/workers/ixbrl_batch.py b/services/api/workers/ixbrl_batch.py new file mode 100644 index 0000000..a01d0cd --- /dev/null +++ b/services/api/workers/ixbrl_batch.py @@ -0,0 +1,60 @@ +""" +0pt data engine — batch-runner för Bolagsverket iXBRL-extraktion. + +Kör hela korpusen (ytterzip -> innerzip -> XHTML -> fakta) och skriver JSONL +(en rad per bolag). Progress till stderr, fortsättningsbar via manifest över +redan körda ytterzip-filer. + +Användning: + python workers/ixbrl_batch.py /data/0pt/landing/bolagsverket/arsredovisningar \ + /data/0pt/output/bolagsverket_arsredovisningar.jsonl +""" +from __future__ import annotations + +import os +import sys +import time + +from sources.ixbrl import iter_corpus, to_jsonl + + +def main(): + base_dir = sys.argv[1] + out_path = sys.argv[2] + + os.makedirs(os.path.dirname(out_path), exist_ok=True) + + n = 0 + n_facts = 0 + n_err = 0 + t0 = time.time() + last = t0 + + with open(out_path, "a", encoding="utf-8") as out: + for rec in iter_corpus(base_dir): + out.write(to_jsonl(rec) + "\n") + n += 1 + if rec.get("facts"): + n_facts += rec["fact_count"] + else: + n_err += 1 + + now = time.time() + if now - last >= 30: + rate = n / (now - t0) + sys.stderr.write( + f"# bolag={n} fakta={n_facts} fel={n_err} " + f"{rate:.2f} bolag/s ({n/(now-t0)/3600:.0f} avklarad skattning 140k)\n" + ) + sys.stderr.flush() + last = now + + dt = time.time() - t0 + sys.stderr.write( + f"# KLAR bolag={n} fakta={n_facts} fel={n_err} tid={dt:.0f}s " + f"({n/dt:.2f} bolag/s)\n" + ) + + +if __name__ == "__main__": + main()