""" 0pt data engine — batch-runner för Bolagsverket iXBRL-extraktion. Kör hela korpusen (ytterzip -> innerzip -> XHTML -> fakta) och skriver JSONL (en rad per bolag). Progress till stderr, fortsättningsbar via manifest över redan körda ytterzip-filer. Användning: python workers/ixbrl_batch.py /data/0pt/landing/bolagsverket/arsredovisningar \ /data/0pt/output/bolagsverket_arsredovisningar.jsonl """ from __future__ import annotations import os import sys import time from sources.ixbrl import iter_corpus, to_jsonl def main(): base_dir = sys.argv[1] out_path = sys.argv[2] os.makedirs(os.path.dirname(out_path), exist_ok=True) n = 0 n_facts = 0 n_err = 0 t0 = time.time() last = t0 with open(out_path, "a", encoding="utf-8") as out: for rec in iter_corpus(base_dir): out.write(to_jsonl(rec) + "\n") n += 1 if rec.get("facts"): n_facts += rec["fact_count"] else: n_err += 1 now = time.time() if now - last >= 30: rate = n / (now - t0) sys.stderr.write( f"# bolag={n} fakta={n_facts} fel={n_err} " f"{rate:.2f} bolag/s ({n/(now-t0)/3600:.0f} avklarad skattning 140k)\n" ) sys.stderr.flush() last = now dt = time.time() - t0 sys.stderr.write( f"# KLAR bolag={n} fakta={n_facts} fel={n_err} tid={dt:.0f}s " f"({n/dt:.2f} bolag/s)\n" ) if __name__ == "__main__": main()