61 lines
1.6 KiB
Python
61 lines
1.6 KiB
Python
"""
|
|
0pt data engine — batch-runner för Bolagsverket iXBRL-extraktion.
|
|
|
|
Kör hela korpusen (ytterzip -> innerzip -> XHTML -> fakta) och skriver JSONL
|
|
(en rad per bolag). Progress till stderr, fortsättningsbar via manifest över
|
|
redan körda ytterzip-filer.
|
|
|
|
Användning:
|
|
python workers/ixbrl_batch.py /data/0pt/landing/bolagsverket/arsredovisningar \
|
|
/data/0pt/output/bolagsverket_arsredovisningar.jsonl
|
|
"""
|
|
from __future__ import annotations
|
|
|
|
import os
|
|
import sys
|
|
import time
|
|
|
|
from sources.ixbrl import iter_corpus, to_jsonl
|
|
|
|
|
|
def main():
|
|
base_dir = sys.argv[1]
|
|
out_path = sys.argv[2]
|
|
|
|
os.makedirs(os.path.dirname(out_path), exist_ok=True)
|
|
|
|
n = 0
|
|
n_facts = 0
|
|
n_err = 0
|
|
t0 = time.time()
|
|
last = t0
|
|
|
|
with open(out_path, "a", encoding="utf-8") as out:
|
|
for rec in iter_corpus(base_dir):
|
|
out.write(to_jsonl(rec) + "\n")
|
|
n += 1
|
|
if rec.get("facts"):
|
|
n_facts += rec["fact_count"]
|
|
else:
|
|
n_err += 1
|
|
|
|
now = time.time()
|
|
if now - last >= 30:
|
|
rate = n / (now - t0)
|
|
sys.stderr.write(
|
|
f"# bolag={n} fakta={n_facts} fel={n_err} "
|
|
f"{rate:.2f} bolag/s ({n/(now-t0)/3600:.0f} avklarad skattning 140k)\n"
|
|
)
|
|
sys.stderr.flush()
|
|
last = now
|
|
|
|
dt = time.time() - t0
|
|
sys.stderr.write(
|
|
f"# KLAR bolag={n} fakta={n_facts} fel={n_err} tid={dt:.0f}s "
|
|
f"({n/dt:.2f} bolag/s)\n"
|
|
)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|