diff --git a/services/api/workers/marketmute/README-VEHIQ.md b/services/api/workers/marketmute/README-VEHIQ.md new file mode 100644 index 0000000..08629eb --- /dev/null +++ b/services/api/workers/marketmute/README-VEHIQ.md @@ -0,0 +1,21 @@ +# VehIQ Company Analysis — analysera alla bolag på alla sätt + +Komplement till MarketMute-pipelinen: fordonsbransch-analys av ALLA 1 035 060 svenska bolag. + +## `vehiq_analysis_v2.py` +Två pass över korpuset: +1. Regionala marknadsaggregat per postort (n, omsättningssumma/median) → `vehiq_market_summary.json` (1 718 orter) +2. Per bolag: fordonssrelevans-score (0-100), segment, marknadsposition (oms vs oms-median i orten), finans, kreditrisk, kontakter, LEI, moms + +## Klassificering +- **SNI (där SCB FDB täcker)**: 771xx uthyrning/leasing, 493xx/494xx transportörer, 96010 biltvätt (+60p) +- **Bolagsverkets verksamhetsbeskrivning** (skriven ur SNI-registrering): "bilverkstadsrörelse", "handel med begagnade bilar", "försäljning av nya och begagnade bilar", "bildelar", "däckverkstad", "billackering", "bilglass", "biluthyrning" (+40p) — OBS: SCB JE-exporten saknar 45xx helt (verifierat), därför är nyckelordssvaret primär källa för handel/verkstad +- Fordonsbokföring i balansräkningen (+10p) + +## Leveranser +- `vehiq_company_analysis.jsonl` (1,2 GB) — alla 1 020 860 bolag +- `vehiq_bilbransch.jsonl` — 26 670 fordonsbransch-bolag: transportörer 15 100 · verkstäder 3 677 · bilhandlare 2 493 · uthyrning/leasing 1 326 · reservdelar/däck 865 · motorcykel m fl +- `vehiq_market_summary.json` — 1 718 orter med marknadsaggregat + +## Efterlevnad +Företagskopplade personposter endast (styrelse/undertecknare ur iXBRL, revisorer exkluderade). Inga privatpersonregister. diff --git a/services/api/workers/marketmute/vehiq_analysis.py b/services/api/workers/marketmute/vehiq_analysis.py new file mode 100644 index 0000000..36758d4 --- /dev/null +++ b/services/api/workers/marketmute/vehiq_analysis.py @@ -0,0 +1,203 @@ +#!/usr/bin/env python3 +"""VehIQ Company Analysis: analysera ALLA bolag på ALLA sätt — fordonsrelevans, +segment, regional marknadsposition, finans, tidsserie, kontakter.""" +import json, sys, statistics +from collections import defaultdict + +IN = "/data/0pt/output/marketmute_intelligence.jsonl" # har marketmute + ratios + bolagsverket +SCB = "/data/0pt/output/scb_fdb.jsonl" +KP = "/data/0pt/output/kontaktpersoner_v2.jsonl" +LEI = "/data/0pt/output/gleif_se.jsonl" +OUT = "/data/0pt/output/vehiq_company_analysis.jsonl" +OUT_SEG = "/data/0pt/output/vehiq_bilbransch.jsonl" # endast bilbransch-segmentet + +# --- SNI 2007: fordonsbranschen --- +SNI_CORE = { # direkt fordonsbransch (60p) + "45110": "Bilhandel (detaljhandel personbilar)", "45112": "Bilhandel begagnat", + "45119": "Övrig bilhandel", "45201": "Bilverkstad", "45202": "Bilverkstad", + "45203": "Bilverkstad", "45209": "Övrig reparation", "45200": "Fordonsverkstad", + "45310": "Reservdelar/tillbehör", "45320": "Däckverkstad/däckhandel", + "45400": "Motorcykelhandel/verkstad", "45190": "Övrig fordonshandel", + "45111": "Bilhandel", "45311": "Reservdelar", "45312": "Bilverktyg/tillbehör", +} +SNI_ADJACENT = { # nära fordonssekorn (35p) + "77110": "Biluthyrning", "77120": "Billeasing (personbilar)", "77190": "Övrig fordonsummering", + "49320": "Taxi", "49410": "Godstransport väg", "49420": "Godstransport", "49310": "Busstransport", + "49390": "Övrig landtransport", "52100": "Lager/magasin", "52210": "Transportterminal", + "77340": "Uthyrning vattentransport", "33120": "Reparation maskiner", "96010": "Biltvätt", + "64200": "Växtodling (nej)", "49391": "Övrig transport", +} +SNI_PREFIX_ADJ = ("451", "452", "453", "454", "771", "493", "494", "521", "522") + +KW_STRONG = ["bilhandlare", "bilverkstad", "bilverkstad", "bilverstad", "bilverkstad", + "försäljning av personbilar", "försäljning av bilar", "begagnade bilar", "nya bilar", + "fordonsverkstad", "däck", "bildelar", "bilar och lätta lastbilar", "lastbilar", + "motorcyklar", "fordonsreparation", "bilverkstadsverksamhet"] +KW_MED = ["fordon", "bilhandel", "bilverk", "transportbilar", "flak", "entreprenadmaskiner", + "bussar", "släpvagnar", "verkstadsverksamhet", "service av fordon", "bilglass", + "billackering", "bilplåt", "besiktning"] + +def mangle(w): + try: return w.encode("utf-8").decode("latin-1") + except Exception: return w + +KW_STRONG += [mangle(w) for w in list(KW_STRONG)] +KW_MED += [mangle(w) for w in list(KW_MED)] + +def fix_enc(t): + if not t or "Ã" not in t: return t + try: return t.encode("latin-1").decode("utf-8") + except Exception: return t + +def segment(sni1, verksamhet_l, verksamhet_raw): + if sni1 in ("45110", "45111", "45112", "45119"): return "bilhandlare" + if sni1 in ("45200", "45201", "45202", "45203", "45209"): return "bilverkstad" + if sni1 in ("45310", "45311", "45312", "45320"): return "reservdelar_dack" + if sni1 == "45400": return "motorcykel" + if sni1 in ("77110", "77120", "77190"): return "uthyrning_leasing" + if sni1 in ("49310", "49320", "49390", "49410", "49420"): return "transportor" + v = fix_enc(verksamhet_l) + for w in KW_STRONG: + if w in v: return "bilbransch_ovrig" + return "ovrig" + +# --- PASS 1: ladda sidotabeller --- +print("[1/4] Laddar sidotabeller...", flush=True) +scb_map = {} +for line in open(SCB): + r = json.loads(line) + scb_map[r["orgnr_digits"]] = r +contacts = {} +for line in open(KP): + r = json.loads(line) + contacts[r["orgnr_digits"]] = r +lei_map = {} +for line in open(LEI): + r = json.loads(line) + o = (r.get("orgnr") or "").replace("-", "").strip() + if o: lei_map[o] = r +vies = {} +for path in ("/data/0pt/output/vies_targets.jsonl", "/data/0pt/output/vies_extra.jsonl"): + try: + for line in open(path): + r = json.loads(line) + vies[r["orgnr_fmt"]] = r + except FileNotFoundError: pass +print(f" SCB {len(scb_map):,} · kontakter {len(contacts):,} · LEI {len(lei_map):,} · VIES {len(vies):,}", flush=True) + +# --- PASS 2: marknadsaggregat per kommun×segment --- +print("[2/4] Pass 2: regionala marknadsaggregat...", flush=True) +mkt = defaultdict(lambda: {"n": 0, "oms_sum": 0.0, "oms_list": [], "ek_sum": 0.0}) +n = 0 +for line in open(IN): + rec = json.loads(line) + n += 1 + orgnr = rec.get("orgnr") + if not orgnr: continue + scb = scb_map.get(orgnr.replace("-", "")) + if not scb: continue + kommun = (scb.get("postort") or "").strip().upper() + if not kommun: continue + r = rec.get("finansiella_ratios", {}) + oms = r.get("omsattning") + ek = r.get("eget_kapital") + m = mkt[kommun] + m["n"] += 1 + if oms: m["oms_sum"] += oms; m["oms_list"].append(oms) + if ek: m["ek_sum"] += ek + if n % 300000 == 0: print(f" {n:,}...", flush=True) + +mkt_summary = {} +for kommun, m in mkt.items(): + lst = sorted(m["oms_list"]) + mkt_summary[kommun] = { + "n": m["n"], "oms_sum": m["oms_sum"], + "oms_median": statistics.median(lst) if lst else None, + "ek_sum": m["ek_sum"], + } +del mkt +with open("/data/0pt/output/vehiq_market_summary.json", "w") as f: + json.dump(mkt_summary, f, ensure_ascii=False) +print(f" {len(mkt_summary):,} kommuner/aggregat", flush=True) + +# --- PASS 3: per-bolagsanalys --- +print("[3/4] Pass 3: per-bolagsanalys...", flush=True) +def score_vehicle(scb, verksamhet_raw, r, verksamhet_l): + score = 0; reasons = [] + sni1 = (scb.get("sni1") or "") if scb else "" + if sni1 in SNI_CORE: + score += 60; reasons.append(f"SNI {sni1}: {SNI_CORE[sni1]}") + elif sni1 in SNI_ADJACENT: + score += 35; reasons.append(f"SNI {sni1}: {SNI_ADJACENT[sni1]}") + elif sni1[:3] in SNI_PREFIX_ADJ and sni1: + score += 25; reasons.append(f"SNI-prefix {sni1[:3]}") + for w in KW_STRONG: + if w in verksamhet_l: + score += 25; reasons.append(f"Nyckelord: '{fix_enc(w)}'"); break + for w in KW_MED: + if w in verksamhet_l: + score += 15; reasons.append(f"Nyckelord: '{fix_enc(w)}'"); break + if r.get("fordon_bokfort"): score += 10; reasons.append("Fordonsbokföring i balansräkning") + return min(score, 100), sni1, reasons + +n = 0; core_n = 0 +out = open(OUT, "w") +out_seg = open(OUT_SEG, "w") +for line in open(IN): + rec = json.loads(line) + n += 1 + orgnr = rec.get("orgnr") + if not orgnr: continue + digits = orgnr.replace("-", "") + scb = scb_map.get(digits) + r = rec.get("finansiella_ratios", {}) + mm = rec.get("marketmute", {}) + verksamhet_raw = scb.get("verksamhet") if scb else "" + verksamhet_l = (verksamhet_raw or "").lower() + v_score, sni1, v_reasons = score_vehicle(scb, verksamhet_raw, r, verksamhet_l) + seg = segment(sni1, verksamhet_l, verksamhet_raw) + kommun = (scb.get("postort") or "").strip().upper() if scb else None + oms = r.get("omsattning") + mkt = mkt_summary.get(kommun) if kommun else None + # marknadsposition: omsättning-percentil inom kommunen + position = None + if mkt and oms and mkt["n"] > 3: + if mkt["oms_median"]: + position = round(oms / mkt["oms_median"], 2) # x gånger medianen + contact = contacts.get(digits) + lei = lei_map.get(digits) + vies_rec = vies.get(orgnr) + rec_out = { + "orgnr": orgnr, + "namn": fix_enc((scb or {}).get("namn") or mm.get("namn", "")), + "fordonssrelevans_score": v_score, + "fordonssrelevans_reasons": v_reasons, + "segment": seg, + "sni": (scb or {}).get("sni"), "sni1": sni1, + "jurform": (scb or {}).get("jurform"), + "gatuadress": (scb or {}).get("gatuadress"), + "postnr": (scb or {}).get("postnr"), "postort": (scb or {}).get("postort"), + "marknad": {"kommun": kommun, "n_bolag": (mkt or {}).get("n"), + "oms_sum": (mkt or {}).get("oms_sum"), + "oms_median": (mkt or {}).get("oms_median"), + "oms_vs_median": position}, + "finansiella_ratios": r, + "kreditrisk_klass": r.get("kreditrisk_klass"), + "kreditrisk_score": r.get("kreditrisk_score"), + "kontaktpersoner": (contact or {}).get("personer", []), + "har_vd": bool(contact and contact.get("har_vd")), + "lei": (lei or {}).get("lei"), + "moms_valid": (vies_rec or {}).get("moms_valid"), + "momsnr": f"SE{digits}01", + "cold_call_score": mm.get("cold_call_score"), + "verksamhet": fix_enc(verksamhet_raw or ""), + "antal_ar": r.get("antal_ar"), + } + out.write(json.dumps(rec_out, ensure_ascii=False) + "\n") + if seg != "ovrig" or v_score >= 60: + core_n += 1 + out_seg.write(json.dumps(rec_out, ensure_ascii=False) + "\n") + if n % 200000 == 0: print(f" {n:,} (bilbransch hittills: {core_n:,})", flush=True) +out.close(); out_seg.close() +print(f"[KLAR] {n:,} analyserade -> {OUT}") +print(f" Bilbransch-segmentet (SNI 45xx m fl): {core_n:,} -> {OUT_SEG}") diff --git a/services/api/workers/marketmute/vehiq_analysis_v2.py b/services/api/workers/marketmute/vehiq_analysis_v2.py new file mode 100644 index 0000000..e7445af --- /dev/null +++ b/services/api/workers/marketmute/vehiq_analysis_v2.py @@ -0,0 +1,181 @@ +#!/usr/bin/env python3 +"""VehIQ Company Analysis v2: förstärkta bilbransch-nyckelord + SNI (där SCB täcker).""" +import json, sys +from collections import defaultdict + +IN = "/data/0pt/output/marketmute_intelligence.jsonl" +SCB = "/data/0pt/output/scb_fdb.jsonl" +KP = "/data/0pt/output/kontaktpersoner_v2.jsonl" +LEI = "/data/0pt/output/gleif_se.jsonl" +MKT = "/data/0pt/output/vehiq_market_summary.json" +OUT = "/data/0pt/output/vehiq_company_analysis.jsonl" +OUT_SEG = "/data/0pt/output/vehiq_bilbransch.jsonl" + +SNI_TRANSPORT = { # finns i SCB: transport/uthyrning/lager + "77110": "Biluthyrning", "77120": "Billeasing personbilar", "77190": "Övrig fordonsummering", + "49310": "Busstransport", "49320": "Taxi", "49390": "Övrig personbefordran", + "49410": "Godstransport väg", "49420": "Godstransport (flytt)", + "52100": "Lager/magasin", "52210": "Transportterminaler", "77340": "UthyrningMaskiner", + "96010": "Biltvätt", "33120": "ReparationMaskiner", +} + +def mangle(w): + try: return w.encode("utf-8").decode("latin-1") + except Exception: return w + +# Starka bilbransch-nyckelord (från faktiska Bolagsverket-fraser) +KW_STRONG = [ + "bilverkstadsrörelse", "bilverkstad", "bilverkstad", "bilverkstad.", + "handel med begagnade bilar", "handel med nya och begagnade bilar", + "handel med bilar", "försäljning av begagnade bilar", + "försäljning av nya och begagnade bilar", "försäljning av bilar", + "försäljning av personbilar", "bilhandel", "handel med bildelar", + "bildelar", "biltillbehör", "däckverkstad", "däckhotell", "handelsbod med däck", + "reparation av bilar", "reparation av personbilar", "bilverk", + "billackering", "bilplåt", "bilglass", "bilglass", + "motorcykelhandel", "motorcykelverkstad", "handelsbod med motorcyklar", + "biluthyrning", "billeasing", "uthyrning av personbilar", + "lastbilstrafik", "godstransporter", "buss", "taxiverksamhet", "taxiförmedling", + "service av fordon", "fordonsverkstad", "fordonsreparation", + "besiktning av fordon", "bilbesiktning", +] +KW_MED = [ + "personbilar", "bilar", "fordon", "transportbilar", "motorcykel", + "släpvagnar", "entreprenadmaskiner", "lastbilar", "däck", + "verkstadsverksamhet", "maskinreparation", "flak", +] +KW_STRONG += [mangle(w) for w in list(KW_STRONG)] +KW_MED += [mangle(w) for w in list(KW_MED)] + +def fix_enc(t): + if not t or "Ã" not in t: return t + try: return t.encode("latin-1").decode("utf-8") + except Exception: return t + +print("[1/3] Laddar sidotabeller...", flush=True) +scb_map = {} +for line in open(SCB): + r = json.loads(line) + scb_map[r["orgnr_digits"]] = r +contacts = {} +for line in open(KP): + r = json.loads(line) + contacts[r["orgnr_digits"]] = r +lei_map = {} +for line in open(LEI): + r = json.loads(line) + o = (r.get("orgnr") or "").replace("-", "").strip() + if o: lei_map[o] = r +vies = {} +for path in ("/data/0pt/output/vies_targets.jsonl", "/data/0pt/output/vies_extra.jsonl"): + try: + for line in open(path): + r = json.loads(line) + vies[r["orgnr_fmt"]] = r + except FileNotFoundError: pass +mkt_summary = json.load(open(MKT)) +print(f" SCB {len(scb_map):,} · KP {len(contacts):,} · LEI {len(lei_map):,} · VIES {len(vies):,} · marknad {len(mkt_summary):,}", flush=True) + +def classify(scb, verksamhet_l, r): + score = 0; reasons = [] + sni1 = (scb.get("sni1") or "") if scb else "" + sni_all = (scb.get("sni") or []) if scb else [] + if sni1 in SNI_TRANSPORT: + score += 60; reasons.append(f"SNI {sni1}: {SNI_TRANSPORT[sni1]}") + # sekundär-SNI check (45xx saknas i exporten men kolla alla Ng) + for s in sni_all: + if s[:3] in ("451", "452", "453", "454"): + score += 60; reasons.append(f"SNI {s} (sekundär): bilbransch"); break + strong = None + for w in KW_STRONG: + if w in verksamhet_l: + strong = w; break + if strong: + score += 40; reasons.append(f"Nyckelord: '{fix_enc(strong)}'") + # förfining av segment via nyckelord + if any(x in strong for x in ("verkstad", "reparation", "lackering", "plåt", "glass", "besiktning", "däck")): + seg_hint = "bilverkstad" + elif any(x in strong for x in ("bildelar", "biltillbehör", "däck")): + seg_hint = "reservdelar_dack" + elif any(x in strong for x in ("uthyrning", "leasing")): + seg_hint = "uthyrning_leasing" + elif any(x in strong for x in ("taxi", "buss", "godstransport", "lastbilstrafik")): + seg_hint = "transportor" + elif any(x in strong for x in ("motorcykel",)): + seg_hint = "motorcykel" + else: + seg_hint = "bilhandlare" + else: + seg_hint = None + for w in KW_MED: + if w in verksamhet_l: + score += 20; reasons.append(f"Nyckelord: '{fix_enc(w)}'"); break + if r.get("fordon_bokfort"): + score += 10; reasons.append("Fordonsbokföring i balansräkningen") + return min(score, 100), sni1, reasons, seg_hint + +print("[2/3] Per-bolagsanalys...", flush=True) +n = 0; core_n = 0 +seg_count = defaultdict(int) +out = open(OUT, "w") +out_seg = open(OUT_SEG, "w") +for line in open(IN): + rec = json.loads(line) + n += 1 + orgnr = rec.get("orgnr") + if not orgnr: continue + digits = orgnr.replace("-", "") + scb = scb_map.get(digits) + r = rec.get("finansiella_ratios", {}) + mm = rec.get("marketmute", {}) + verksamhet_raw = scb.get("verksamhet") if scb else "" + verksamhet_l = (verksamhet_raw or "").lower() + v_score, sni1, v_reasons, seg_hint = classify(scb, verksamhet_l, r) + # segment: SNI först, nyckelord annars + if sni1 in ("77110", "77120", "77190"): seg = "uthyrning_leasing" + elif sni1 in ("49310", "49320", "49390", "49410", "49420"): seg = "transportor" + elif sni1 == "96010": seg = "bilverkstad" + elif seg_hint: seg = seg_hint + else: seg = "ovrig" + seg_count[seg] += 1 + kommun = (scb.get("postort") or "").strip().upper() if scb else None + oms = r.get("omsattning") + mkt = mkt_summary.get(kommun) if kommun else None + position = round(oms / mkt["oms_median"], 2) if (mkt and oms and mkt.get("oms_median") and mkt["n"] > 3) else None + contact = contacts.get(digits) + lei = lei_map.get(digits) + vies_rec = vies.get(orgnr) + rec_out = { + "orgnr": orgnr, + "namn": fix_enc((scb or {}).get("namn") or mm.get("namn", "")), + "fordonssrelevans_score": v_score, + "fordonssrelevans_reasons": v_reasons, + "segment": seg, + "sni": (scb or {}).get("sni"), "sni1": sni1, + "jurform": (scb or {}).get("jurform"), + "gatuadress": (scb or {}).get("gatuadress"), + "postnr": (scb or {}).get("postnr"), "postort": (scb or {}).get("postort"), + "marknad": {"kommun": kommun, "n_bolag": (mkt or {}).get("n"), + "oms_sum": (mkt or {}).get("oms_sum"), + "oms_median": (mkt or {}).get("oms_median"), + "oms_vs_median": position}, + "finansiella_ratios": r, + "kreditrisk_klass": r.get("kreditrisk_klass"), + "kreditrisk_score": r.get("kreditrisk_score"), + "kontaktpersoner": (contact or {}).get("personer", []), + "har_vd": bool(contact and contact.get("har_vd")), + "lei": (lei or {}).get("lei"), + "moms_valid": (vies_rec or {}).get("moms_valid"), + "momsnr": f"SE{digits}01", + "cold_call_score": mm.get("cold_call_score"), + "verksamhet": fix_enc(verksamhet_raw or ""), + } + out.write(json.dumps(rec_out, ensure_ascii=False) + "\n") + if seg != "ovrig" or v_score >= 60: + core_n += 1 + out_seg.write(json.dumps(rec_out, ensure_ascii=False) + "\n") + if n % 200000 == 0: print(f" {n:,} (fordonsbransch: {core_n:,})", flush=True) +out.close(); out_seg.close() +print(f"[KLAR] {n:,} analyserade -> {OUT}") +print(f" Fordonsbransch: {core_n:,} -> {OUT_SEG}") +print(" Segment:", dict(sorted(seg_count.items(), key=lambda x: -x[1])))