614 lines
23 KiB
Python
614 lines
23 KiB
Python
#!/usr/bin/env python3
|
|
"""Data-vyer för Scrape Console: dataset-register, tabell, postdetalj, export.
|
|
Endast stdlib. Anropas från server.py (routes) och serverar /data-sidan."""
|
|
|
|
import csv, gzip, json, os, time
|
|
|
|
HOME = os.path.expanduser("~")
|
|
SF = os.path.join(HOME, "Documents/SF_Extractor")
|
|
CI = os.path.join(HOME, ".local/share/carinfo")
|
|
COUNT_CACHE = {"ts": 0, "data": {}}
|
|
|
|
|
|
def _files_jsonl_gz(d):
|
|
try:
|
|
return sorted(
|
|
[
|
|
os.path.join(d, f)
|
|
for f in os.listdir(d)
|
|
if f.endswith(".jsonl.gz") and os.path.getsize(os.path.join(d, f)) > 0
|
|
]
|
|
)
|
|
except Exception:
|
|
return []
|
|
|
|
|
|
def _iter_jsonl_gz(path):
|
|
try:
|
|
with gzip.open(path, "rt", errors="replace") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
yield json.loads(line)
|
|
except Exception:
|
|
continue
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _iter_jsonl(path):
|
|
try:
|
|
with open(path, errors="replace") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
try:
|
|
yield json.loads(line)
|
|
except Exception:
|
|
continue
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _iter_csv(path):
|
|
try:
|
|
with open(path, newline="", errors="replace") as f:
|
|
for row in csv.DictReader(f):
|
|
yield row
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _count(it, cap=2000000):
|
|
n = 0
|
|
for _ in it:
|
|
n += 1
|
|
if n >= cap:
|
|
break
|
|
return n
|
|
|
|
|
|
SCRAPER_OF = {"biluppgifter_records": "sf", "sf_spool": "sf", "carinfo_trims": "carinfo",
|
|
"vehicles_clean": "sf", "allabolag": "sf", "hitta": "sf", "ratsit": "sf",
|
|
"bolagsfakta": "sf", "hitta_companies": "sf", "reco_companies": "sf",
|
|
"regnr": "sf", "queue_biluppgifter": "sf", "queue_biluppgifter_crawled": "sf",
|
|
"queue_multi_fordonssok": "sf", "queue_multi_fordonskollen": "sf",
|
|
"queue_multi_carinfo": "sf"}
|
|
DATASETS = [
|
|
{
|
|
"id": "biluppgifter_records",
|
|
"label": "SF · biluppgifter records (extraherat)",
|
|
"kind": "multi-jsonl-gz",
|
|
"dir": os.path.join(SF, "full_data/biluppgifter"),
|
|
"prefix": "records-",
|
|
"desc": "titel, tabeller, json_ld, text per fordonssida",
|
|
},
|
|
{
|
|
"id": "sf_spool",
|
|
"label": "SF · spool (råcaptures, live)",
|
|
"kind": "multi-jsonl-gz",
|
|
"dir": os.path.join(SF, "spool"),
|
|
"prefix": "multi-",
|
|
"desc": "{url, html} per hämtad sida, nyast först",
|
|
},
|
|
{
|
|
"id": "carinfo_trims",
|
|
"label": "car.info · trims/specs",
|
|
"kind": "jsonl",
|
|
"path": os.path.join(CI, "trims.jsonl"),
|
|
"desc": "{url, make, model, specs{...}}",
|
|
},
|
|
{
|
|
"id": "vehicles_clean",
|
|
"label": "SF · vehicles_clean.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "cleaned_allabolag/vehicles_clean.csv"),
|
|
"desc": "40+ fordonskolumner (regnr, vin, make, model, skatt, besiktning ...)",
|
|
},
|
|
{
|
|
"id": "allabolag",
|
|
"label": "SF · extracted_allabolag.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "cleaned_allabolag/extracted_allabolag.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "hitta",
|
|
"label": "SF · extracted_hitta.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "cleaned_allabolag/extracted_hitta.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "ratsit",
|
|
"label": "SF · extracted_ratsit.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "cleaned_allabolag/extracted_ratsit.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "bolagsfakta",
|
|
"label": "SF · extracted_bolagsfakta.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "cleaned_allabolag/extracted_bolagsfakta.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "hitta_companies",
|
|
"label": "SF · hitta companies.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "multisite_data/hitta/companies.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "regnr",
|
|
"label": "Register · regnr x domän (14,5M, indexerat)",
|
|
"kind": "sqlite",
|
|
"desc": "prefixsök på regnr — ms-svar via index",
|
|
},
|
|
{
|
|
"id": "queue_biluppgifter",
|
|
"label": "SF · kö: biluppgifter frontier (1,8M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "full_data/biluppgifter/frontier.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "queue_biluppgifter_crawled",
|
|
"label": "SF · kö: biluppgifter crawled",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "full_data/biluppgifter/crawled.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "queue_multi_fordonssok",
|
|
"label": "SF · kö: multi/fordonssok crawled",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "full_data/multi/fordonssok/crawled.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "queue_multi_fordonskollen",
|
|
"label": "SF · kö: multi/fordonskollen crawled",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "full_data/multi/fordonskollen/crawled.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "queue_multi_carinfo",
|
|
"label": "SF · kö: multi/carinfo crawled",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "full_data/multi/carinfo/crawled.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "reco_companies",
|
|
"label": "SF · reco companies.csv",
|
|
"kind": "csv",
|
|
"path": os.path.join(SF, "multisite_data/reco/companies.csv"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "hitta_person_reg",
|
|
"label": "Register · hitta personer (6,08M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "hitta_person_urls.txt"),
|
|
"desc": "person-URL:er från hitta.se sitemaps",
|
|
},
|
|
{
|
|
"id": "hitta_company_reg",
|
|
"label": "Register · hitta företag (1,59M rena)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "hitta_company_clean.txt"),
|
|
"desc": "malformade sök-URL:er borttagna",
|
|
},
|
|
{
|
|
"id": "ratsit_person_reg",
|
|
"label": "Register · ratsit personer (6,2M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "ratsit_person_urls.txt"),
|
|
"desc": "namn+ort+födelsedatum i URL",
|
|
},
|
|
{
|
|
"id": "ratsit_company_reg",
|
|
"label": "Register · ratsit företag (1,31M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "ratsit_company_urls.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "bolagsfakta_reg",
|
|
"label": "Register · bolagsfakta företag (1,54M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "bolagsfakta_company_urls.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "allabolag_reg",
|
|
"label": "Register · allabolag företag (1,29M)",
|
|
"kind": "lines",
|
|
"path": os.path.join(SF, "allabolag_urls_for_sf.txt"),
|
|
"desc": "",
|
|
},
|
|
{
|
|
"id": "relations",
|
|
"label": "VEHIQ · relationsgraf fordon→ägare→person→företag→adress",
|
|
"kind": "multi-jsonl-gz",
|
|
"dir": os.path.join(HOME, "sf-s10/full_data/relations"),
|
|
"prefix": "relations-",
|
|
"desc": "kanter: src_kind/src_id/relation/dst_kind/dst_id/weight",
|
|
},
|
|
{"id":"queue_hitta_person","label":"SF · kö: multi/hitta_person crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/hitta_person/crawled.txt"),"desc":""},
|
|
{"id":"queue_hitta_company","label":"SF · kö: multi/hitta_company crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/hitta_company/crawled.txt"),"desc":""},
|
|
{"id":"queue_ratsit_person","label":"SF · kö: multi/ratsit_person crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/ratsit_person/crawled.txt"),"desc":""},
|
|
{"id":"queue_bolagsfakta_company","label":"SF · kö: multi/bolagsfakta_company crawled","kind":"lines","path":os.path.join(SF,"full_data/multi/bolagsfakta_company/crawled.txt"),"desc":""},
|
|
{
|
|
"id": "identified",
|
|
"label": "SF · identifierade records (_id + regnr/orgnr/personnummer)",
|
|
"kind": "multi-jsonl-gz",
|
|
"dir": os.path.join(HOME, "sf-s10/full_data/identified"),
|
|
"prefix": "identified",
|
|
"desc": "_id (vehicle:/company:/person:), id_type, regnr, orgnr, personnummer",
|
|
},
|
|
{
|
|
"id": "relations_v2",
|
|
"label": "VEHIQ · relations v2 (företag→adress, företag→person, samma adress)",
|
|
"kind": "multi-jsonl-gz",
|
|
"dir": os.path.join(HOME, "sf-s10/full_data/relations"),
|
|
"prefix": "relations_v2",
|
|
"desc": "located_at / related_person / related_company / same_address / brand / model",
|
|
},
|
|
]
|
|
|
|
|
|
def _wc(path):
|
|
try:
|
|
with open(path, "rb") as f:
|
|
return sum(1 for _ in f)
|
|
except Exception:
|
|
return -1
|
|
|
|
|
|
def _ds(dsid):
|
|
for d in DATASETS:
|
|
if d["id"] == dsid:
|
|
return d
|
|
return None
|
|
|
|
|
|
def _iter_ds(d, reverse_spool=True):
|
|
if d["kind"] == "multi-jsonl-gz":
|
|
files = [
|
|
f
|
|
for f in _files_jsonl_gz(d["dir"])
|
|
if os.path.basename(f).startswith(d.get("prefix", ""))
|
|
]
|
|
if reverse_spool:
|
|
files = sorted(files, key=os.path.getmtime, reverse=True)
|
|
for fp in files:
|
|
yield from _iter_jsonl_gz(fp)
|
|
elif d["kind"] == "jsonl":
|
|
yield from _iter_jsonl(d["path"])
|
|
elif d["kind"] == "csv":
|
|
yield from _iter_csv(d["path"])
|
|
elif d["kind"] == "lines":
|
|
yield from _iter_lines(d["path"])
|
|
|
|
|
|
def counts():
|
|
if time.time() - COUNT_CACHE["ts"] < 120:
|
|
return COUNT_CACHE["data"]
|
|
cfile = os.path.join(HOME, ".local/share/scrape-console/counts.json")
|
|
try:
|
|
saved = json.load(open(cfile))
|
|
except Exception:
|
|
saved = {}
|
|
|
|
def fkey(d):
|
|
try:
|
|
if d["kind"] in ("csv", "jsonl", "lines"):
|
|
st = os.stat(d["path"])
|
|
return "%s:%s" % (st.st_mtime, st.st_size)
|
|
fs = sorted(_files_jsonl_gz(d["dir"]))
|
|
return "%s:%s" % (len(fs), sum(os.path.getsize(f) for f in fs))
|
|
except Exception:
|
|
return "?"
|
|
|
|
out, dirty = {}, False
|
|
for d in DATASETS:
|
|
if d["kind"] == "sqlite":
|
|
out[d["id"]] = {"rows": sum(DOMAIN_COUNTS.values()), "bytes": 4100000000}
|
|
continue
|
|
key = fkey(d)
|
|
if saved.get(d["id"], {}).get("key") == key:
|
|
out[d["id"]] = saved[d["id"]]
|
|
continue
|
|
try:
|
|
if d["kind"] == "csv":
|
|
n = _count(_iter_csv(d["path"]))
|
|
elif d["kind"] == "jsonl":
|
|
n = _count(_iter_jsonl(d["path"]))
|
|
elif d["kind"] == "lines":
|
|
n = _count(_iter_lines(d["path"]))
|
|
else:
|
|
files = [
|
|
f
|
|
for f in _files_jsonl_gz(d["dir"])
|
|
if os.path.basename(f).startswith(d.get("prefix", ""))
|
|
]
|
|
n = 0
|
|
for fp in files:
|
|
n += _count(_iter_jsonl_gz(fp))
|
|
if n >= 2000000:
|
|
break
|
|
except Exception as e:
|
|
out[d["id"]] = {"rows": -1, "bytes": -1, "err": str(e)[:80]}
|
|
continue
|
|
out[d["id"]] = {"rows": n, "bytes": -1, "key": key}
|
|
saved[d["id"]] = out[d["id"]]
|
|
dirty = True
|
|
if dirty:
|
|
try:
|
|
json.dump(saved, open(cfile, "w"))
|
|
except Exception:
|
|
pass
|
|
COUNT_CACHE.update(ts=time.time(), data=out)
|
|
return out
|
|
|
|
|
|
REGDB = os.path.expanduser("~/.local/share/regnr-register/regnr.db")
|
|
DOMAIN_COUNTS = {"fordonssok": 11524863, "fordonskollen": 13366899,
|
|
"carinfo": 11075425, "minbilkoll": 98658} # snapshot 2026-10-05
|
|
|
|
def _sqlite_sightings(q="", page=1, per=50):
|
|
import sqlite3
|
|
if not os.path.exists(REGDB):
|
|
raise FileNotFoundError("registret återbyggs — klart inom ~1h")
|
|
con = sqlite3.connect("file:" + REGDB + "?mode=ro", uri=True)
|
|
cur = con.cursor()
|
|
q = (q or "").strip().upper()
|
|
if q:
|
|
cur.execute("SELECT regnr, domain, url FROM sightings WHERE regnr LIKE ? "
|
|
"ORDER BY regnr, domain LIMIT 500", (q + "%",))
|
|
rows = [{"regnr": r[0], "domain": r[1], "url": r[2]} for r in cur.fetchall()]
|
|
else:
|
|
cur.execute("SELECT regnr, domain, url FROM sightings ORDER BY regnr LIMIT 50")
|
|
rows = [{"regnr": r[0], "domain": r[1], "url": r[2]} for r in cur.fetchall()]
|
|
con.close()
|
|
return rows
|
|
|
|
|
|
def _iter_lines(path):
|
|
try:
|
|
with open(path, errors="replace") as f:
|
|
for line in f:
|
|
line = line.strip()
|
|
if line:
|
|
yield {"rad": line[:300]}
|
|
except Exception:
|
|
return
|
|
|
|
|
|
def _spool_title(html):
|
|
import re
|
|
m = re.search(r"<title>(.*?)</title>", html or "", re.S | re.I)
|
|
return (m.group(1).strip()[:140] if m else "-")
|
|
|
|
|
|
def _flat(rec):
|
|
|
|
"""Platta till en nivå för tabellvisning (kapslade -> JSON-sträng, trunkerad)."""
|
|
if not isinstance(rec, dict):
|
|
return {"value": str(rec)[:300]}
|
|
if set(rec.keys()) == {"u", "h"}:
|
|
return {"url": rec.get("u", "")[:200], "titel": _spool_title(rec.get("h", "")),
|
|
"html_kb": round(len(rec.get("h", "")) / 1024, 1)}
|
|
out = {}
|
|
for k, v in rec.items():
|
|
if isinstance(v, (dict, list)):
|
|
s = json.dumps(v, ensure_ascii=False)
|
|
out[k] = (s[:220] + "…") if len(s) > 220 else s
|
|
else:
|
|
s = str(v)
|
|
out[k] = (s[:220] + "…") if len(s) > 220 else s
|
|
return out
|
|
|
|
|
|
def query(dsid, q="", page=1, per=50, scan_cap=400000, sort=None, sdir="asc"):
|
|
d = _ds(dsid)
|
|
if not d:
|
|
return {"error": "okänt dataset"}
|
|
if d["kind"] == "sqlite":
|
|
try:
|
|
rows = _sqlite_sightings(q, page, per)
|
|
except Exception as e:
|
|
return {"error": str(e)[:120], "total": 0, "page": page, "per": per,
|
|
"pages": 1, "cols": [], "rows": [], "scanned": 0, "truncated": False}
|
|
cols = ["regnr", "domain", "url"]
|
|
if sort == "domain":
|
|
rows.sort(key=lambda r: r["domain"], reverse=(sdir == "desc"))
|
|
start = (page - 1) * per
|
|
total = len(rows) if q else sum(DOMAIN_COUNTS.values())
|
|
return {"total": total, "page": page, "per": per,
|
|
"pages": max(1, (min(total, 500) + per - 1) // per),
|
|
"cols": cols, "rows": rows[start:start + per],
|
|
"scanned": len(rows), "truncated": len(rows) >= 500,
|
|
"sort": sort, "dir": sdir}
|
|
if d["kind"] == "lines" and not q and not sort:
|
|
total = _wc(d["path"])
|
|
start = (page - 1) * per
|
|
rows, i = [], 0
|
|
for rec in _iter_lines(d["path"]):
|
|
if i >= start and len(rows) < per:
|
|
rows.append(_flat(rec))
|
|
i += 1
|
|
if i >= start + per:
|
|
break
|
|
return {"total": total if total >= 0 else 0, "page": page, "per": per,
|
|
"pages": max(1, (max(total, 0) + per - 1) // per),
|
|
"cols": ["rad"], "rows": rows, "scanned": i,
|
|
"truncated": False, "sort": sort, "dir": sdir}
|
|
q = (q or "").lower()
|
|
matched, scanned = [], 0
|
|
for rec in _iter_ds(d):
|
|
scanned += 1
|
|
if scanned > scan_cap:
|
|
break
|
|
if q and q not in json.dumps(rec, ensure_ascii=False).lower():
|
|
continue
|
|
matched.append(rec)
|
|
cols = list(_flat(matched[0]).keys())[:12] if matched else []
|
|
if sort and sort in cols:
|
|
try:
|
|
matched.sort(key=lambda r: str(_flat(r).get(sort, "")),
|
|
reverse=(sdir == "desc"))
|
|
except Exception:
|
|
pass
|
|
total = len(matched)
|
|
start = (page - 1) * per
|
|
return {
|
|
"total": total,
|
|
"page": page,
|
|
"per": per,
|
|
"pages": max(1, (total + per - 1) // per),
|
|
"cols": cols,
|
|
"rows": [_flat(r) for r in matched[start:start + per]],
|
|
"scanned": scanned,
|
|
"truncated": scanned >= scan_cap,
|
|
"sort": sort,
|
|
"dir": sdir,
|
|
}
|
|
|
|
|
|
def record_at(dsid, n, q=""):
|
|
d = _ds(dsid)
|
|
if not d:
|
|
return {"error": "okänt dataset"}
|
|
q = (q or "").lower()
|
|
i = -1
|
|
for rec in _iter_ds(d):
|
|
if q and q not in json.dumps(rec, ensure_ascii=False).lower():
|
|
continue
|
|
i += 1
|
|
if i == n:
|
|
return {"n": n, "record": rec}
|
|
if i > n + 5:
|
|
break
|
|
return {"error": "finns ej"}
|
|
|
|
|
|
def export(dsid, q="", fmt="csv", cap=100000):
|
|
d = _ds(dsid)
|
|
q = (q or "").lower()
|
|
rows, cols, n = [], [], 0
|
|
for rec in _iter_ds(d):
|
|
if q and q not in json.dumps(rec, ensure_ascii=False).lower():
|
|
continue
|
|
f = _flat(rec)
|
|
if not cols:
|
|
cols = list(f.keys())
|
|
rows.append(f)
|
|
n += 1
|
|
if n >= cap:
|
|
break
|
|
if fmt == "json":
|
|
body = json.dumps(rows, ensure_ascii=False, indent=1).encode()
|
|
return body, "application/json", f"{dsid}.json"
|
|
if fmt == "html":
|
|
h = ["<table><tr>" + "".join(f"<th>{c}</th>" for c in cols) + "</tr>"]
|
|
for r in rows:
|
|
h.append("<tr>" + "".join(f"<td>{str(r.get(c, ''))}</td>" for c in cols) + "</tr>")
|
|
h.append("</table>")
|
|
return ("".join(h)).encode(), "text/html", f"{dsid}.html"
|
|
if fmt == "xml":
|
|
import xml.etree.ElementTree as ET
|
|
root = ET.Element("rows")
|
|
for r in rows:
|
|
it = ET.SubElement(root, "row")
|
|
for c in cols:
|
|
e = ET.SubElement(it, "f", name=c)
|
|
e.text = str(r.get(c, ""))
|
|
return ET.tostring(root, encoding="unicode").encode(), "application/xml", f"{dsid}.xml"
|
|
import io
|
|
|
|
buf = io.StringIO()
|
|
w = csv.DictWriter(buf, fieldnames=cols, extrasaction="ignore")
|
|
w.writeheader()
|
|
w.writerows(rows)
|
|
return buf.getvalue().encode(), "text/csv", f"{dsid}.csv"
|
|
|
|
|
|
PAGE = """<!DOCTYPE html><html lang="sv"><head><meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Scrape Console · Alla data</title>
|
|
<style>body{background:#0d1117;color:#e6edf3;font-family:system-ui,sans-serif;margin:0;padding:16px}
|
|
a{color:#58a6ff}h1{font-size:20px;margin:0 0 8px}.bar{display:flex;gap:8px;flex-wrap:wrap;margin-bottom:12px;align-items:center}
|
|
select,input{background:#0d1117;color:#e6edf3;border:1px solid #30363d;border-radius:6px;padding:6px 10px;font-size:13px}
|
|
button{background:#21262d;color:#e6edf3;border:1px solid #30363d;border-radius:6px;padding:6px 12px;cursor:pointer;font-size:13px}
|
|
button:hover{background:#30363d}
|
|
table{border-collapse:collapse;width:100%;font-size:12px}th,td{border:1px solid #30363d;padding:5px 8px;text-align:left;vertical-align:top;max-width:280px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap}
|
|
th{background:#161b22;position:sticky;top:0;cursor:pointer}th:first-child,td:first-child{position:sticky;left:0;background:#161b22;z-index:1}
|
|
tr:hover td{background:#1c2530;cursor:pointer}
|
|
#info{color:#8b949e;font-size:12px;margin:8px 0}
|
|
#cols{font-size:11px;color:#8b949e;display:flex;gap:8px;flex-wrap:wrap;margin-bottom:8px}
|
|
#cols label{cursor:pointer}
|
|
#modal{position:fixed;inset:0;background:rgba(0,0,0,.7);display:none;align-items:center;justify-content:center}
|
|
#mbox{background:#161b22;border:1px solid #30363d;border-radius:8px;max-width:90%;max-height:85%;overflow:auto;padding:16px}
|
|
pre{font-size:11px;white-space:pre-wrap}</style></head>
|
|
<body><h1>🕷️ <a href="/">Scrape Console</a> · <a href="/dash">Min dashboard</a> · Alla data</h1>
|
|
<div class="bar"><select id="scr"><option value="">Alla scrapers</option><option value="sf">SF multi-scrape</option><option value="carinfo">car.info</option></select><select id="ds"></select><input id="q" placeholder="sök (delsträng)…" size="28">
|
|
<button onclick="load(1)">Sök</button><button onclick="exp('csv')">CSV</button><button onclick="exp('json')">JSON</button>
|
|
<span id="info"></span></div>
|
|
<div id="cols"></div>
|
|
<div id="pg" class="bar"></div>
|
|
<table><thead id="th"></thead><tbody id="tb"></tbody></table>
|
|
<div id="modal" onclick="this.style.display='none'"><div id="mbox" onclick="event.stopPropagation()"></div></div>
|
|
<script>
|
|
let DS='',Q='',PAGEG=1,COLS=[],HIDE=new Set(),SORT=null,DIR='asc',ALLDS=[];
|
|
async function init(){
|
|
ALLDS = await (await fetch('/datasets')).json();
|
|
const prm = new URLSearchParams(location.search);
|
|
if(prm.get('scraper')) document.getElementById('scr').value = prm.get('scraper');
|
|
document.getElementById('scr').onchange = fillDS;
|
|
fillDS();
|
|
if(prm.get('ds')) document.getElementById('ds').value = prm.get('ds');
|
|
DS = document.getElementById('ds').value;
|
|
document.getElementById('q').addEventListener('keydown',e=>{if(e.key==='Enter')load(1);});
|
|
load(1);
|
|
}
|
|
function fillDS(){
|
|
const f = document.getElementById('scr').value;
|
|
const s = document.getElementById('ds');
|
|
s.innerHTML = ALLDS.filter(d=>!f||d.scraper===f).map(x=>`<option value="${x.id}">${x.label} (${x.rows})</option>`).join('');
|
|
DS = s.value;
|
|
}
|
|
async function load(p){
|
|
Q = document.getElementById('q').value; PAGEG = p;
|
|
DS = document.getElementById('ds').value;
|
|
let url = `/data?ds=${DS}&q=${encodeURIComponent(Q)}&page=${p}`;
|
|
if(SORT) url += `&sort=${encodeURIComponent(SORT)}&dir=${DIR}`;
|
|
const r = await (await fetch(url)).json();
|
|
if(r.error){document.getElementById('info').textContent=r.error;return;}
|
|
COLS = r.cols;
|
|
document.getElementById('info').textContent = `${r.total} träffar · sida ${r.page}/${r.pages}${r.truncated?' · cappad genomsökning':''}${r.sort?` · sort ${r.sort} ${r.dir}`:''}`;
|
|
document.getElementById('cols').innerHTML = 'Kolumner: ' + COLS.map(c=>`<label><input type="checkbox" ${HIDE.has(c)?'':'checked'} onchange="tglCol('${c.replace(/'/g,'')}')"> ${c}</label>`).join('');
|
|
const vis = COLS.filter(c=>!HIDE.has(c));
|
|
document.getElementById('th').innerHTML = '<tr><th>#</th>'+vis.map(c=>`<th onclick="sortBy('${c.replace(/'/g,'')}')">${c}${SORT===c?(DIR==='asc'?' ▲':' ▼'):''}</th>`).join('')+'</tr>';
|
|
document.getElementById('tb').innerHTML = r.rows.map((row,i)=>`<tr onclick="openRec(${(r.page-1)*r.per+i})"><td>${(r.page-1)*r.per+i}</td>`+vis.map(c=>`<td>${String(row[c]??'').replace(/</g,'<')}</td>`).join('')+'</tr>').join('');
|
|
let pg = '';
|
|
for(const pp of [1,r.page-1,r.page,r.page+1,r.pages]) if(pp>=1&&pp<=r.pages) pg += `<button ${pp===r.page?'disabled':''} onclick="load(${pp})">${pp}</button>`;
|
|
document.getElementById('pg').innerHTML = pg;
|
|
}
|
|
function tglCol(c){HIDE.has(c)?HIDE.delete(c):HIDE.add(c);load(PAGEG);}
|
|
function sortBy(c){if(SORT===c)DIR=DIR==='asc'?'desc':'asc';else{SORT=c;DIR='asc';}load(PAGEG);}
|
|
async function openRec(n){
|
|
const r = await (await fetch(`/record?ds=${DS}&n=${n}&q=${encodeURIComponent(Q)}`)).json();
|
|
document.getElementById('mbox').innerHTML = `<h3>Post #${n} (${DS})</h3><pre>${JSON.stringify(r.record,null,1).replace(/</g,'<').slice(0,60000)}</pre>`;
|
|
document.getElementById('modal').style.display = 'flex';
|
|
}
|
|
function exp(fmt){location.href = `/export?ds=${DS}&q=${encodeURIComponent(Q)}&fmt=${fmt}`;}
|
|
init();
|
|
</script></body></html>
|
|
"""
|