407 lines
17 KiB
Python
407 lines
17 KiB
Python
#!/usr/bin/env python3
|
|
"""Scrape Console v2: live-översikt + historik + kontroller (localhost-only).
|
|
GET / /state /history /log?src=multi&n=50 · POST /control (127.0.0.1 only)."""
|
|
|
|
import json, os, re, shutil, subprocess, time
|
|
from http.server import BaseHTTPRequestHandler, HTTPServer
|
|
from urllib.parse import urlparse, parse_qs
|
|
import data_views
|
|
import dash_views
|
|
import scraper_views
|
|
import ops_views
|
|
|
|
HOME = os.path.expanduser("~")
|
|
SF = os.path.join(HOME, "Documents/SF_Extractor")
|
|
CI = os.path.join(HOME, ".local/share/carinfo")
|
|
HIST = os.path.join(HOME, ".local/share/scrape-console/history.jsonl")
|
|
CACHE = {"ts": 0, "data": None}
|
|
|
|
|
|
def sys_show(unit, *props):
|
|
try:
|
|
out = subprocess.run(
|
|
["systemctl", "--user", "show", unit, "-p", ",".join(props)],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=10,
|
|
).stdout
|
|
d = {}
|
|
for line in out.splitlines():
|
|
if "=" in line:
|
|
k, v = line.split("=", 1)
|
|
d[k] = v
|
|
return d
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def svc(unit, label, desc):
|
|
p = sys_show(
|
|
unit,
|
|
"ActiveState",
|
|
"MainPID",
|
|
"MemoryCurrent",
|
|
"CPUUsageNSec",
|
|
"ExecMainStartTimestamp",
|
|
)
|
|
mem = int(p.get("MemoryCurrent") or 0)
|
|
return {
|
|
"unit": unit,
|
|
"label": label,
|
|
"desc": desc,
|
|
"active": p.get("ActiveState", "?"),
|
|
"pid": p.get("MainPID", "?"),
|
|
"mem_mb": round(mem / 1024 / 1024, 1),
|
|
"cpu_s": round(int(p.get("CPUUsageNSec") or 0) / 1e9, 1),
|
|
"started": (p.get("ExecMainStartTimestamp") or "")[4:16],
|
|
}
|
|
|
|
|
|
def tail(path, n=8):
|
|
try:
|
|
with open(path, errors="replace") as f:
|
|
return "".join(f.readlines()[-n:])
|
|
except Exception as e:
|
|
return f"(kan inte läsa {path}: {e})"
|
|
|
|
|
|
def sf_progress():
|
|
out = {}
|
|
try:
|
|
with open(os.path.join(SF, "logs/multi_scrape.log"), errors="replace") as f:
|
|
for line in f:
|
|
m = re.match(
|
|
r"\[(.+?)\]\s+n=(\d+)\s+ok=(\d+)\s+fail=(\d+)\s+@(\d+)/(\d+)",
|
|
line.strip(),
|
|
)
|
|
if m:
|
|
src, n, ok, fail, pos, total = m.groups()
|
|
out[src] = {
|
|
"n": int(n),
|
|
"ok": int(ok),
|
|
"fail": int(fail),
|
|
"pos": int(pos),
|
|
"total": int(total),
|
|
}
|
|
except Exception as e:
|
|
out["_error"] = str(e)
|
|
return out
|
|
|
|
|
|
def du(path):
|
|
try:
|
|
return round(
|
|
sum(
|
|
os.path.getsize(os.path.join(dp, f))
|
|
for dp, _, fns in os.walk(path)
|
|
for f in fns
|
|
)
|
|
/ 1024
|
|
/ 1024,
|
|
1,
|
|
)
|
|
except Exception:
|
|
return -1
|
|
|
|
|
|
def collect():
|
|
try:
|
|
d = json.load(open(os.path.join(CI, "ckpt.json")))
|
|
ci_ckpt = {k: (len(v) if isinstance(v, list) else v) for k, v in d.items()}
|
|
except Exception as e:
|
|
ci_ckpt = {"_error": str(e)[:100]}
|
|
try:
|
|
spool = len(os.listdir(os.path.join(SF, "spool")))
|
|
except Exception:
|
|
spool = -1
|
|
prog = sf_progress()
|
|
mem = {
|
|
s["unit"]: s["mem_mb"]
|
|
for s in [
|
|
svc("sf-multi-scrape.service", "", ""),
|
|
svc("carinfo-crawl.service", "", ""),
|
|
]
|
|
}
|
|
data = {
|
|
"ts": time.strftime("%H:%M:%S"),
|
|
"scrapers": [
|
|
dict(
|
|
svc(
|
|
"sf-multi-scrape.service",
|
|
"SF multi-scrape",
|
|
"fordon: biluppgifter/fordonskollen/fordonssok/car.info",
|
|
),
|
|
progress=prog,
|
|
spool_files=spool,
|
|
data_mb={"full_data": du(os.path.join(SF, "full_data"))},
|
|
log=tail(os.path.join(SF, "logs/multi_scrape.log"), 6),
|
|
),
|
|
dict(
|
|
svc(
|
|
"carinfo-crawl.service", "car.info specs", "VEH-HIST-030, resumable"
|
|
),
|
|
ckpt=ci_ckpt,
|
|
log=tail(os.path.join(CI, "crawl.log"), 6),
|
|
),
|
|
dict(svc("siax-enrich.service", "SIAX enrich", "LinkedIn jobbsidor")),
|
|
dict(
|
|
svc(
|
|
"siax-laptop-scanner.service",
|
|
"SIAX laptop-scanner",
|
|
"jobb, residential IP",
|
|
)
|
|
),
|
|
],
|
|
"cron": {
|
|
"p0lytics_health": tail("/tmp/p0lytics-health.log", 2).strip(),
|
|
"p0lytics_rate": tail("/tmp/p0lytics-rate.log", 2).strip(),
|
|
},
|
|
}
|
|
# historik (1 prov/min)
|
|
try:
|
|
last = 0
|
|
if os.path.exists(HIST):
|
|
with open(HIST, errors="replace") as f:
|
|
for line in f:
|
|
pass
|
|
try:
|
|
last = json.loads(line).get("t", 0)
|
|
except Exception:
|
|
pass
|
|
now = int(time.time())
|
|
if now - last >= 55:
|
|
snap = {
|
|
"t": now,
|
|
"ok": {k: v["ok"] for k, v in prog.items() if not k.startswith("_")},
|
|
"mem": mem,
|
|
}
|
|
with open(HIST, "a") as f:
|
|
f.write(json.dumps(snap) + "\n")
|
|
# kapa till ~1500 rader
|
|
with open(HIST, errors="replace") as f:
|
|
lines = f.readlines()
|
|
if len(lines) > 1500:
|
|
open(HIST, "w").writelines(lines[-1500:])
|
|
except Exception:
|
|
pass
|
|
return data
|
|
|
|
|
|
HTML = """<!DOCTYPE html><html lang="sv"><head><meta charset="utf-8">
|
|
<meta name="viewport" content="width=device-width,initial-scale=1">
|
|
<title>Scrape Console live</title>
|
|
<style>body{background:#0d1117;color:#e6edf3;font-family:system-ui,sans-serif;margin:0;padding:16px}
|
|
h1{font-size:20px;margin:0 0 4px}#ts{color:#8b949e;font-size:12px;margin-bottom:12px}
|
|
.grid{display:grid;grid-template-columns:repeat(auto-fit,minmax(360px,1fr));gap:12px}
|
|
.card{background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px}
|
|
.card h2{font-size:15px;margin:0 0 2px}.desc{color:#8b949e;font-size:12px;margin-bottom:8px}
|
|
.badge{display:inline-block;font-size:11px;padding:2px 8px;border-radius:10px;margin:0 4px 8px 0}
|
|
.on{background:#1a7f37}.off{background:#8c1515}.qq{background:#9e6a03}
|
|
.kv{font-size:12px;color:#8b949e;display:flex;gap:12px;flex-wrap:wrap;margin-bottom:6px}
|
|
.kv b{color:#e6edf3}.bar{background:#30363d;border-radius:4px;height:8px;margin:4px 0 8px}.bar>i{display:block;height:8px;border-radius:4px;background:#1f6feb}
|
|
.src{font-size:12px;margin:2px 0}.src b{color:#e6edf3}
|
|
pre{background:#0d1117;border:1px solid #30363d;border-radius:6px;padding:8px;font-size:11px;overflow:auto;max-height:150px;white-space:pre-wrap}
|
|
button{background:#21262d;color:#e6edf3;border:1px solid #30363d;border-radius:6px;padding:4px 10px;margin:0 4px 4px 0;cursor:pointer;font-size:12px}
|
|
button:hover{background:#30363d}svg{width:100%;height:44px;margin:2px 0 6px}</style></head>
|
|
<body><h1>🕷️ Scrape Console <span style="font-size:12px;color:#8b949e">live</span> <a href="/data-page" style="font-size:13px">Alla data →</a> <a href="/dash" style="font-size:13px">Min dashboard →</a></h1>
|
|
<div id="ts"></div><div id="alerts" style="margin-bottom:12px"></div><div class="grid" id="g"></div>
|
|
<script>
|
|
const LOCAL = ['localhost','127.0.0.1'].includes(location.hostname);
|
|
let hist = [];
|
|
let HISTG = [];
|
|
async function tick(){
|
|
const s = await (await fetch('/state')).json();
|
|
try { HISTG = await (await fetch('/history')).json(); } catch(e) {}
|
|
try {
|
|
const ops = await (await fetch('/ops')).json();
|
|
const col = {crit:'#8c1515', err:'#8c1515', warn:'#9e6a03'};
|
|
document.getElementById('alerts').innerHTML = (ops.alerts||[]).map(x =>
|
|
`<div class="card" style="border-left:4px solid ${col[x.level]||'#666'};margin-bottom:8px"><b>${x.level.toUpperCase()}</b> · ${x.text}</div>`).join('');
|
|
} catch(e) {}
|
|
document.getElementById('ts').textContent = 'Uppdaterad ' + s.ts + ' (auto 10 s)' + (LOCAL ? '' : ' · skrivskyddad vy');
|
|
hist = await (await fetch('/history')).json();
|
|
const g = document.getElementById('g'); g.innerHTML = '';
|
|
for (const c of s.scrapers) {
|
|
const on = c.active === 'active';
|
|
let h = `<div class="card"><span class="badge ${on?'on':'off'}">${c.active} · pid ${c.pid}</span>`;
|
|
if (LOCAL) h += `<button onclick="ctl('${c.unit}','restart')">↻</button><button onclick="ctl('${c.unit}','stop')">⏹</button><button onclick="ctl('${c.unit}','start')">▶</button>`;
|
|
const SCR0 = {'SF multi-scrape':'sf','car.info specs':'carinfo','SIAX enrich':'enrich','SIAX laptop-scanner':'scanner'};
|
|
const sid0 = SCR0[c.label] || '';
|
|
h += `<h2><a href="/scraper/${sid0}" style="color:#e6edf3">${c.label}</a></h2><div class="desc">${c.desc}<br>${c.unit}</div><div class="kv"><span>minne <b>${c.mem_mb} MB</b></span><span>cpu <b>${c.cpu_s}s</b></span><span>start <b>${c.started||'-'}</b></span></div>`;
|
|
if (sid0) h += `<div class="src"><a href="/data-page?scraper=${sid0}">Visa all data →</a></div>`;
|
|
if (c.progress) for (const [k,v] of Object.entries(c.progress)) {
|
|
if (k.startsWith('_')) continue;
|
|
const pct = (100*v.pos/v.total);
|
|
const okr = (100*v.ok/Math.max(1,v.ok+v.fail)).toFixed(1);
|
|
const eta = etaStr(k, v);
|
|
h += `<div class="src"><b>${k}</b> ${v.ok} ok / ${v.fail} fail (ok-rate ${okr}%) · ${v.pos}/${v.total} (${pct.toFixed(2)}%)${eta}</div><div class="bar"><i style="width:${Math.min(100,pct)}%"></i></div>`;
|
|
h += spark(k);
|
|
}
|
|
if (c.spool_files !== undefined) h += `<div class="src">spool-filer: <b>${c.spool_files}</b> · full_data: <b>${c.data_mb.full_data} MB</b></div>`;
|
|
if (c.ckpt) h += `<div class="src">checkpoint: <b>${Object.entries(c.ckpt).map(([k,v])=>k+'='+v).join(' · ')}</b></div>`;
|
|
if (c.log) h += `<pre>${c.log.replace(/</g,'<')}</pre>`;
|
|
g.innerHTML += h + '</div>';
|
|
}
|
|
g.innerHTML += `<div class="card"><span class="badge qq">cron</span><h2>p0lytics monitors</h2><div class="desc">health var 15:e min · rate var 30:e</div><pre>health: ${s.cron.p0lytics_health.replace(/</g,'<')}\nrate: ${s.cron.p0lytics_rate.replace(/</g,'<')}</pre></div>`;
|
|
}
|
|
function etaStr(src, v){
|
|
const pts = HISTG.filter(h=>h.ok && h.ok[src]!==undefined).slice(-12);
|
|
if(pts.length < 2) return '';
|
|
const dv = pts[pts.length-1].ok[src]-pts[0].ok[src], dt = (pts[pts.length-1].t-pts[0].t)/60;
|
|
if(dv <= 0 || dt <= 0) return '';
|
|
const left = (v.total-v.pos)/(dv/dt), h = Math.floor(left/60), m = Math.round(left%60);
|
|
return ` · ETA ${h>0?h+'h ':''}${m}m`;
|
|
}
|
|
function spark(src){
|
|
const pts = hist.filter(h=>h.ok && h.ok[src]!==undefined).slice(-60);
|
|
if (pts.length < 2) return '';
|
|
const vals = pts.map(h=>h.ok[src]); const max = Math.max(...vals), min = Math.min(...vals);
|
|
const W=300,H=40, path = vals.map((v,i)=>`${(i/(vals.length-1)*W).toFixed(1)},${(H-2-((v-min)/Math.max(1,max-min))*(H-4)).toFixed(1)}`).join(' ');
|
|
const rate = ((vals[vals.length-1]-vals[0])/Math.max(1,(pts[pts.length-1].t-pts[0].t)/60)).toFixed(0);
|
|
return `<svg viewBox="0 0 ${W} ${H}"><polyline points="${path}" fill="none" stroke="#1f6feb" stroke-width="1.5"/></svg><div class="src">takt: <b>${rate}/min</b> senaste timmen</div>`;
|
|
}
|
|
async function ctl(unit,action){
|
|
if(!confirm(action+' '+unit+'?')) return;
|
|
const r = await fetch('/control',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({unit,action})});
|
|
alert(await r.text()); tick();
|
|
}
|
|
tick(); setInterval(tick, 10000);
|
|
</script></body></html>"""
|
|
|
|
|
|
class H(BaseHTTPRequestHandler):
|
|
def log_message(self, *a):
|
|
pass
|
|
|
|
def _send(self, body, ctype):
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
def do_GET(self):
|
|
u = urlparse(self.path)
|
|
if u.path == "/state":
|
|
if time.time() - CACHE["ts"] > 10:
|
|
CACHE["data"] = collect()
|
|
CACHE["ts"] = time.time()
|
|
self._send(json.dumps(CACHE["data"]).encode(), "application/json")
|
|
elif u.path == "/history":
|
|
try:
|
|
with open(HIST, errors="replace") as f:
|
|
lines = f.readlines()
|
|
data = [json.loads(l) for l in lines[-90:] if l.strip()]
|
|
except Exception:
|
|
data = []
|
|
self._send(json.dumps(data).encode(), "application/json")
|
|
elif u.path == "/log":
|
|
q = parse_qs(u.query)
|
|
src = q.get("src", ["multi"])[0]
|
|
n = min(int(q.get("n", ["50"])[0]), 200)
|
|
path = {
|
|
"multi": os.path.join(SF, "logs/multi_scrape.log"),
|
|
"carinfo": os.path.join(CI, "crawl.log"),
|
|
}.get(src, "")
|
|
self._send(tail(path, n).encode(), "text/plain; charset=utf-8")
|
|
elif u.path == "/data-page":
|
|
self._send(data_views.PAGE.encode(), "text/html; charset=utf-8")
|
|
elif u.path == "/dash":
|
|
self._send(dash_views.DASH_PAGE.encode(), "text/html; charset=utf-8")
|
|
elif u.path == "/datasets":
|
|
counts = data_views.counts()
|
|
out = [
|
|
{
|
|
"id": d["id"],
|
|
"label": d["label"],
|
|
"scraper": data_views.SCRAPER_OF.get(d["id"], ""),
|
|
"rows": counts.get(d["id"], {}).get("rows", -1),
|
|
}
|
|
for d in data_views.DATASETS
|
|
]
|
|
self._send(json.dumps(out).encode(), "application/json")
|
|
elif u.path == "/data":
|
|
q = parse_qs(u.query)
|
|
res = data_views.query(
|
|
q.get("ds", [""])[0],
|
|
q.get("q", [""])[0],
|
|
int(q.get("page", ["1"])[0]),
|
|
sort=q.get("sort", [None])[0],
|
|
sdir=q.get("dir", ["asc"])[0],
|
|
)
|
|
self._send(json.dumps(res, ensure_ascii=False).encode(), "application/json")
|
|
elif u.path == "/record":
|
|
q = parse_qs(u.query)
|
|
res = data_views.record_at(
|
|
q.get("ds", [""])[0], int(q.get("n", ["0"])[0]), q.get("q", [""])[0]
|
|
)
|
|
self._send(json.dumps(res, ensure_ascii=False).encode(), "application/json")
|
|
elif u.path == "/export":
|
|
q = parse_qs(u.query)
|
|
body, ctype, fname = data_views.export(
|
|
q.get("ds", [""])[0], q.get("q", [""])[0], q.get("fmt", ["csv"])[0]
|
|
)
|
|
self.send_response(200)
|
|
self.send_header("Content-Type", ctype)
|
|
self.send_header("Content-Disposition", "attachment; filename=" + fname)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
elif u.path.startswith("/scraper/"):
|
|
sid = u.path.split("/scraper/")[-1].split("?")[0]
|
|
sc = scraper_views.SCRAPERS.get(sid)
|
|
if not sc:
|
|
self.send_response(404); self.end_headers(); return
|
|
html = scraper_views.DETAIL.replace("__ID__", sid).replace(
|
|
"__UNIT__", sc["unit"]).replace("__LABEL__", sc["label"])
|
|
self._send(html.encode(), "text/html; charset=utf-8")
|
|
elif u.path == "/scraper-meta":
|
|
q = parse_qs(u.query)
|
|
sid = q.get("id", [""])[0]
|
|
sc = scraper_views.SCRAPERS.get(sid, {})
|
|
self._send(json.dumps({"key": sid, "label": sc.get("label", sid),
|
|
"desc": sc.get("desc", ""), "cmd": sc.get("cmd", ""),
|
|
"spec": sc.get("spec", [])}).encode(),
|
|
"application/json")
|
|
elif u.path == "/ops":
|
|
self._send(json.dumps(ops_views.ops()).encode(), "application/json")
|
|
else:
|
|
self._send(HTML.encode(), "text/html; charset=utf-8")
|
|
|
|
def do_POST(self):
|
|
if self.path != "/control":
|
|
self.send_response(404)
|
|
self.end_headers()
|
|
return
|
|
if self.client_address[0] not in ("127.0.0.1", "::1"):
|
|
self.send_response(403)
|
|
self.end_headers()
|
|
self.wfile.write(b"localhost only")
|
|
return
|
|
try:
|
|
d = json.loads(self.rfile.read(int(self.headers["Content-Length"] or 0)))
|
|
unit, action = d["unit"], d["action"]
|
|
assert re.match(r"^[a-z0-9@._-]+\.service$", unit) and action in (
|
|
"start",
|
|
"stop",
|
|
"restart",
|
|
)
|
|
r = subprocess.run(
|
|
["systemctl", "--user", action, unit],
|
|
capture_output=True,
|
|
text=True,
|
|
timeout=60,
|
|
)
|
|
msg = f"{action} {unit}: {'OK' if r.returncode == 0 else r.stderr[:200]}"
|
|
except Exception as e:
|
|
msg = f"fel: {e}"
|
|
body = msg.encode()
|
|
self.send_response(200)
|
|
self.send_header("Content-Length", str(len(body)))
|
|
self.end_headers()
|
|
self.wfile.write(body)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
HTTPServer((os.getenv("HOST", "127.0.0.1"), int(os.getenv("PORT", "8899"))), H).serve_forever()
|