scrape-console/scraper_views.py

#!/usr/bin/env python3
"""Scraper-detaljsidor: /scraper/<id> - spec + live-data per scraper."""

SCRAPERS = {
    "sf": {
        "label": "SF multi-scrape", "unit": "sf-multi-scrape.service",
        "desc": "Multi-source vehicle scraper: biluppgifter/fordonskollen via pool, fordonssok/car.info direkt",
        "cmd": "Documents/SF_Extractor/.venv/bin/python scrape_sources.py",
        "sources": ["biluppgifter", "fordonskollen", "fordonssok", "carinfo"],
        "spec": [
            ("Källor", "biluppgifter.se, fordonskollen.se, fordonssok.se, car.info (sv-se)"),
            ("Indata", "scrape_urls/*.txt (14,5M regnr, sitemap-enumererade)"),
            ("Utdata", "spool/*.jsonl.gz (rå HTML) + full_data/*/records-*.jsonl.gz (extraherat)"),
            ("Takt", "~4 parallella per domän; biluppgifter via pool (CF IP-block)"),
            ("Drift", "systemd always-restart (startar om efter stop-timeout, återupptar)"),
            ("Logg", "Documents/SF_Extractor/logs/multi_scrape.log — [källa] n= ok= fail= @pos/total"),
        ],
    },
    "carinfo": {
        "label": "car.info specs", "unit": "carinfo-crawl.service",
        "desc": "car.info spec crawler, resumable (vehiq VEH-HIST-030)",
        "cmd": "/usr/bin/python3 ~/.local/share/carinfo/crawl.py",
        "sources": ["car.info"],
        "spec": [
            ("Källa", "car.info modell/generation/motorvarianter (en-se)"),
            ("Utdata", "~/.local/share/carinfo/trims.jsonl {url, make, model, specs}"),
            ("Checkpoint", "ckpt.json (makes_done, models_done, gens_done) — återupptar"),
            ("Drift", "systemd; cooldown-loop vid tunna modeller"),
            ("Logg", "~/.local/share/carinfo/crawl.log"),
        ],
    },
    "enrich": {
        "label": "SIAX enrich", "unit": "siax-enrich.service",
        "desc": "Job Description Enricher (LinkedIn jobbsidor)",
        "cmd": "job-automation/deploy/scanner/enrich-descriptions-laptop.py",
        "sources": ["linkedin"],
        "spec": [
            ("Källa", "LinkedIn jobbsidor"),
            ("Utdata", "ingen lokal resultatfil (fjeder/nätverk) — endast status synlig"),
            ("Drift", "systemd"),
        ],
    },
    "scanner": {
        "label": "SIAX laptop-scanner", "unit": "siax-laptop-scanner.service",
        "desc": "Laptop Job Scanner (residential IP, pushar till server6)",
        "cmd": "job-automation/deploy/scanner/laptop-scanner.py",
        "sources": ["jobbsajter"],
        "spec": [
            ("Källa", "jobbsajter via residential IP"),
            ("Utdata", "pushas till server6 — ingen lokal resultatfil"),
            ("Drift", "systemd"),
        ],
    },
}

DETAIL = """<!DOCTYPE html><html lang="sv"><head><meta charset="utf-8">
<meta name="viewport" content="width=device-width,initial-scale=1">
<title>Scrape Console · __LABEL__</title>
<style>body{background:#0d1117;color:#e6edf3;font-family:system-ui,sans-serif;margin:0;padding:16px}
a{color:#58a6ff}h1{font-size:20px;margin:0 0 4px}h2{font-size:15px;margin:16px 0 6px}
.card{background:#161b22;border:1px solid #30363d;border-radius:8px;padding:12px;margin-bottom:12px}
table{border-collapse:collapse;width:100%;font-size:12px}th,td{border:1px solid #30363d;padding:5px 8px;text-align:left;vertical-align:top}
th{background:#0d1117;width:140px;color:#8b949e}
.badge{display:inline-block;font-size:11px;padding:2px 8px;border-radius:10px}
.on{background:#1a7f37}.off{background:#8c1515}
.kv{font-size:12px;color:#8b949e;display:flex;gap:12px;flex-wrap:wrap}.kv b{color:#e6edf3}
.bar{background:#30363d;border-radius:4px;height:8px;margin:4px 0 8px}.bar>i{display:block;height:8px;border-radius:4px;background:#1f6feb}
.src{font-size:12px;margin:2px 0}.src b{color:#e6edf3}
pre{background:#0d1117;border:1px solid #30363d;border-radius:6px;padding:8px;font-size:11px;overflow:auto;max-height:200px;white-space:pre-wrap}
button{background:#21262d;color:#e6edf3;border:1px solid #30363d;border-radius:6px;padding:4px 10px;margin-right:4px;cursor:pointer;font-size:12px}</style></head>
<body><h1>🕷️ <a href="/">Scrape Console</a> · <span id="t"></span></h1>
<div class="card"><span id="badge"></span> <span id="ctl"></span><div class="kv" id="kv"></div><div id="prog"></div></div>
<div class="card"><h2>Specifikation</h2><table id="spec"></table><div class="kv" id="cmd" style="margin-top:8px"></div></div>
<div class="card"><h2>Dataset</h2><div id="ds"></div></div>
<div class="card"><h2>Logg</h2><pre id="log"></pre></div>
<div class="card"><h2>Körningar (runs-historik, 3 dygn)</h2><div id="runs"></div></div>
<div class="card" id="pipeCard" style="display:none"><h2>Pipeline-steg</h2><div id="pipe"></div></div>
<div class="card" id="proxyCard" style="display:none"><h2>Proxy-pool (IP-rotation)</h2><div id="proxy"></div></div>
<div class="card"><h2>Schema</h2><div id="sched"></div></div>
<script>
const ID = "__ID__", LOCAL = ['localhost','127.0.0.1'].includes(location.hostname);
async function tick(){
  const s = await (await fetch('/state')).json();
  const c = s.scrapers.find(x=>x.unit=== "__UNIT__");
  const meta = await (await fetch('/scraper-meta?id='+ID)).json();
  document.title = 'Scrape Console · ' + meta.label;
  document.getElementById('t').textContent = meta.label;
  const on = c && c.active==='active';
  document.getElementById('badge').innerHTML = `<span class="badge ${on?'on':'off'}">${c?c.active+' · pid '+c.pid:'okänd'}</span>`;
  document.getElementById('ctl').innerHTML = LOCAL
    ? `<button onclick="ctl('restart')">↻ Omstart</button><button onclick="ctl('stop')">⏹ Stopp</button><button onclick="ctl('start')">▶ Start</button><a href="/data-page?scraper=${meta.key}"><button>Alla data →</button></a>`
    : `<a href="/data-page?scraper=${meta.key}"><button>Alla data →</button></a>`;
  document.getElementById('kv').innerHTML = c ? `<span>minne <b>${c.mem_mb} MB</b></span><span>cpu <b>${c.cpu_s}s</b></span><span>start <b>${c.started||'-'}</b></span>` : '';
  let ph = '';
  if (c && c.progress) for (const [k,v] of Object.entries(c.progress)) {
    if (k.startsWith('_')) continue;
    const pct = 100*v.pos/v.total, okr = (100*v.ok/Math.max(1,v.ok+v.fail)).toFixed(1);
    ph += `<div class="src"><b>${k}</b> ${v.ok} ok / ${v.fail} fail (ok-rate ${okr}%) · ${v.pos}/${v.total} (${pct.toFixed(2)}%)</div><div class="bar"><i style="width:${Math.min(100,pct)}%"></i></div>`;
  }
  if (c && c.ckpt) ph += `<div class="src">checkpoint: <b>${Object.entries(c.ckpt).map(([k,v])=>k+'='+v).join(' · ')}</b></div>`;
  document.getElementById('prog').innerHTML = ph;
  document.getElementById('spec').innerHTML = meta.spec.map(([k,v])=>`<tr><th>${k}</th><td>${v}</td></tr>`).join('');
  document.getElementById('cmd').innerHTML = `<span>kommando: <b>${meta.cmd}</b></span><span>${meta.desc}</span>`;
  const ds = await (await fetch('/datasets')).json();
  document.getElementById('ds').innerHTML = ds.filter(d=>d.scraper===meta.key).map(d=>`<div class="src"><b>${d.label}</b> — ${d.rows} rader · <a href="/data-page?ds=${d.id}">öppna tabell →</a></div>`).join('') || '<div class="kv">inga lokala dataset (statusläge)</div>';
  document.getElementById('log').textContent = (c && c.log) || '(ingen logg)';
  const ops = await (await fetch('/ops')).json();
  const runs = (ops.runs && ops.runs["__UNIT__"]) || [];
  document.getElementById('runs').innerHTML = runs.length ? runs.slice().reverse().map(r =>
    `<div class="src"><b>${r.day||''} ${r.start} → ${r.end}</b> · ${r.result}</div>`).join('')
    : '<div class="kv">inga journalförda starter senaste 3 dygnen</div>';
  document.getElementById('sched').innerHTML = (ops.schedules||[]).map(s =>
    `<div class="src">nästa <b>${s.next}</b> · ${String(s.unit).slice(0,80)}</div>`).join('') || '<div class="kv">inga scheman</div>';
  if (ID === 'sf') {
    document.getElementById('pipeCard').style.display = 'block';
    document.getElementById('proxyCard').style.display = 'block';
    document.getElementById('pipe').innerHTML = Object.entries(ops.pipeline||{}).map(([k,v]) =>
      `<div class="src">${k}: <b>${v}</b></div>`).join('');
    const px = ops.proxy||{};
    document.getElementById('proxy').innerHTML = px.error ? px.error :
      `<div class="kv"><span>proxies <b>${px.total}</b></span><span>fräscha &lt;1h <b>${px.fresh_1h}</b></span><span>friska <b>${px.healthy}</b></span><span>latens p50 <b>${px.lat_p50} ms</b></span></div>
       <div class="src">topp-länder: <b>${(px.top_countries||[]).map(([c,n])=>c+' '+n).join(' · ')}</b></div>`;
  }
}
async function ctl(action){
  if(!confirm(action+'?')) return;
  const r = await fetch('/control',{method:'POST',headers:{'Content-Type':'application/json'},body:JSON.stringify({unit:"__UNIT__",action})});
  alert(await r.text()); tick();
}
tick(); setInterval(tick, 10000);
</script></body></html>"""