87 lines
3.7 KiB
Python
87 lines
3.7 KiB
Python
"""
|
|
0pt Scraper Benchmark - test alla källor for DK/NO/DE
|
|
"""
|
|
import sys, os, time, json
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from engine import scrape_best, scrape_async, benchmark_source, OUTPUT_DIR
|
|
from datetime import datetime
|
|
|
|
# === ALLA KALLOR TESTADE ===
|
|
SOURCES = {
|
|
"DK": [
|
|
("CVR API (cvrapi.dk)", "https://cvrapi.dk/api?search=Novo+Nordisk&country=dk", "api"),
|
|
("Proff.dk", "https://proff.dk", "agg"),
|
|
("Krak.dk", "https://www.krak.dk/", "dir"),
|
|
("118.dk", "https://118.dk/", "dir"),
|
|
("De Gule Sider", "https://www.degulesider.dk/", "dir"),
|
|
("Companydata.dk", "https://companydata.dk", "agg"),
|
|
("Grundigt.dk", "https://grundigt.dk", "agg"),
|
|
("Datafordeler.dk", "https://datafordeler.dk", "gov"),
|
|
("CVRcheck.dk", "https://cvrcheck.dk", "api"),
|
|
("Virk", "https://datacvr.virk.dk/", "gov"),
|
|
],
|
|
"NO": [
|
|
("Brreg API (enhet)", "https://data.brreg.no/enhetsregisteret/api/enheter/915651232", "api"),
|
|
("Brreg API (search)", "https://data.brreg.no/enhetsregisteret/api/enheter?size=10", "api"),
|
|
("Brreg API (underenheter)", "https://data.brreg.no/enhetsregisteret/api/underenheter?size=10", "api"),
|
|
("Proff.no", "https://proff.no", "agg"),
|
|
("1881.no", "https://www.1881.no/", "dir"),
|
|
("Gulesider.no", "https://www.gulesider.no/", "dir"),
|
|
("Selskapssok.no", "https://selskapssok.no", "agg"),
|
|
("Flowfirma.no", "https://flowfirma.no", "agg"),
|
|
("Offentligdata.no", "https://offentligdata.no", "agg"),
|
|
("Regnskapsbasen.no", "https://www.regnskapsbasen.no", "agg"),
|
|
("Okonomi.no", "https://okonomi.no/bedrifter", "agg"),
|
|
("Visverdi.no", "https://visverdi.no", "agg"),
|
|
],
|
|
"DE": [
|
|
("Northdata.de (Bayer)", "https://www.northdata.de/Bayer+AG", "agg"),
|
|
("Handelsregister.de", "https://www.handelsregister.de", "gov"),
|
|
("Unternehmensregister.de", "https://www.unternehmensregister.de", "gov"),
|
|
("Bundesanzeiger.de", "https://www.bundesanzeiger.de", "gov"),
|
|
("Companyhouse.de", "https://www.companyhouse.de", "agg"),
|
|
("Moneyhouse.de", "https://www.moneyhouse.de", "agg"),
|
|
("Creditreform.de", "https://www.creditreform.de", "agg"),
|
|
("Personensuche.de", "https://www.personensuche.de", "dir"),
|
|
("Registercheck.de", "https://www.registercheck.de", "agg"),
|
|
("Whitepages.de", "https://www.whitepages.de", "dir"),
|
|
],
|
|
}
|
|
|
|
def run():
|
|
print("=" * 80)
|
|
print(f"0PT SCRAPER BENCHMARK - {datetime.now().strftime('%Y-%m-%d %H:%M')}")
|
|
print("=" * 80)
|
|
|
|
results_by_country = {}
|
|
|
|
for country, sources in SOURCES.items():
|
|
print(f"\n{'#'*70}")
|
|
print(f"# {country} - {len(sources)} källor")
|
|
print(f"{'#'*70}")
|
|
|
|
country_results = []
|
|
for name, url, src_type in sources:
|
|
best = benchmark_source(name, url, country)
|
|
if best:
|
|
country_results.append({"name": name, "url": url, "method": best.method, "elapsed": best.elapsed, "status": best.status, "bytes": best.bytes})
|
|
time.sleep(0.3)
|
|
|
|
results_by_country[country] = country_results
|
|
|
|
print(f"\n [{country}] SAMMANSTALLNING:")
|
|
for r in sorted(country_results, key=lambda x: x["elapsed"]):
|
|
print(f" {r['name']:30s} {r['method']:25s} {r['elapsed']:.3f}s {r['bytes']}b")
|
|
|
|
# Save report
|
|
report = {
|
|
"timestamp": datetime.now().isoformat(),
|
|
"results": results_by_country,
|
|
}
|
|
path = OUTPUT_DIR / f"benchmark_{datetime.now().strftime('%Y%m%d_%H%M%S')}.json"
|
|
path.write_text(json.dumps(report, indent=2, ensure_ascii=False), encoding="utf-8")
|
|
print(f"\n Rapport sparad: {path}")
|
|
|
|
if __name__ == "__main__":
|
|
run()
|