Files
0pt/services/scrapers/run_pipeline.py
T
2026-07-13 12:57:15 +02:00

236 lines
8.3 KiB
Python

"""
0pt Scraper Pipeline - Production Batch Processor
Kör batch-scraping för angivet land/källa med checkpointing.
Data sparas lokalt och synkas till MinIO S3.
Användning:
python -m scrapers.run_pipeline --country NO --source brreg
python -m scrapers.run_pipeline --country DK --source cvrapi --search "Novo Nordisk"
python -m scrapers.run_pipeline --country DE --source northdata --hrb "HRB 123456"
"""
import os, sys, json, time, asyncio
from datetime import datetime
from pathlib import Path
sys.path.insert(0, os.path.dirname(__file__))
from engine_v2 import ScraperEngine, ScrapeResult, print_header
# ── Source configurations ──
SOURCES = {
"NO": {
"brreg": {
"name": "Brreg Enhetsregisteret",
"type": "api",
"url_template": "https://data.brreg.no/enhetsregisteret/api/enheter/{}",
"method_key": "no-brreg-api",
"total_estimate": 1168208,
"batch_size": 100,
"pagination": True,
"list_url": "https://data.brreg.no/enhetsregisteret/api/enheter?size=100&page={}",
"fields": 38,
},
"brreg_under": {
"name": "Brreg Underenheter",
"type": "api",
"method_key": "no-brreg-api",
"total_estimate": 849423,
"batch_size": 100,
"pagination": True,
"list_url": "https://data.brreg.no/enhetsregisteret/api/underenheter?size=100&page={}",
},
"proff": {
"name": "Proff.no",
"type": "web",
"url_template": "https://proff.no/company/companydetails/{}",
"method_key": "no-proff",
},
"1881": {
"name": "1881.no",
"type": "web",
"url_template": "https://www.1881.no/sok/?q={}",
"method_key": "no-1881",
},
"flowfirma": {
"name": "Flowfirma.no",
"type": "web",
"url_template": "https://flowfirma.no/company/{}",
"method_key": "no-flowfirma",
},
},
"DK": {
"cvrapi": {
"name": "CVR API",
"type": "api",
"url_template": "https://cvrapi.dk/api?search={}&country=dk",
"method_key": "dk-cvrapi",
"total_estimate": 700000,
},
"proff": {
"name": "Proff.dk",
"type": "web",
"url_template": "https://proff.dk/company/companydetails/{}",
"method_key": "dk-proff",
},
"grundigt": {
"name": "Grundigt.dk",
"type": "web",
"url_template": "https://grundigt.dk/virksomhed/{}",
"method_key": "dk-grundigt",
},
},
"DE": {
"northdata": {
"name": "Northdata.de",
"type": "web",
"url_template": "https://www.northdata.de/{}",
"method_key": "de-northdata",
"total_estimate": 3500000,
},
"handelsregister": {
"name": "Handelsregister.de",
"type": "web",
"url_template": "https://www.handelsregister.de/{}",
"method_key": "de-handelsregister",
},
"moneyhouse": {
"name": "Moneyhouse.de",
"type": "web",
"url_template": "https://www.moneyhouse.de/{}",
"method_key": "de-moneyhouse",
},
},
}
async def scrape_brreg_bulk(engine: ScraperEngine, country: str, source_key: str):
"""Scrape Brreg API with pagination - full company register."""
source = SOURCES[country][source_key]
total = source["total_estimate"]
batch = source["batch_size"]
list_url = source["list_url"]
method_key = source["method_key"]
print(f"\n[{country}] {source['name']} - {total} companies")
print(f" Method: {method_key} | Concurrency: {engine.MAX_CONCURRENCY}")
engine.stats["start_time"] = time.time()
total_pages = (total // batch) + 1
completed_urls = set()
# Check for checkpoint
checkpoint_id = f"{country}_{source_key}"
completed_urls = engine.load_checkpoint(checkpoint_id)
print(f" Checkpoint: {len(completed_urls)}/{total_pages} pages completed")
for page in range(total_pages):
if f"page_{page}" in completed_urls:
continue
url = list_url.format(page)
result = engine.scrape_url(url, country, method_key)
if result.status == 200 and not result.cf_blocked:
import json as j
try:
data = j.loads(result.data) if result.data else {}
items = data.get("_embedded", {}).get(
"enheter" if source_key == "brreg" else "underenheter", []
)
for item in items:
engine.save_result(ScrapeResult(
url=url, method=result.method,
status=200, elapsed=result.elapsed,
bytes=len(j.dumps(item)),
data=item
), j.dumps(item))
except:
pass
# Track progress
if (page + 1) % 10 == 0:
elapsed = time.time() - engine.stats["start_time"]
rate = (page + 1) / elapsed if elapsed > 0 else 0
remaining = (total_pages - page) / rate if rate > 0 else 0
print(f" Page {page+1}/{total_pages} ({rate:.0f} pages/s, ETA: {remaining:.0f}s)")
engine.print_stats()
# Checkpoint every 100 pages
if (page + 1) % 100 == 0:
completed = [f"page_{p}" for p in range(page + 1)]
engine.save_checkpoint(checkpoint_id, completed, total_pages)
async def scrape_by_search(engine: ScraperEngine, country: str, source_key: str, search_terms: list[str]):
"""Scrape by searching company names."""
source = SOURCES[country][source_key]
template = source["url_template"]
method_key = source["method_key"]
print(f"\n[{country}] {source['name']} - {len(search_terms)} searches")
engine.stats["start_time"] = time.time()
for i, term in enumerate(search_terms):
url = template.format(term.replace(" ", "+"))
result = engine.scrape_url(url, country, method_key)
if result.status == 200:
engine.save_result(result)
if (i + 1) % 10 == 0:
elapsed = time.time() - engine.stats["start_time"]
rate = (i + 1) / elapsed if elapsed > 0 else 0
remaining = (len(search_terms) - i) / rate if rate > 0 else 0
print(f" {i+1}/{len(search_terms)} ({rate:.0f} req/s, ETA: {remaining:.0f}s)")
engine.print_stats()
def parse_args():
import argparse
parser = argparse.ArgumentParser(description="0pt Scraper Pipeline")
parser.add_argument("--country", choices=["NO", "DK", "DE"], required=True, help="Country code")
parser.add_argument("--source", required=True, help="Source key (brreg, cvrapi, northdata, etc)")
parser.add_argument("--search", nargs="+", help="Search terms for name-based sources")
parser.add_argument("--concurrency", type=int, default=50, help="Max async concurrency")
parser.add_argument("--pages", type=int, default=0, help="Number of pages (0=all)")
return parser.parse_args()
def main():
print_header()
args = parse_args()
country = args.country
source_key = args.source
max_conc = args.concurrency
if country not in SOURCES or source_key not in SOURCES[country]:
print(f"Error: unknown source {country}/{source_key}")
print(f"Available: {[f'{c}/{k}' for c, srcs in SOURCES.items() for k in srcs.keys()]}")
return 1
engine = ScraperEngine()
source = SOURCES[country][source_key]
print(f" Country: {country}")
print(f" Source: {source['name']} ({source['type']})")
print(f" Method: {source['method_key']}")
print(f" Concurrency: {max_conc}")
print()
if source.get("pagination"):
asyncio.run(scrape_brreg_bulk(engine, country, source_key))
elif args.search:
asyncio.run(scrape_by_search(engine, country, source_key, args.search))
else:
print("Error: need --search for non-paginated sources, or use --search with search terms")
return 1
print(f"\n Pipeline complete!")
engine.print_stats()
return 0
if __name__ == "__main__":
sys.exit(main())