236 lines
8.3 KiB
Python
236 lines
8.3 KiB
Python
"""
|
|
0pt Scraper Pipeline - Production Batch Processor
|
|
|
|
Kör batch-scraping för angivet land/källa med checkpointing.
|
|
Data sparas lokalt och synkas till MinIO S3.
|
|
|
|
Användning:
|
|
python -m scrapers.run_pipeline --country NO --source brreg
|
|
python -m scrapers.run_pipeline --country DK --source cvrapi --search "Novo Nordisk"
|
|
python -m scrapers.run_pipeline --country DE --source northdata --hrb "HRB 123456"
|
|
"""
|
|
import os, sys, json, time, asyncio
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
|
|
sys.path.insert(0, os.path.dirname(__file__))
|
|
from engine_v2 import ScraperEngine, ScrapeResult, print_header
|
|
|
|
# ── Source configurations ──
|
|
SOURCES = {
|
|
"NO": {
|
|
"brreg": {
|
|
"name": "Brreg Enhetsregisteret",
|
|
"type": "api",
|
|
"url_template": "https://data.brreg.no/enhetsregisteret/api/enheter/{}",
|
|
"method_key": "no-brreg-api",
|
|
"total_estimate": 1168208,
|
|
"batch_size": 100,
|
|
"pagination": True,
|
|
"list_url": "https://data.brreg.no/enhetsregisteret/api/enheter?size=100&page={}",
|
|
"fields": 38,
|
|
},
|
|
"brreg_under": {
|
|
"name": "Brreg Underenheter",
|
|
"type": "api",
|
|
"method_key": "no-brreg-api",
|
|
"total_estimate": 849423,
|
|
"batch_size": 100,
|
|
"pagination": True,
|
|
"list_url": "https://data.brreg.no/enhetsregisteret/api/underenheter?size=100&page={}",
|
|
},
|
|
"proff": {
|
|
"name": "Proff.no",
|
|
"type": "web",
|
|
"url_template": "https://proff.no/company/companydetails/{}",
|
|
"method_key": "no-proff",
|
|
},
|
|
"1881": {
|
|
"name": "1881.no",
|
|
"type": "web",
|
|
"url_template": "https://www.1881.no/sok/?q={}",
|
|
"method_key": "no-1881",
|
|
},
|
|
"flowfirma": {
|
|
"name": "Flowfirma.no",
|
|
"type": "web",
|
|
"url_template": "https://flowfirma.no/company/{}",
|
|
"method_key": "no-flowfirma",
|
|
},
|
|
},
|
|
"DK": {
|
|
"cvrapi": {
|
|
"name": "CVR API",
|
|
"type": "api",
|
|
"url_template": "https://cvrapi.dk/api?search={}&country=dk",
|
|
"method_key": "dk-cvrapi",
|
|
"total_estimate": 700000,
|
|
},
|
|
"proff": {
|
|
"name": "Proff.dk",
|
|
"type": "web",
|
|
"url_template": "https://proff.dk/company/companydetails/{}",
|
|
"method_key": "dk-proff",
|
|
},
|
|
"grundigt": {
|
|
"name": "Grundigt.dk",
|
|
"type": "web",
|
|
"url_template": "https://grundigt.dk/virksomhed/{}",
|
|
"method_key": "dk-grundigt",
|
|
},
|
|
},
|
|
"DE": {
|
|
"northdata": {
|
|
"name": "Northdata.de",
|
|
"type": "web",
|
|
"url_template": "https://www.northdata.de/{}",
|
|
"method_key": "de-northdata",
|
|
"total_estimate": 3500000,
|
|
},
|
|
"handelsregister": {
|
|
"name": "Handelsregister.de",
|
|
"type": "web",
|
|
"url_template": "https://www.handelsregister.de/{}",
|
|
"method_key": "de-handelsregister",
|
|
},
|
|
"moneyhouse": {
|
|
"name": "Moneyhouse.de",
|
|
"type": "web",
|
|
"url_template": "https://www.moneyhouse.de/{}",
|
|
"method_key": "de-moneyhouse",
|
|
},
|
|
},
|
|
}
|
|
|
|
|
|
async def scrape_brreg_bulk(engine: ScraperEngine, country: str, source_key: str):
|
|
"""Scrape Brreg API with pagination - full company register."""
|
|
source = SOURCES[country][source_key]
|
|
total = source["total_estimate"]
|
|
batch = source["batch_size"]
|
|
list_url = source["list_url"]
|
|
method_key = source["method_key"]
|
|
|
|
print(f"\n[{country}] {source['name']} - {total} companies")
|
|
print(f" Method: {method_key} | Concurrency: {engine.MAX_CONCURRENCY}")
|
|
|
|
engine.stats["start_time"] = time.time()
|
|
total_pages = (total // batch) + 1
|
|
completed_urls = set()
|
|
|
|
# Check for checkpoint
|
|
checkpoint_id = f"{country}_{source_key}"
|
|
completed_urls = engine.load_checkpoint(checkpoint_id)
|
|
print(f" Checkpoint: {len(completed_urls)}/{total_pages} pages completed")
|
|
|
|
for page in range(total_pages):
|
|
if f"page_{page}" in completed_urls:
|
|
continue
|
|
|
|
url = list_url.format(page)
|
|
result = engine.scrape_url(url, country, method_key)
|
|
|
|
if result.status == 200 and not result.cf_blocked:
|
|
import json as j
|
|
try:
|
|
data = j.loads(result.data) if result.data else {}
|
|
items = data.get("_embedded", {}).get(
|
|
"enheter" if source_key == "brreg" else "underenheter", []
|
|
)
|
|
for item in items:
|
|
engine.save_result(ScrapeResult(
|
|
url=url, method=result.method,
|
|
status=200, elapsed=result.elapsed,
|
|
bytes=len(j.dumps(item)),
|
|
data=item
|
|
), j.dumps(item))
|
|
except:
|
|
pass
|
|
|
|
# Track progress
|
|
if (page + 1) % 10 == 0:
|
|
elapsed = time.time() - engine.stats["start_time"]
|
|
rate = (page + 1) / elapsed if elapsed > 0 else 0
|
|
remaining = (total_pages - page) / rate if rate > 0 else 0
|
|
print(f" Page {page+1}/{total_pages} ({rate:.0f} pages/s, ETA: {remaining:.0f}s)")
|
|
engine.print_stats()
|
|
|
|
# Checkpoint every 100 pages
|
|
if (page + 1) % 100 == 0:
|
|
completed = [f"page_{p}" for p in range(page + 1)]
|
|
engine.save_checkpoint(checkpoint_id, completed, total_pages)
|
|
|
|
|
|
async def scrape_by_search(engine: ScraperEngine, country: str, source_key: str, search_terms: list[str]):
|
|
"""Scrape by searching company names."""
|
|
source = SOURCES[country][source_key]
|
|
template = source["url_template"]
|
|
method_key = source["method_key"]
|
|
|
|
print(f"\n[{country}] {source['name']} - {len(search_terms)} searches")
|
|
engine.stats["start_time"] = time.time()
|
|
|
|
for i, term in enumerate(search_terms):
|
|
url = template.format(term.replace(" ", "+"))
|
|
result = engine.scrape_url(url, country, method_key)
|
|
|
|
if result.status == 200:
|
|
engine.save_result(result)
|
|
|
|
if (i + 1) % 10 == 0:
|
|
elapsed = time.time() - engine.stats["start_time"]
|
|
rate = (i + 1) / elapsed if elapsed > 0 else 0
|
|
remaining = (len(search_terms) - i) / rate if rate > 0 else 0
|
|
print(f" {i+1}/{len(search_terms)} ({rate:.0f} req/s, ETA: {remaining:.0f}s)")
|
|
engine.print_stats()
|
|
|
|
|
|
def parse_args():
|
|
import argparse
|
|
parser = argparse.ArgumentParser(description="0pt Scraper Pipeline")
|
|
parser.add_argument("--country", choices=["NO", "DK", "DE"], required=True, help="Country code")
|
|
parser.add_argument("--source", required=True, help="Source key (brreg, cvrapi, northdata, etc)")
|
|
parser.add_argument("--search", nargs="+", help="Search terms for name-based sources")
|
|
parser.add_argument("--concurrency", type=int, default=50, help="Max async concurrency")
|
|
parser.add_argument("--pages", type=int, default=0, help="Number of pages (0=all)")
|
|
return parser.parse_args()
|
|
|
|
|
|
def main():
|
|
print_header()
|
|
args = parse_args()
|
|
|
|
country = args.country
|
|
source_key = args.source
|
|
max_conc = args.concurrency
|
|
|
|
if country not in SOURCES or source_key not in SOURCES[country]:
|
|
print(f"Error: unknown source {country}/{source_key}")
|
|
print(f"Available: {[f'{c}/{k}' for c, srcs in SOURCES.items() for k in srcs.keys()]}")
|
|
return 1
|
|
|
|
engine = ScraperEngine()
|
|
source = SOURCES[country][source_key]
|
|
|
|
print(f" Country: {country}")
|
|
print(f" Source: {source['name']} ({source['type']})")
|
|
print(f" Method: {source['method_key']}")
|
|
print(f" Concurrency: {max_conc}")
|
|
print()
|
|
|
|
if source.get("pagination"):
|
|
asyncio.run(scrape_brreg_bulk(engine, country, source_key))
|
|
elif args.search:
|
|
asyncio.run(scrape_by_search(engine, country, source_key, args.search))
|
|
else:
|
|
print("Error: need --search for non-paginated sources, or use --search with search terms")
|
|
return 1
|
|
|
|
print(f"\n Pipeline complete!")
|
|
engine.print_stats()
|
|
return 0
|
|
|
|
|
|
if __name__ == "__main__":
|
|
sys.exit(main())
|