""" 0pt Scraper Pipeline - Production Batch Processor Kör batch-scraping för angivet land/källa med checkpointing. Data sparas lokalt och synkas till MinIO S3. Användning: python -m scrapers.run_pipeline --country NO --source brreg python -m scrapers.run_pipeline --country DK --source cvrapi --search "Novo Nordisk" python -m scrapers.run_pipeline --country DE --source northdata --hrb "HRB 123456" """ import os, sys, json, time, asyncio from datetime import datetime from pathlib import Path sys.path.insert(0, os.path.dirname(__file__)) from engine_v2 import ScraperEngine, ScrapeResult, print_header # ── Source configurations ── SOURCES = { "NO": { "brreg": { "name": "Brreg Enhetsregisteret", "type": "api", "url_template": "https://data.brreg.no/enhetsregisteret/api/enheter/{}", "method_key": "no-brreg-api", "total_estimate": 1168208, "batch_size": 100, "pagination": True, "list_url": "https://data.brreg.no/enhetsregisteret/api/enheter?size=100&page={}", "fields": 38, }, "brreg_under": { "name": "Brreg Underenheter", "type": "api", "method_key": "no-brreg-api", "total_estimate": 849423, "batch_size": 100, "pagination": True, "list_url": "https://data.brreg.no/enhetsregisteret/api/underenheter?size=100&page={}", }, "proff": { "name": "Proff.no", "type": "web", "url_template": "https://proff.no/company/companydetails/{}", "method_key": "no-proff", }, "1881": { "name": "1881.no", "type": "web", "url_template": "https://www.1881.no/sok/?q={}", "method_key": "no-1881", }, "flowfirma": { "name": "Flowfirma.no", "type": "web", "url_template": "https://flowfirma.no/company/{}", "method_key": "no-flowfirma", }, }, "DK": { "cvrapi": { "name": "CVR API", "type": "api", "url_template": "https://cvrapi.dk/api?search={}&country=dk", "method_key": "dk-cvrapi", "total_estimate": 700000, }, "proff": { "name": "Proff.dk", "type": "web", "url_template": "https://proff.dk/company/companydetails/{}", "method_key": "dk-proff", }, "grundigt": { "name": "Grundigt.dk", "type": "web", "url_template": "https://grundigt.dk/virksomhed/{}", "method_key": "dk-grundigt", }, }, "DE": { "northdata": { "name": "Northdata.de", "type": "web", "url_template": "https://www.northdata.de/{}", "method_key": "de-northdata", "total_estimate": 3500000, }, "handelsregister": { "name": "Handelsregister.de", "type": "web", "url_template": "https://www.handelsregister.de/{}", "method_key": "de-handelsregister", }, "moneyhouse": { "name": "Moneyhouse.de", "type": "web", "url_template": "https://www.moneyhouse.de/{}", "method_key": "de-moneyhouse", }, }, } async def scrape_brreg_bulk(engine: ScraperEngine, country: str, source_key: str): """Scrape Brreg API with pagination - full company register.""" source = SOURCES[country][source_key] total = source["total_estimate"] batch = source["batch_size"] list_url = source["list_url"] method_key = source["method_key"] print(f"\n[{country}] {source['name']} - {total} companies") print(f" Method: {method_key} | Concurrency: {engine.MAX_CONCURRENCY}") engine.stats["start_time"] = time.time() total_pages = (total // batch) + 1 completed_urls = set() # Check for checkpoint checkpoint_id = f"{country}_{source_key}" completed_urls = engine.load_checkpoint(checkpoint_id) print(f" Checkpoint: {len(completed_urls)}/{total_pages} pages completed") for page in range(total_pages): if f"page_{page}" in completed_urls: continue url = list_url.format(page) result = engine.scrape_url(url, country, method_key) if result.status == 200 and not result.cf_blocked: import json as j try: data = j.loads(result.data) if result.data else {} items = data.get("_embedded", {}).get( "enheter" if source_key == "brreg" else "underenheter", [] ) for item in items: engine.save_result(ScrapeResult( url=url, method=result.method, status=200, elapsed=result.elapsed, bytes=len(j.dumps(item)), data=item ), j.dumps(item)) except: pass # Track progress if (page + 1) % 10 == 0: elapsed = time.time() - engine.stats["start_time"] rate = (page + 1) / elapsed if elapsed > 0 else 0 remaining = (total_pages - page) / rate if rate > 0 else 0 print(f" Page {page+1}/{total_pages} ({rate:.0f} pages/s, ETA: {remaining:.0f}s)") engine.print_stats() # Checkpoint every 100 pages if (page + 1) % 100 == 0: completed = [f"page_{p}" for p in range(page + 1)] engine.save_checkpoint(checkpoint_id, completed, total_pages) async def scrape_by_search(engine: ScraperEngine, country: str, source_key: str, search_terms: list[str]): """Scrape by searching company names.""" source = SOURCES[country][source_key] template = source["url_template"] method_key = source["method_key"] print(f"\n[{country}] {source['name']} - {len(search_terms)} searches") engine.stats["start_time"] = time.time() for i, term in enumerate(search_terms): url = template.format(term.replace(" ", "+")) result = engine.scrape_url(url, country, method_key) if result.status == 200: engine.save_result(result) if (i + 1) % 10 == 0: elapsed = time.time() - engine.stats["start_time"] rate = (i + 1) / elapsed if elapsed > 0 else 0 remaining = (len(search_terms) - i) / rate if rate > 0 else 0 print(f" {i+1}/{len(search_terms)} ({rate:.0f} req/s, ETA: {remaining:.0f}s)") engine.print_stats() def parse_args(): import argparse parser = argparse.ArgumentParser(description="0pt Scraper Pipeline") parser.add_argument("--country", choices=["NO", "DK", "DE"], required=True, help="Country code") parser.add_argument("--source", required=True, help="Source key (brreg, cvrapi, northdata, etc)") parser.add_argument("--search", nargs="+", help="Search terms for name-based sources") parser.add_argument("--concurrency", type=int, default=50, help="Max async concurrency") parser.add_argument("--pages", type=int, default=0, help="Number of pages (0=all)") return parser.parse_args() def main(): print_header() args = parse_args() country = args.country source_key = args.source max_conc = args.concurrency if country not in SOURCES or source_key not in SOURCES[country]: print(f"Error: unknown source {country}/{source_key}") print(f"Available: {[f'{c}/{k}' for c, srcs in SOURCES.items() for k in srcs.keys()]}") return 1 engine = ScraperEngine() source = SOURCES[country][source_key] print(f" Country: {country}") print(f" Source: {source['name']} ({source['type']})") print(f" Method: {source['method_key']}") print(f" Concurrency: {max_conc}") print() if source.get("pagination"): asyncio.run(scrape_brreg_bulk(engine, country, source_key)) elif args.search: asyncio.run(scrape_by_search(engine, country, source_key, args.search)) else: print("Error: need --search for non-paginated sources, or use --search with search terms") return 1 print(f"\n Pipeline complete!") engine.print_stats() return 0 if __name__ == "__main__": sys.exit(main())