Files
0pt/services/scrapers/clickhouse_pipeline.py
T

262 lines
9.7 KiB
Python

"""
ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
Ersätter:
- BigQuery (används aldrig)
- MinIO/S3 fil-lagring (mellanlagring)
Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
"""
import os, json, time, gzip
from datetime import datetime
from pathlib import Path
from typing import Optional, Any
CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
class ClickHouseWriter:
"""Write scraped data to ClickHouse via HTTP interface."""
def __init__(self):
self.client = None
self._connect()
def _connect(self):
from clickhouse_driver import Client
self.client = Client(
host=CLICKHOUSE_HOST,
port=CLICKHOUSE_PORT,
user=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
database=CLICKHOUSE_DB,
settings={"max_memory_usage": 8000000000},
)
def create_database(self):
self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
def ensure_tables(self):
"""Create all required tables for scraped data."""
# Companies table (multi-country)
self.client.execute("""
CREATE TABLE IF NOT EXISTS companies (
org_nr String,
country FixedString(2),
source String,
name String,
legal_form String DEFAULT '',
status String DEFAULT '',
address String DEFAULT '',
postal_code String DEFAULT '',
city String DEFAULT '',
country_code String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
website String DEFAULT '',
employees UInt32 DEFAULT 0,
founded_date Date DEFAULT '1970-01-01',
registered_capital Float64 DEFAULT 0,
currency FixedString(3) DEFAULT '',
industry_code String DEFAULT '',
industry_description String DEFAULT '',
lei String DEFAULT '',
last_updated DateTime DEFAULT now(),
raw_json String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr)
PARTITION BY country
SETTINGS index_granularity = 8192
""")
# Contacts table (board members, executives)
self.client.execute("""
CREATE TABLE IF NOT EXISTS contacts (
org_nr String,
country FixedString(2),
full_name String,
role String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
source String DEFAULT '',
confidence Float32 DEFAULT 0,
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr, full_name)
PARTITION BY country
""")
# Vehicles table (for DE vehicle data)
self.client.execute("""
CREATE TABLE IF NOT EXISTS vehicles (
registration String,
country FixedString(2),
vin String DEFAULT '',
make String DEFAULT '',
model String DEFAULT '',
year UInt16 DEFAULT 0,
fuel_type String DEFAULT '',
owner_name String DEFAULT '',
owner_org_nr String DEFAULT '',
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, registration)
PARTITION BY country
""")
# Raw scrape log (for debugging/audit)
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url String,
source String,
country FixedString(2),
method String DEFAULT '',
status UInt16 DEFAULT 0,
elapsed_ms UInt32 DEFAULT 0,
bytes UInt32 DEFAULT 0,
cf_blocked UInt8 DEFAULT 0,
error String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = MergeTree
ORDER BY (country, scraped_at)
PARTITION BY toYYYYMM(scraped_at)
TTL scraped_at + INTERVAL 90 DAY
""")
# Statistics for monitoring
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_stats (
country FixedString(2),
source String,
date Date DEFAULT today(),
total_requests UInt64 DEFAULT 0,
successful UInt64 DEFAULT 0,
cf_blocked UInt64 DEFAULT 0,
errors UInt64 DEFAULT 0,
total_bytes UInt64 DEFAULT 0,
avg_elapsed_ms Float64 DEFAULT 0,
PRIMARY KEY (country, source, date)
) ENGINE = SummingMergeTree
ORDER BY (country, source, date)
""")
def insert_company(self, org_nr: str, country: str, source: str, **fields):
"""Insert or update a company record."""
data = {
"org_nr": org_nr,
"country": country,
"source": source,
"name": fields.get("name", ""),
"legal_form": fields.get("legal_form", ""),
"status": fields.get("status", ""),
"address": fields.get("address", ""),
"postal_code": fields.get("postal_code", ""),
"city": fields.get("city", ""),
"country_code": fields.get("country_code", ""),
"email": fields.get("email", ""),
"phone": fields.get("phone", ""),
"website": fields.get("website", ""),
"employees": int(fields.get("employees", 0)),
"founded_date": fields.get("founded_date", "1970-01-01"),
"registered_capital": float(fields.get("registered_capital", 0)),
"currency": fields.get("currency", ""),
"industry_code": fields.get("industry_code", ""),
"industry_description": fields.get("industry_description", ""),
"lei": fields.get("lei", ""),
"raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
}
self.client.execute(
"INSERT INTO companies VALUES",
[data],
)
def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
email: str = "", phone: str = "", source: str = "", confidence: float = 0):
self.client.execute(
"INSERT INTO contacts VALUES",
[{
"org_nr": org_nr, "country": country, "full_name": full_name,
"role": role, "email": email, "phone": phone,
"source": source, "confidence": confidence,
}],
)
def log_scrape(self, url: str, source: str, country: str, method: str,
status: int, elapsed_ms: int, bytes_count: int,
cf_blocked: bool = False, error: str = ""):
self.client.execute(
"INSERT INTO scrape_log VALUES",
[{
"url": url[:500], "source": source, "country": country,
"method": method, "status": status, "elapsed_ms": elapsed_ms,
"bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
}],
)
def query(self, sql: str) -> list:
"""Execute SQL query and return results."""
return self.client.execute(sql)
def close(self):
if self.client:
self.client.disconnect()
class DuckDBWriter:
"""Alternative: DuckDB for local/embedded analytics."""
def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
import duckdb
self.db_path = db_path
self.conn = duckdb.connect(db_path)
self._ensure_tables()
def _ensure_tables(self):
self.conn.execute("""
CREATE TABLE IF NOT EXISTS companies AS
SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
""")
self.conn.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url VARCHAR, source VARCHAR, country VARCHAR,
method VARCHAR, status INTEGER, elapsed_ms INTEGER,
bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
)
""")
def import_parquet(self, parquet_dir: str = "/data/parquet"):
"""Import all parquet files into DuckDB."""
import glob
files = glob.glob(f"{parquet_dir}/*.parquet")
if not files:
return 0
self.conn.execute(f"""
INSERT INTO companies
SELECT * FROM read_parquet({files!r})
""")
return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
def query(self, sql: str) -> list:
return self.conn.execute(sql).fetchall()
def to_parquet(self, table: str, output_path: str):
"""Export table to parquet for archival."""
self.conn.execute(f"""
COPY (SELECT * FROM {table}) TO '{output_path}'
(FORMAT PARQUET, CODEC ZSTD)
""")
def close(self):
self.conn.close()