262 lines
9.7 KiB
Python
262 lines
9.7 KiB
Python
"""
|
|
ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
|
|
|
|
Ersätter:
|
|
- BigQuery (används aldrig)
|
|
- MinIO/S3 fil-lagring (mellanlagring)
|
|
|
|
Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
|
|
"""
|
|
import os, json, time, gzip
|
|
from datetime import datetime
|
|
from pathlib import Path
|
|
from typing import Optional, Any
|
|
|
|
CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
|
|
CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
|
|
CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
|
|
CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
|
|
CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
|
|
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
|
|
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
|
|
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
|
|
|
|
|
|
class ClickHouseWriter:
|
|
"""Write scraped data to ClickHouse via HTTP interface."""
|
|
|
|
def __init__(self):
|
|
self.client = None
|
|
self._connect()
|
|
|
|
def _connect(self):
|
|
from clickhouse_driver import Client
|
|
self.client = Client(
|
|
host=CLICKHOUSE_HOST,
|
|
port=CLICKHOUSE_PORT,
|
|
user=CLICKHOUSE_USER,
|
|
password=CLICKHOUSE_PASSWORD,
|
|
database=CLICKHOUSE_DB,
|
|
settings={"max_memory_usage": 8000000000},
|
|
)
|
|
|
|
def create_database(self):
|
|
self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
|
|
|
|
def ensure_tables(self):
|
|
"""Create all required tables for scraped data."""
|
|
|
|
# Companies table (multi-country)
|
|
self.client.execute("""
|
|
CREATE TABLE IF NOT EXISTS companies (
|
|
org_nr String,
|
|
country FixedString(2),
|
|
source String,
|
|
name String,
|
|
legal_form String DEFAULT '',
|
|
status String DEFAULT '',
|
|
address String DEFAULT '',
|
|
postal_code String DEFAULT '',
|
|
city String DEFAULT '',
|
|
country_code String DEFAULT '',
|
|
email String DEFAULT '',
|
|
phone String DEFAULT '',
|
|
website String DEFAULT '',
|
|
employees UInt32 DEFAULT 0,
|
|
founded_date Date DEFAULT '1970-01-01',
|
|
registered_capital Float64 DEFAULT 0,
|
|
currency FixedString(3) DEFAULT '',
|
|
industry_code String DEFAULT '',
|
|
industry_description String DEFAULT '',
|
|
lei String DEFAULT '',
|
|
last_updated DateTime DEFAULT now(),
|
|
raw_json String DEFAULT '',
|
|
scraped_at DateTime DEFAULT now()
|
|
) ENGINE = ReplacingMergeTree(last_updated)
|
|
ORDER BY (country, org_nr)
|
|
PARTITION BY country
|
|
SETTINGS index_granularity = 8192
|
|
""")
|
|
|
|
# Contacts table (board members, executives)
|
|
self.client.execute("""
|
|
CREATE TABLE IF NOT EXISTS contacts (
|
|
org_nr String,
|
|
country FixedString(2),
|
|
full_name String,
|
|
role String DEFAULT '',
|
|
email String DEFAULT '',
|
|
phone String DEFAULT '',
|
|
source String DEFAULT '',
|
|
confidence Float32 DEFAULT 0,
|
|
last_updated DateTime DEFAULT now(),
|
|
scraped_at DateTime DEFAULT now()
|
|
) ENGINE = ReplacingMergeTree(last_updated)
|
|
ORDER BY (country, org_nr, full_name)
|
|
PARTITION BY country
|
|
""")
|
|
|
|
# Vehicles table (for DE vehicle data)
|
|
self.client.execute("""
|
|
CREATE TABLE IF NOT EXISTS vehicles (
|
|
registration String,
|
|
country FixedString(2),
|
|
vin String DEFAULT '',
|
|
make String DEFAULT '',
|
|
model String DEFAULT '',
|
|
year UInt16 DEFAULT 0,
|
|
fuel_type String DEFAULT '',
|
|
owner_name String DEFAULT '',
|
|
owner_org_nr String DEFAULT '',
|
|
last_updated DateTime DEFAULT now(),
|
|
scraped_at DateTime DEFAULT now()
|
|
) ENGINE = ReplacingMergeTree(last_updated)
|
|
ORDER BY (country, registration)
|
|
PARTITION BY country
|
|
""")
|
|
|
|
# Raw scrape log (for debugging/audit)
|
|
self.client.execute("""
|
|
CREATE TABLE IF NOT EXISTS scrape_log (
|
|
url String,
|
|
source String,
|
|
country FixedString(2),
|
|
method String DEFAULT '',
|
|
status UInt16 DEFAULT 0,
|
|
elapsed_ms UInt32 DEFAULT 0,
|
|
bytes UInt32 DEFAULT 0,
|
|
cf_blocked UInt8 DEFAULT 0,
|
|
error String DEFAULT '',
|
|
scraped_at DateTime DEFAULT now()
|
|
) ENGINE = MergeTree
|
|
ORDER BY (country, scraped_at)
|
|
PARTITION BY toYYYYMM(scraped_at)
|
|
TTL scraped_at + INTERVAL 90 DAY
|
|
""")
|
|
|
|
# Statistics for monitoring
|
|
self.client.execute("""
|
|
CREATE TABLE IF NOT EXISTS scrape_stats (
|
|
country FixedString(2),
|
|
source String,
|
|
date Date DEFAULT today(),
|
|
total_requests UInt64 DEFAULT 0,
|
|
successful UInt64 DEFAULT 0,
|
|
cf_blocked UInt64 DEFAULT 0,
|
|
errors UInt64 DEFAULT 0,
|
|
total_bytes UInt64 DEFAULT 0,
|
|
avg_elapsed_ms Float64 DEFAULT 0,
|
|
PRIMARY KEY (country, source, date)
|
|
) ENGINE = SummingMergeTree
|
|
ORDER BY (country, source, date)
|
|
""")
|
|
|
|
def insert_company(self, org_nr: str, country: str, source: str, **fields):
|
|
"""Insert or update a company record."""
|
|
data = {
|
|
"org_nr": org_nr,
|
|
"country": country,
|
|
"source": source,
|
|
"name": fields.get("name", ""),
|
|
"legal_form": fields.get("legal_form", ""),
|
|
"status": fields.get("status", ""),
|
|
"address": fields.get("address", ""),
|
|
"postal_code": fields.get("postal_code", ""),
|
|
"city": fields.get("city", ""),
|
|
"country_code": fields.get("country_code", ""),
|
|
"email": fields.get("email", ""),
|
|
"phone": fields.get("phone", ""),
|
|
"website": fields.get("website", ""),
|
|
"employees": int(fields.get("employees", 0)),
|
|
"founded_date": fields.get("founded_date", "1970-01-01"),
|
|
"registered_capital": float(fields.get("registered_capital", 0)),
|
|
"currency": fields.get("currency", ""),
|
|
"industry_code": fields.get("industry_code", ""),
|
|
"industry_description": fields.get("industry_description", ""),
|
|
"lei": fields.get("lei", ""),
|
|
"raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
|
|
}
|
|
self.client.execute(
|
|
"INSERT INTO companies VALUES",
|
|
[data],
|
|
)
|
|
|
|
def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
|
|
email: str = "", phone: str = "", source: str = "", confidence: float = 0):
|
|
self.client.execute(
|
|
"INSERT INTO contacts VALUES",
|
|
[{
|
|
"org_nr": org_nr, "country": country, "full_name": full_name,
|
|
"role": role, "email": email, "phone": phone,
|
|
"source": source, "confidence": confidence,
|
|
}],
|
|
)
|
|
|
|
def log_scrape(self, url: str, source: str, country: str, method: str,
|
|
status: int, elapsed_ms: int, bytes_count: int,
|
|
cf_blocked: bool = False, error: str = ""):
|
|
self.client.execute(
|
|
"INSERT INTO scrape_log VALUES",
|
|
[{
|
|
"url": url[:500], "source": source, "country": country,
|
|
"method": method, "status": status, "elapsed_ms": elapsed_ms,
|
|
"bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
|
|
}],
|
|
)
|
|
|
|
def query(self, sql: str) -> list:
|
|
"""Execute SQL query and return results."""
|
|
return self.client.execute(sql)
|
|
|
|
def close(self):
|
|
if self.client:
|
|
self.client.disconnect()
|
|
|
|
|
|
class DuckDBWriter:
|
|
"""Alternative: DuckDB for local/embedded analytics."""
|
|
|
|
def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
|
|
import duckdb
|
|
self.db_path = db_path
|
|
self.conn = duckdb.connect(db_path)
|
|
self._ensure_tables()
|
|
|
|
def _ensure_tables(self):
|
|
self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS companies AS
|
|
SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
|
|
""")
|
|
self.conn.execute("""
|
|
CREATE TABLE IF NOT EXISTS scrape_log (
|
|
url VARCHAR, source VARCHAR, country VARCHAR,
|
|
method VARCHAR, status INTEGER, elapsed_ms INTEGER,
|
|
bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
|
|
)
|
|
""")
|
|
|
|
def import_parquet(self, parquet_dir: str = "/data/parquet"):
|
|
"""Import all parquet files into DuckDB."""
|
|
import glob
|
|
files = glob.glob(f"{parquet_dir}/*.parquet")
|
|
if not files:
|
|
return 0
|
|
self.conn.execute(f"""
|
|
INSERT INTO companies
|
|
SELECT * FROM read_parquet({files!r})
|
|
""")
|
|
return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
|
|
|
|
def query(self, sql: str) -> list:
|
|
return self.conn.execute(sql).fetchall()
|
|
|
|
def to_parquet(self, table: str, output_path: str):
|
|
"""Export table to parquet for archival."""
|
|
self.conn.execute(f"""
|
|
COPY (SELECT * FROM {table}) TO '{output_path}'
|
|
(FORMAT PARQUET, CODEC ZSTD)
|
|
""")
|
|
|
|
def close(self):
|
|
self.conn.close()
|