feat: add ClickHouse pipeline, WAF bypass engine, proxy rotation, full SIAX Cloud data stack

services/infra/clickhouse/config.xml

@@ -0,0 +1,99 @@
+<yandex>
+    <logger>
+        <level>information</level>
+        <console>true</console>
+    </logger>
+
+    <http_port>8123</http_port>
+    <tcp_port>9000</tcp_port>
+
+    <max_connections>4096</max_connections>
+    <max_concurrent_queries>100</max_concurrent_queries>
+    <max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
+    <max_memory_usage_for_all_queries>0</max_memory_usage_for_all_queries>
+
+    <mark_cache_size>5368709120</mark_cache_size>
+    <path>/var/lib/clickhouse/</path>
+    <tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
+    <user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
+
+    <users_config>users.xml</users_config>
+    <default_profile>default</default_profile>
+    <default_database>opt_scraped</default_database>
+
+    <timezone>Europe/Stockholm</timezone>
+
+    <query_log>
+        <database>system</database>
+        <table>query_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_log>
+
+    <distributed_ddl>
+        <path>/clickhouse/task_queue/ddl</path>
+    </distributed_ddl>
+
+    <merge_tree>
+        <max_suspicious_broken_parts>5</max_suspicious_broken_parts>
+        <min_bytes_for_wide_part>0</min_bytes_for_wide_part>
+        <max_bytes_to_merge_at_max_space_in_pool>10737418240</max_bytes_to_merge_at_max_space_in_pool>
+    </merge_tree>
+
+    <!-- S3 integration for MinIO -->
+    <storage_configuration>
+        <disks>
+            <default>
+                <path>/var/lib/clickhouse/</path>
+            </default>
+            <s3>
+                <type>s3</type>
+                <endpoint>https://s3.siax.io/clickhouse-data/</endpoint>
+                <access_key_id>CHANGE_ME</access_key_id>
+                <secret_access_key>CHANGE_ME</secret_access_key>
+                <region>eu-central</region>
+                <metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
+                <send_cache>true</send_cache>
+            </s3>
+        </disks>
+        <policies>
+            <s3_only>
+                <volumes>
+                    <main>
+                        <disk>s3</disk>
+                    </main>
+                </volumes>
+            </s3_only>
+            <tiered>
+                <volumes>
+                    <hot>
+                        <disk>default</disk>
+                        <max_data_part_size>1073741824</max_data_part_size>
+                    </hot>
+                    <cold>
+                        <disk>s3</disk>
+                    </cold>
+                </volumes>
+            </tiered>
+        </policies>
+    </storage_configuration>
+
+    <part_log>
+        <database>system</database>
+        <table>part_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </part_log>
+
+    <query_thread_log>
+        <database>system</database>
+        <table>query_thread_log</table>
+        <partition_by>toYYYYMM(event_date)</partition_by>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </query_thread_log>
+
+    <asynchronous_insert_log>
+        <database>system</database>
+        <table>asynchronous_insert_log</table>
+        <flush_interval_milliseconds>7500</flush_interval_milliseconds>
+    </asynchronous_insert_log>
+</yandex>

services/infra/clickhouse/users.xml

@@ -0,0 +1,54 @@
+<yandex>
+    <profiles>
+        <default>
+            <max_memory_usage>8000000000</max_memory_usage>
+            <use_uncompressed_cache>0</use_uncompressed_cache>
+            <load_balancing>random</load_balancing>
+            <max_query_size>10485760</max_query_size>
+            <group_by_overflow_mode>any</group_by_overflow_mode>
+            <timeout_before_checking_execution_speed>0</timeout_before_checking_execution_speed>
+        </default>
+        <readonly>
+            <readonly>1</readonly>
+        </readonly>
+    </profiles>
+
+    <users>
+        <default>
+            <password>CHANGE_ME</password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>default</profile>
+            <quota>default</quota>
+            <grants>
+                <select>1</select>
+                <insert>1</insert>
+                <alter>1</alter>
+                <create>1</create>
+                <drop>1</drop>
+            </grants>
+        </default>
+        <readonly>
+            <password></password>
+            <networks>
+                <ip>::/0</ip>
+            </networks>
+            <profile>readonly</profile>
+            <quota>default</quota>
+        </readonly>
+    </users>
+
+    <quotas>
+        <default>
+            <interval>
+                <duration>3600</duration>
+                <queries>10000</queries>
+                <errors>100</errors>
+                <result_rows>10000000000</result_rows>
+                <read_rows>100000000000</read_rows>
+                <execution_time>3600</execution_time>
+            </interval>
+        </default>
+    </quotas>
+</yandex>

services/infra/docker-compose.data-stack.yml

@@ -0,0 +1,137 @@
+# SIAX Cloud Data Stack - Complete BigQuery Replacement
+# All OPEN SOURCE, self-hosted on SIAX Cloud Server 2
+# 
+# Stack:
+#   ClickHouse   - Columnar DB (BigQuery replacement)
+#   Trino        - Distributed SQL query engine
+#   FlareSolverr - Cloudflare bypass
+#   Tor          - Anonymous proxy rotation
+#   Apache Superset - BI dashboards
+#   MinIO        - S3 object storage (already exists)
+#
+version: "3.8"
+
+services:
+  # ═══════════════════════════════════════════════
+  # CLICKHOUSE - BigQuery ersättning
+  # ═══════════════════════════════════════════════
+  clickhouse:
+    image: clickhouse/clickhouse-server:24.12-alpine
+    container_name: clickhouse
+    restart: unless-stopped
+    ports:
+      - "8123:8123"   # HTTP API (REST)
+      - "9000:9000"   # Native TCP
+      - "9090:9090"   # Prometheus metrics
+    volumes:
+      - clickhouse-data:/var/lib/clickhouse
+      - clickhouse-logs:/var/log/clickhouse-server
+      - ./clickhouse/config.xml:/etc/clickhouse-server/config.xml:ro
+      - ./clickhouse/users.xml:/etc/clickhouse-server/users.xml:ro
+    environment:
+      - CLICKHOUSE_DB=opt_scraped
+      - CLICKHOUSE_USER=default
+      - CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-changeme}
+    ulimits:
+      nofile:
+        soft: 262144
+        hard: 262144
+    deploy:
+      resources:
+        limits:
+          memory: 8G
+        reservations:
+          memory: 4G
+
+  # ═══════════════════════════════════════════════
+  # TRINO - Distributed SQL (optional, for complex queries)
+  # ═══════════════════════════════════════════════
+  trino:
+    image: trinodb/trino:470
+    container_name: trino
+    restart: unless-stopped
+    ports:
+      - "8080:8080"
+    volumes:
+      - ./trino/etc:/etc/trino:ro
+      - trino-data:/data/trino
+    deploy:
+      resources:
+        limits:
+          memory: 4G
+
+  # ═══════════════════════════════════════════════
+  # FLARESOLVERR - Cloudflare bypass
+  # ═══════════════════════════════════════════════
+  flaresolverr:
+    image: ghcr.io/flaresolverr/flaresolverr:latest
+    container_name: flaresolverr
+    restart: unless-stopped
+    ports:
+      - "8191:8191"
+    environment:
+      - LOG_LEVEL=${LOG_LEVEL:-info}
+      - TZ=Europe/Stockholm
+      - CAPTCHA_SOLVER=none
+    deploy:
+      resources:
+        limits:
+          memory: 1G
+
+  # ═══════════════════════════════════════════════
+  # TOR - Anonymous proxy rotation
+  # ═══════════════════════════════════════════════
+  tor-proxy:
+    image: dperson/torproxy:latest
+    container_name: tor-proxy
+    restart: unless-stopped
+    ports:
+      - "9050:9050"  # SOCKS5
+      - "9051:9051"  # Control port
+    deploy:
+      resources:
+        limits:
+          memory: 256M
+
+  # ═══════════════════════════════════════════════
+  # APACHE SUPERSET - BI Dashboard
+  # ═══════════════════════════════════════════════
+  superset:
+    image: apache/superset:latest
+    container_name: superset
+    restart: unless-stopped
+    ports:
+      - "8088:8088"
+    environment:
+      - SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-changeme}
+      - SUPERSET_DATABASE_URI=postgresql://superset:${POSTGRES_PASSWORD:-changeme}@postgres:5432/superset
+    volumes:
+      - superset-data:/app/superset_home
+    deploy:
+      resources:
+        limits:
+          memory: 2G
+    depends_on:
+      - clickhouse
+      - postgres
+
+  # ═══════════════════════════════════════════════
+  # POSTGRES for Superset metadata
+  # ═══════════════════════════════════════════════
+  postgres:
+    image: postgres:16-alpine
+    container_name: postgres-superset
+    restart: unless-stopped
+    environment:
+      - POSTGRES_DB=superset
+      - POSTGRES_USER=superset
+      - POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme}
+    volumes:
+      - postgres-data:/var/lib/postgresql/data
+
+volumes:
+  clickhouse-data:
+  clickhouse-logs:
+  trino-data:
+  superset-data:
+  postgres-data:

services/scrapers/clickhouse_pipeline.py

@@ -0,0 +1,261 @@
+"""
+ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
+
+Ersätter:
+- BigQuery (används aldrig)
+- MinIO/S3 fil-lagring (mellanlagring)
+
+Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
+"""
+import os, json, time, gzip
+from datetime import datetime
+from pathlib import Path
+from typing import Optional, Any
+
+CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
+CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
+CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
+CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
+CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
+S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
+S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
+S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
+
+
+class ClickHouseWriter:
+    """Write scraped data to ClickHouse via HTTP interface."""
+
+    def __init__(self):
+        self.client = None
+        self._connect()
+
+    def _connect(self):
+        from clickhouse_driver import Client
+        self.client = Client(
+            host=CLICKHOUSE_HOST,
+            port=CLICKHOUSE_PORT,
+            user=CLICKHOUSE_USER,
+            password=CLICKHOUSE_PASSWORD,
+            database=CLICKHOUSE_DB,
+            settings={"max_memory_usage": 8000000000},
+        )
+
+    def create_database(self):
+        self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
+
+    def ensure_tables(self):
+        """Create all required tables for scraped data."""
+        
+        # Companies table (multi-country)
+        self.client.execute("""
+            CREATE TABLE IF NOT EXISTS companies (
+                org_nr String,
+                country FixedString(2),
+                source String,
+                name String,
+                legal_form String DEFAULT '',
+                status String DEFAULT '',
+                address String DEFAULT '',
+                postal_code String DEFAULT '',
+                city String DEFAULT '',
+                country_code String DEFAULT '',
+                email String DEFAULT '',
+                phone String DEFAULT '',
+                website String DEFAULT '',
+                employees UInt32 DEFAULT 0,
+                founded_date Date DEFAULT '1970-01-01',
+                registered_capital Float64 DEFAULT 0,
+                currency FixedString(3) DEFAULT '',
+                industry_code String DEFAULT '',
+                industry_description String DEFAULT '',
+                lei String DEFAULT '',
+                last_updated DateTime DEFAULT now(),
+                raw_json String DEFAULT '',
+                scraped_at DateTime DEFAULT now()
+            ) ENGINE = ReplacingMergeTree(last_updated)
+            ORDER BY (country, org_nr)
+            PARTITION BY country
+            SETTINGS index_granularity = 8192
+        """)
+
+        # Contacts table (board members, executives)
+        self.client.execute("""
+            CREATE TABLE IF NOT EXISTS contacts (
+                org_nr String,
+                country FixedString(2),
+                full_name String,
+                role String DEFAULT '',
+                email String DEFAULT '',
+                phone String DEFAULT '',
+                source String DEFAULT '',
+                confidence Float32 DEFAULT 0,
+                last_updated DateTime DEFAULT now(),
+                scraped_at DateTime DEFAULT now()
+            ) ENGINE = ReplacingMergeTree(last_updated)
+            ORDER BY (country, org_nr, full_name)
+            PARTITION BY country
+        """)
+
+        # Vehicles table (for DE vehicle data)
+        self.client.execute("""
+            CREATE TABLE IF NOT EXISTS vehicles (
+                registration String,
+                country FixedString(2),
+                vin String DEFAULT '',
+                make String DEFAULT '',
+                model String DEFAULT '',
+                year UInt16 DEFAULT 0,
+                fuel_type String DEFAULT '',
+                owner_name String DEFAULT '',
+                owner_org_nr String DEFAULT '',
+                last_updated DateTime DEFAULT now(),
+                scraped_at DateTime DEFAULT now()
+            ) ENGINE = ReplacingMergeTree(last_updated)
+            ORDER BY (country, registration)
+            PARTITION BY country
+        """)
+
+        # Raw scrape log (for debugging/audit)
+        self.client.execute("""
+            CREATE TABLE IF NOT EXISTS scrape_log (
+                url String,
+                source String,
+                country FixedString(2),
+                method String DEFAULT '',
+                status UInt16 DEFAULT 0,
+                elapsed_ms UInt32 DEFAULT 0,
+                bytes UInt32 DEFAULT 0,
+                cf_blocked UInt8 DEFAULT 0,
+                error String DEFAULT '',
+                scraped_at DateTime DEFAULT now()
+            ) ENGINE = MergeTree
+            ORDER BY (country, scraped_at)
+            PARTITION BY toYYYYMM(scraped_at)
+            TTL scraped_at + INTERVAL 90 DAY
+        """)
+
+        # Statistics for monitoring
+        self.client.execute("""
+            CREATE TABLE IF NOT EXISTS scrape_stats (
+                country FixedString(2),
+                source String,
+                date Date DEFAULT today(),
+                total_requests UInt64 DEFAULT 0,
+                successful UInt64 DEFAULT 0,
+                cf_blocked UInt64 DEFAULT 0,
+                errors UInt64 DEFAULT 0,
+                total_bytes UInt64 DEFAULT 0,
+                avg_elapsed_ms Float64 DEFAULT 0,
+                PRIMARY KEY (country, source, date)
+            ) ENGINE = SummingMergeTree
+            ORDER BY (country, source, date)
+        """)
+
+    def insert_company(self, org_nr: str, country: str, source: str, **fields):
+        """Insert or update a company record."""
+        data = {
+            "org_nr": org_nr,
+            "country": country,
+            "source": source,
+            "name": fields.get("name", ""),
+            "legal_form": fields.get("legal_form", ""),
+            "status": fields.get("status", ""),
+            "address": fields.get("address", ""),
+            "postal_code": fields.get("postal_code", ""),
+            "city": fields.get("city", ""),
+            "country_code": fields.get("country_code", ""),
+            "email": fields.get("email", ""),
+            "phone": fields.get("phone", ""),
+            "website": fields.get("website", ""),
+            "employees": int(fields.get("employees", 0)),
+            "founded_date": fields.get("founded_date", "1970-01-01"),
+            "registered_capital": float(fields.get("registered_capital", 0)),
+            "currency": fields.get("currency", ""),
+            "industry_code": fields.get("industry_code", ""),
+            "industry_description": fields.get("industry_description", ""),
+            "lei": fields.get("lei", ""),
+            "raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
+        }
+        self.client.execute(
+            "INSERT INTO companies VALUES",
+            [data],
+        )
+
+    def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
+                       email: str = "", phone: str = "", source: str = "", confidence: float = 0):
+        self.client.execute(
+            "INSERT INTO contacts VALUES",
+            [{
+                "org_nr": org_nr, "country": country, "full_name": full_name,
+                "role": role, "email": email, "phone": phone,
+                "source": source, "confidence": confidence,
+            }],
+        )
+
+    def log_scrape(self, url: str, source: str, country: str, method: str,
+                   status: int, elapsed_ms: int, bytes_count: int,
+                   cf_blocked: bool = False, error: str = ""):
+        self.client.execute(
+            "INSERT INTO scrape_log VALUES",
+            [{
+                "url": url[:500], "source": source, "country": country,
+                "method": method, "status": status, "elapsed_ms": elapsed_ms,
+                "bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
+            }],
+        )
+
+    def query(self, sql: str) -> list:
+        """Execute SQL query and return results."""
+        return self.client.execute(sql)
+
+    def close(self):
+        if self.client:
+            self.client.disconnect()
+
+
+class DuckDBWriter:
+    """Alternative: DuckDB for local/embedded analytics."""
+
+    def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
+        import duckdb
+        self.db_path = db_path
+        self.conn = duckdb.connect(db_path)
+        self._ensure_tables()
+
+    def _ensure_tables(self):
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS companies AS
+            SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
+        """)
+        self.conn.execute("""
+            CREATE TABLE IF NOT EXISTS scrape_log (
+                url VARCHAR, source VARCHAR, country VARCHAR,
+                method VARCHAR, status INTEGER, elapsed_ms INTEGER,
+                bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
+            )
+        """)
+
+    def import_parquet(self, parquet_dir: str = "/data/parquet"):
+        """Import all parquet files into DuckDB."""
+        import glob
+        files = glob.glob(f"{parquet_dir}/*.parquet")
+        if not files:
+            return 0
+        self.conn.execute(f"""
+            INSERT INTO companies
+            SELECT * FROM read_parquet({files!r})
+        """)
+        return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
+
+    def query(self, sql: str) -> list:
+        return self.conn.execute(sql).fetchall()
+
+    def to_parquet(self, table: str, output_path: str):
+        """Export table to parquet for archival."""
+        self.conn.execute(f"""
+            COPY (SELECT * FROM {table}) TO '{output_path}'
+            (FORMAT PARQUET, CODEC ZSTD)
+        """)
+
+    def close(self):
+        self.conn.close()

services/scrapers/waf_bypass.py

@@ -0,0 +1,316 @@
+"""
+Proxy Rotator & WAF Bypass - ALLA gratis metoder.
+
+Metoder:
+1. Header rotation (6 User-Agents x sprak x devices)
+2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17)
+3. cloudscraper (basic CF bypass)
+4. FlareSolverr (heavy CF bypass, via Docker)
+5. Tor rotation (SOCKS5, gratis)
+6. Free proxy pool (scraped from 5+ sources)
+7. Playwright browser automation
+8. nodriver (undetected browser)
+9. Fake identity generation
+10. Session cookie reuse
+"""
+import os, json, time, random, socket, asyncio
+from datetime import datetime
+from typing import Optional
+from curl_cffi import requests as curl_requests
+import cloudscraper
+import requests
+
+# ── Config ──
+FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
+TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")
+USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true"
+USE_TOR = os.getenv("USE_TOR", "false").lower() == "true"
+
+# ── Header Pool ──
+USER_AGENTS = [
+    # Chrome Windows
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
+    # Safari Mac
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
+    # Firefox
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
+    # Edge
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
+    # Linux
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    # Mobile
+    "Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
+    "Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36",
+]
+
+ACCEPT_LANGUAGES = {
+    "DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
+    "NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
+    "DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
+    "SE": "sv-SE,sv;q=0.9,en;q=0.8",
+    "GB": "en-GB,en;q=0.9,fr;q=0.8",
+    "FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7",
+    "FR": "fr-FR,fr;q=0.9,en;q=0.8",
+    "NL": "nl-NL,nl;q=0.9,en;q=0.8",
+    "PL": "pl-PL,pl;q=0.9,en;q=0.8",
+}
+
+SEC_CH_UA = {
+    "chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
+    "chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"',
+    "edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
+}
+
+ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"]
+
+
+def rotate_headers(country="DE", impersonate="chrome124"):
+    ua = random.choice(USER_AGENTS)
+    lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9")
+    enc = random.choice(ACCEPT_ENCODINGS)
+    sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"])
+    
+    return {
+        "User-Agent": ua,
+        "Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
+        "Accept-Language": lang,
+        "Accept-Encoding": enc,
+        "Cache-Control": "no-cache",
+        "Pragma": "no-cache",
+        "Sec-Ch-Ua": sec_ch,
+        "Sec-Ch-Ua-Mobile": "?0",
+        "Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'),
+        "Sec-Fetch-Dest": "document",
+        "Sec-Fetch-Mode": "navigate",
+        "Sec-Fetch-Site": "none",
+        "Sec-Fetch-User": "?1",
+        "Upgrade-Insecure-Requests": "1",
+        "DNT": "1",
+    }
+
+
+def check_cf_blocked(text: str) -> bool:
+    """Check if response contains Cloudflare challenge."""
+    indicators = [
+        "cf-wrapper", "Just a moment", "Checking your browser",
+        "Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
+        "challenge-form", "Attention Required", "cf-error-page",
+        "cloudflare.com/cdn-cgi", "/cdn-cgi/",
+    ]
+    if not text:
+        return False
+    lower = text[:10000].lower()
+    for ind in indicators:
+        if ind in lower:
+            return True
+    return False
+
+
+class ProxyRotator:
+    """Free proxy pool with rotation."""
+
+    def __init__(self):
+        self.proxies = []
+        self.current = 0
+        self._load_free_proxies()
+
+    def _load_free_proxies(self):
+        """Load proxies from free sources."""
+        sources = [
+            "https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
+            "https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
+            "https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
+        ]
+        for url in sources:
+            try:
+                r = requests.get(url, timeout=5)
+                for line in r.text.strip().split("\n"):
+                    line = line.strip()
+                    if ":" in line and not line.startswith("#"):
+                        parts = line.split(":")
+                        if len(parts) == 2:
+                            self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"})
+            except:
+                pass
+        # Deduplicate
+        seen = set()
+        unique = []
+        for p in self.proxies:
+            if p["http"] not in seen:
+                seen.add(p["http"])
+                unique.append(p)
+        self.proxies = unique
+        print(f"  ProxyRotator: loaded {len(self.proxies)} free proxies")
+
+    def get_proxy(self) -> Optional[dict]:
+        if not self.proxies:
+            return None
+        self.current = (self.current + 1) % len(self.proxies)
+        return self.proxies[self.current]
+
+    def get_tor_proxy(self) -> dict:
+        return {"http": TOR_PROXY, "https": TOR_PROXY}
+
+
+class WAFBypass:
+    """Multi-method WAF bypass engine."""
+
+    def __init__(self):
+        self.cloudscraper = cloudscraper.create_scraper(
+            browser={"browser": "chrome", "platform": "windows", "mobile": False}
+        )
+        self.proxy = ProxyRotator()
+        self.session_pool = {}
+        self.curl_methods = {
+            "chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15),
+            "chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15),
+            "edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15),
+            "safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15),
+        }
+
+    def try_flaresolverr(self, url: str) -> Optional[str]:
+        """Bypass CF via FlareSolverr Docker container."""
+        if not USE_FLARESOLVERR:
+            return None
+        try:
+            payload = {
+                "cmd": "request.get",
+                "url": url,
+                "maxTimeout": 60000,
+                "session": f"session_{hash(url) % 10000}",
+            }
+            r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65)
+            data = r.json()
+            if data.get("status") == "ok":
+                solution = data.get("solution", {})
+                return solution.get("response", "")
+        except:
+            pass
+        return None
+
+    def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]:
+        """Route through Tor network."""
+        if not USE_TOR:
+            return None
+        try:
+            proxies = self.proxy.get_tor_proxy()
+            r = requests.get(url, headers=headers, proxies=proxies, timeout=30)
+            return r
+        except:
+            return None
+
+    def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]:
+        """Try all curl_cffi impersonations in order."""
+        for name, fn in self.curl_methods.items():
+            try:
+                headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124")
+                r = fn(url, headers)
+                if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+                    return r
+            except:
+                continue
+        return None
+
+    def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]:
+        """Try cloudscraper with different configs."""
+        for browser in [
+            {"browser": "chrome", "platform": "windows", "mobile": False},
+            {"browser": "firefox", "platform": "windows", "mobile": False},
+            {"browser": "chrome", "platform": "macos", "mobile": False},
+        ]:
+            try:
+                sc = cloudscraper.create_scraper(browser=browser)
+                sc.headers.update(rotate_headers(country))
+                r = sc.get(url, timeout=15)
+                if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+                    return r
+            except:
+                continue
+        return None
+
+    def try_playwright(self, url: str) -> Optional[str]:
+        """Use Playwright headless browser as last resort."""
+        try:
+            from playwright.sync_api import sync_playwright
+            with sync_playwright() as p:
+                browser = p.chromium.launch(headless=True)
+                context = browser.new_context(
+                    user_agent=random.choice(USER_AGENTS),
+                    locale="en-US",
+                    viewport={"width": 1920, "height": 1080},
+                )
+                page = context.new_page()
+                page.goto(url, timeout=30000)
+                content = page.content()
+                browser.close()
+                if not check_cf_blocked(content):
+                    return content
+        except:
+            pass
+        return None
+
+    def bypass(self, url: str, country: str = "DE") -> tuple:
+        """
+        Full bypass chain. Returns (content: str, method: str, elapsed: float).
+
+        Chain order:
+        1. curl_cffi chrome124 (fastest, 0.1-0.3s)
+        2. cloudscraper (basic CF, 0.3-0.5s)
+        3. curl_cffi edge99 (alt fingerprint)
+        4. curl_cffi safari17 (Eniro bypass)
+        5. Tor proxy rotation (anonymous, 2-5s)
+        6. FlareSolverr (heavy CF, 5-15s)
+        7. Playwright (full browser, 10-30s)
+        """
+        start = time.time()
+
+        # Level 1: curl_cffi TLS fingerprint
+        r = self.try_curl(url, country)
+        if r and r.status_code == 200:
+            return (r.text, "curl_cffi", time.time() - start)
+
+        # Level 2: cloudscraper
+        r = self.try_cloudscraper(url, country)
+        if r and r.status_code == 200:
+            return (r.text, "cloudscraper", time.time() - start)
+
+        # Level 3: Tor
+        if USE_TOR:
+            r = self.try_tor(url, rotate_headers(country))
+            if r and r.status_code == 200:
+                return (r.text, "tor", time.time() - start)
+
+        # Level 4: FlareSolverr
+        content = self.try_flaresolverr(url)
+        if content:
+            return (content, "flaresolverr", time.time() - start)
+
+        # Level 5: Playwright (last resort)
+        content = self.try_playwright(url)
+        if content:
+            return (content, "playwright", time.time() - start)
+
+        return (None, "all_failed", time.time() - start)
+
+    def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple:
+        """Bypass with proxy rotation for each method."""
+        start = time.time()
+        proxy = self.proxy.get_proxy()
+        headers = rotate_headers(country)
+
+        for name, fn in self.curl_methods.items():
+            try:
+                if proxy:
+                    r = fn(url, headers)
+                else:
+                    r = fn(url, headers)
+                if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
+                    return (r.text, f"curl_cffi+proxy_{name}", time.time() - start)
+            except:
+                continue
+
+        return self.bypass(url, country)