admin/0pt
5
0
Fork 0
Code Issues 9 Pull Requests 1 Actions Packages Projects Releases Wiki Activity

feat: add ClickHouse pipeline, WAF bypass engine, proxy rotation, full SIAX Cloud data stack
Scraper CI / lint (push) Successful in 5s
Details

Browse Source
This commit is contained in:
admin
2026-07-13 13:08:56 +02:00
parent afd301facb
commit bb6be0f62b
5 changed files with 867 additions and 0 deletions
Download Patch File Download Diff File Expand all files Collapse all files
services/infra/clickhouse/config.xml
+99
View File
@@ -0,0 +1,99 @@
<yandex>
<logger>
<level>information</level>
<console>true</console>
</logger>
<http_port>8123</http_port>
<tcp_port>9000</tcp_port>
<max_connections>4096</max_connections>
<max_concurrent_queries>100</max_concurrent_queries>
<max_server_memory_usage_to_ram_ratio>0.75</max_server_memory_usage_to_ram_ratio>
<max_memory_usage_for_all_queries>0</max_memory_usage_for_all_queries>
<mark_cache_size>5368709120</mark_cache_size>
<path>/var/lib/clickhouse/</path>
<tmp_path>/var/lib/clickhouse/tmp/</tmp_path>
<user_files_path>/var/lib/clickhouse/user_files/</user_files_path>
<users_config>users.xml</users_config>
<default_profile>default</default_profile>
<default_database>opt_scraped</default_database>
<timezone>Europe/Stockholm</timezone>
<query_log>
<database>system</database>
<table>query_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_log>
<distributed_ddl>
<path>/clickhouse/task_queue/ddl</path>
</distributed_ddl>
<merge_tree>
<max_suspicious_broken_parts>5</max_suspicious_broken_parts>
<min_bytes_for_wide_part>0</min_bytes_for_wide_part>
<max_bytes_to_merge_at_max_space_in_pool>10737418240</max_bytes_to_merge_at_max_space_in_pool>
</merge_tree>
<!-- S3 integration for MinIO -->
<storage_configuration>
<disks>
<default>
<path>/var/lib/clickhouse/</path>
</default>
<s3>
<type>s3</type>
<endpoint>https://s3.siax.io/clickhouse-data/</endpoint>
<access_key_id>CHANGE_ME</access_key_id>
<secret_access_key>CHANGE_ME</secret_access_key>
<region>eu-central</region>
<metadata_path>/var/lib/clickhouse/disks/s3/</metadata_path>
<send_cache>true</send_cache>
</s3>
</disks>
<policies>
<s3_only>
<volumes>
<main>
<disk>s3</disk>
</main>
</volumes>
</s3_only>
<tiered>
<volumes>
<hot>
<disk>default</disk>
<max_data_part_size>1073741824</max_data_part_size>
</hot>
<cold>
<disk>s3</disk>
</cold>
</volumes>
</tiered>
</policies>
</storage_configuration>
<part_log>
<database>system</database>
<table>part_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</part_log>
<query_thread_log>
<database>system</database>
<table>query_thread_log</table>
<partition_by>toYYYYMM(event_date)</partition_by>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</query_thread_log>
<asynchronous_insert_log>
<database>system</database>
<table>asynchronous_insert_log</table>
<flush_interval_milliseconds>7500</flush_interval_milliseconds>
</asynchronous_insert_log>
</yandex>
services/infra/clickhouse/users.xml
+54
View File
@@ -0,0 +1,54 @@
<yandex>
<profiles>
<default>
<max_memory_usage>8000000000</max_memory_usage>
<use_uncompressed_cache>0</use_uncompressed_cache>
<load_balancing>random</load_balancing>
<max_query_size>10485760</max_query_size>
<group_by_overflow_mode>any</group_by_overflow_mode>
<timeout_before_checking_execution_speed>0</timeout_before_checking_execution_speed>
</default>
<readonly>
<readonly>1</readonly>
</readonly>
</profiles>
<users>
<default>
<password>CHANGE_ME</password>
<networks>
<ip>::/0</ip>
</networks>
<profile>default</profile>
<quota>default</quota>
<grants>
<select>1</select>
<insert>1</insert>
<alter>1</alter>
<create>1</create>
<drop>1</drop>
</grants>
</default>
<readonly>
<password></password>
<networks>
<ip>::/0</ip>
</networks>
<profile>readonly</profile>
<quota>default</quota>
</readonly>
</users>
<quotas>
<default>
<interval>
<duration>3600</duration>
<queries>10000</queries>
<errors>100</errors>
<result_rows>10000000000</result_rows>
<read_rows>100000000000</read_rows>
<execution_time>3600</execution_time>
</interval>
</default>
</quotas>
</yandex>
services/infra/docker-compose.data-stack.yml
+137
View File
@@ -0,0 +1,137 @@
# SIAX Cloud Data Stack - Complete BigQuery Replacement
# All OPEN SOURCE, self-hosted on SIAX Cloud Server 2
#
# Stack:
# ClickHouse - Columnar DB (BigQuery replacement)
# Trino - Distributed SQL query engine
# FlareSolverr - Cloudflare bypass
# Tor - Anonymous proxy rotation
# Apache Superset - BI dashboards
# MinIO - S3 object storage (already exists)
#
version: "3.8"
services:
# ═══════════════════════════════════════════════
# CLICKHOUSE - BigQuery ersättning
# ═══════════════════════════════════════════════
clickhouse:
image: clickhouse/clickhouse-server:24.12-alpine
container_name: clickhouse
restart: unless-stopped
ports:
- "8123:8123" # HTTP API (REST)
- "9000:9000" # Native TCP
- "9090:9090" # Prometheus metrics
volumes:
- clickhouse-data:/var/lib/clickhouse
- clickhouse-logs:/var/log/clickhouse-server
- ./clickhouse/config.xml:/etc/clickhouse-server/config.xml:ro
- ./clickhouse/users.xml:/etc/clickhouse-server/users.xml:ro
environment:
- CLICKHOUSE_DB=opt_scraped
- CLICKHOUSE_USER=default
- CLICKHOUSE_PASSWORD=${CLICKHOUSE_PASSWORD:-changeme}
ulimits:
nofile:
soft: 262144
hard: 262144
deploy:
resources:
limits:
memory: 8G
reservations:
memory: 4G
# ═══════════════════════════════════════════════
# TRINO - Distributed SQL (optional, for complex queries)
# ═══════════════════════════════════════════════
trino:
image: trinodb/trino:470
container_name: trino
restart: unless-stopped
ports:
- "8080:8080"
volumes:
- ./trino/etc:/etc/trino:ro
- trino-data:/data/trino
deploy:
resources:
limits:
memory: 4G
# ═══════════════════════════════════════════════
# FLARESOLVERR - Cloudflare bypass
# ═══════════════════════════════════════════════
flaresolverr:
image: ghcr.io/flaresolverr/flaresolverr:latest
container_name: flaresolverr
restart: unless-stopped
ports:
- "8191:8191"
environment:
- LOG_LEVEL=${LOG_LEVEL:-info}
- TZ=Europe/Stockholm
- CAPTCHA_SOLVER=none
deploy:
resources:
limits:
memory: 1G
# ═══════════════════════════════════════════════
# TOR - Anonymous proxy rotation
# ═══════════════════════════════════════════════
tor-proxy:
image: dperson/torproxy:latest
container_name: tor-proxy
restart: unless-stopped
ports:
- "9050:9050" # SOCKS5
- "9051:9051" # Control port
deploy:
resources:
limits:
memory: 256M
# ═══════════════════════════════════════════════
# APACHE SUPERSET - BI Dashboard
# ═══════════════════════════════════════════════
superset:
image: apache/superset:latest
container_name: superset
restart: unless-stopped
ports:
- "8088:8088"
environment:
- SUPERSET_SECRET_KEY=${SUPERSET_SECRET_KEY:-changeme}
- SUPERSET_DATABASE_URI=postgresql://superset:${POSTGRES_PASSWORD:-changeme}@postgres:5432/superset
volumes:
- superset-data:/app/superset_home
deploy:
resources:
limits:
memory: 2G
depends_on:
- clickhouse
- postgres
# ═══════════════════════════════════════════════
# POSTGRES for Superset metadata
# ═══════════════════════════════════════════════
postgres:
image: postgres:16-alpine
container_name: postgres-superset
restart: unless-stopped
environment:
- POSTGRES_DB=superset
- POSTGRES_USER=superset
- POSTGRES_PASSWORD=${POSTGRES_PASSWORD:-changeme}
volumes:
- postgres-data:/var/lib/postgresql/data
volumes:
clickhouse-data:
clickhouse-logs:
trino-data:
superset-data:
postgres-data:
services/scrapers/clickhouse_pipeline.py
+261
View File
@@ -0,0 +1,261 @@
"""
ClickHouse Data Pipeline - Skickar scrapad data direkt till ClickHouse.
Ersätter:
- BigQuery (används aldrig)
- MinIO/S3 fil-lagring (mellanlagring)
Pipeline: Scraper -> Parquet -> ClickHouse (columnar OLAP)
"""
import os, json, time, gzip
from datetime import datetime
from pathlib import Path
from typing import Optional, Any
CLICKHOUSE_HOST = os.getenv("CLICKHOUSE_HOST", "localhost")
CLICKHOUSE_PORT = int(os.getenv("CLICKHOUSE_PORT", "8123"))
CLICKHOUSE_USER = os.getenv("CLICKHOUSE_USER", "default")
CLICKHOUSE_PASSWORD = os.getenv("CLICKHOUSE_PASSWORD", "")
CLICKHOUSE_DB = os.getenv("CLICKHOUSE_DB", "opt_scraped")
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
S3_SECRET_KEY = os.getenv("S3_SECRET_KEY", "")
class ClickHouseWriter:
"""Write scraped data to ClickHouse via HTTP interface."""
def __init__(self):
self.client = None
self._connect()
def _connect(self):
from clickhouse_driver import Client
self.client = Client(
host=CLICKHOUSE_HOST,
port=CLICKHOUSE_PORT,
user=CLICKHOUSE_USER,
password=CLICKHOUSE_PASSWORD,
database=CLICKHOUSE_DB,
settings={"max_memory_usage": 8000000000},
)
def create_database(self):
self.client.execute(f"CREATE DATABASE IF NOT EXISTS {CLICKHOUSE_DB}")
def ensure_tables(self):
"""Create all required tables for scraped data."""
# Companies table (multi-country)
self.client.execute("""
CREATE TABLE IF NOT EXISTS companies (
org_nr String,
country FixedString(2),
source String,
name String,
legal_form String DEFAULT '',
status String DEFAULT '',
address String DEFAULT '',
postal_code String DEFAULT '',
city String DEFAULT '',
country_code String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
website String DEFAULT '',
employees UInt32 DEFAULT 0,
founded_date Date DEFAULT '1970-01-01',
registered_capital Float64 DEFAULT 0,
currency FixedString(3) DEFAULT '',
industry_code String DEFAULT '',
industry_description String DEFAULT '',
lei String DEFAULT '',
last_updated DateTime DEFAULT now(),
raw_json String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr)
PARTITION BY country
SETTINGS index_granularity = 8192
""")
# Contacts table (board members, executives)
self.client.execute("""
CREATE TABLE IF NOT EXISTS contacts (
org_nr String,
country FixedString(2),
full_name String,
role String DEFAULT '',
email String DEFAULT '',
phone String DEFAULT '',
source String DEFAULT '',
confidence Float32 DEFAULT 0,
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, org_nr, full_name)
PARTITION BY country
""")
# Vehicles table (for DE vehicle data)
self.client.execute("""
CREATE TABLE IF NOT EXISTS vehicles (
registration String,
country FixedString(2),
vin String DEFAULT '',
make String DEFAULT '',
model String DEFAULT '',
year UInt16 DEFAULT 0,
fuel_type String DEFAULT '',
owner_name String DEFAULT '',
owner_org_nr String DEFAULT '',
last_updated DateTime DEFAULT now(),
scraped_at DateTime DEFAULT now()
) ENGINE = ReplacingMergeTree(last_updated)
ORDER BY (country, registration)
PARTITION BY country
""")
# Raw scrape log (for debugging/audit)
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url String,
source String,
country FixedString(2),
method String DEFAULT '',
status UInt16 DEFAULT 0,
elapsed_ms UInt32 DEFAULT 0,
bytes UInt32 DEFAULT 0,
cf_blocked UInt8 DEFAULT 0,
error String DEFAULT '',
scraped_at DateTime DEFAULT now()
) ENGINE = MergeTree
ORDER BY (country, scraped_at)
PARTITION BY toYYYYMM(scraped_at)
TTL scraped_at + INTERVAL 90 DAY
""")
# Statistics for monitoring
self.client.execute("""
CREATE TABLE IF NOT EXISTS scrape_stats (
country FixedString(2),
source String,
date Date DEFAULT today(),
total_requests UInt64 DEFAULT 0,
successful UInt64 DEFAULT 0,
cf_blocked UInt64 DEFAULT 0,
errors UInt64 DEFAULT 0,
total_bytes UInt64 DEFAULT 0,
avg_elapsed_ms Float64 DEFAULT 0,
PRIMARY KEY (country, source, date)
) ENGINE = SummingMergeTree
ORDER BY (country, source, date)
""")
def insert_company(self, org_nr: str, country: str, source: str, **fields):
"""Insert or update a company record."""
data = {
"org_nr": org_nr,
"country": country,
"source": source,
"name": fields.get("name", ""),
"legal_form": fields.get("legal_form", ""),
"status": fields.get("status", ""),
"address": fields.get("address", ""),
"postal_code": fields.get("postal_code", ""),
"city": fields.get("city", ""),
"country_code": fields.get("country_code", ""),
"email": fields.get("email", ""),
"phone": fields.get("phone", ""),
"website": fields.get("website", ""),
"employees": int(fields.get("employees", 0)),
"founded_date": fields.get("founded_date", "1970-01-01"),
"registered_capital": float(fields.get("registered_capital", 0)),
"currency": fields.get("currency", ""),
"industry_code": fields.get("industry_code", ""),
"industry_description": fields.get("industry_description", ""),
"lei": fields.get("lei", ""),
"raw_json": json.dumps(fields.get("raw_data", {}), ensure_ascii=False),
}
self.client.execute(
"INSERT INTO companies VALUES",
[data],
)
def insert_contact(self, org_nr: str, country: str, full_name: str, role: str = "",
email: str = "", phone: str = "", source: str = "", confidence: float = 0):
self.client.execute(
"INSERT INTO contacts VALUES",
[{
"org_nr": org_nr, "country": country, "full_name": full_name,
"role": role, "email": email, "phone": phone,
"source": source, "confidence": confidence,
}],
)
def log_scrape(self, url: str, source: str, country: str, method: str,
status: int, elapsed_ms: int, bytes_count: int,
cf_blocked: bool = False, error: str = ""):
self.client.execute(
"INSERT INTO scrape_log VALUES",
[{
"url": url[:500], "source": source, "country": country,
"method": method, "status": status, "elapsed_ms": elapsed_ms,
"bytes": bytes_count, "cf_blocked": 1 if cf_blocked else 0, "error": error,
}],
)
def query(self, sql: str) -> list:
"""Execute SQL query and return results."""
return self.client.execute(sql)
def close(self):
if self.client:
self.client.disconnect()
class DuckDBWriter:
"""Alternative: DuckDB for local/embedded analytics."""
def __init__(self, db_path: str = "/data/opt_scraped.duckdb"):
import duckdb
self.db_path = db_path
self.conn = duckdb.connect(db_path)
self._ensure_tables()
def _ensure_tables(self):
self.conn.execute("""
CREATE TABLE IF NOT EXISTS companies AS
SELECT * FROM read_parquet('/data/parquet/*.parquet') WHERE 1=0
""")
self.conn.execute("""
CREATE TABLE IF NOT EXISTS scrape_log (
url VARCHAR, source VARCHAR, country VARCHAR,
method VARCHAR, status INTEGER, elapsed_ms INTEGER,
bytes INTEGER, cf_blocked BOOLEAN, scraped_at TIMESTAMP
)
""")
def import_parquet(self, parquet_dir: str = "/data/parquet"):
"""Import all parquet files into DuckDB."""
import glob
files = glob.glob(f"{parquet_dir}/*.parquet")
if not files:
return 0
self.conn.execute(f"""
INSERT INTO companies
SELECT * FROM read_parquet({files!r})
""")
return self.conn.execute("SELECT count(*) FROM companies").fetchone()[0]
def query(self, sql: str) -> list:
return self.conn.execute(sql).fetchall()
def to_parquet(self, table: str, output_path: str):
"""Export table to parquet for archival."""
self.conn.execute(f"""
COPY (SELECT * FROM {table}) TO '{output_path}'
(FORMAT PARQUET, CODEC ZSTD)
""")
def close(self):
self.conn.close()
services/scrapers/waf_bypass.py
+316
View File
@@ -0,0 +1,316 @@
"""
Proxy Rotator & WAF Bypass - ALLA gratis metoder.
Metoder:
1. Header rotation (6 User-Agents x sprak x devices)
2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17)
3. cloudscraper (basic CF bypass)
4. FlareSolverr (heavy CF bypass, via Docker)
5. Tor rotation (SOCKS5, gratis)
6. Free proxy pool (scraped from 5+ sources)
7. Playwright browser automation
8. nodriver (undetected browser)
9. Fake identity generation
10. Session cookie reuse
"""
import os, json, time, random, socket, asyncio
from datetime import datetime
from typing import Optional
from curl_cffi import requests as curl_requests
import cloudscraper
import requests
# ── Config ──
FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")
USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true"
USE_TOR = os.getenv("USE_TOR", "false").lower() == "true"
# ── Header Pool ──
USER_AGENTS = [
# Chrome Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
# Safari Mac
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
# Firefox
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
# Edge
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
# Linux
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
# Mobile
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36",
]
ACCEPT_LANGUAGES = {
"DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
"NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
"DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
"SE": "sv-SE,sv;q=0.9,en;q=0.8",
"GB": "en-GB,en;q=0.9,fr;q=0.8",
"FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7",
"FR": "fr-FR,fr;q=0.9,en;q=0.8",
"NL": "nl-NL,nl;q=0.9,en;q=0.8",
"PL": "pl-PL,pl;q=0.9,en;q=0.8",
}
SEC_CH_UA = {
"chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"',
"edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
}
ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"]
def rotate_headers(country="DE", impersonate="chrome124"):
ua = random.choice(USER_AGENTS)
lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9")
enc = random.choice(ACCEPT_ENCODINGS)
sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"])
return {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": lang,
"Accept-Encoding": enc,
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Sec-Ch-Ua": sec_ch,
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'),
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"DNT": "1",
}
def check_cf_blocked(text: str) -> bool:
"""Check if response contains Cloudflare challenge."""
indicators = [
"cf-wrapper", "Just a moment", "Checking your browser",
"Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
"challenge-form", "Attention Required", "cf-error-page",
"cloudflare.com/cdn-cgi", "/cdn-cgi/",
]
if not text:
return False
lower = text[:10000].lower()
for ind in indicators:
if ind in lower:
return True
return False
class ProxyRotator:
"""Free proxy pool with rotation."""
def __init__(self):
self.proxies = []
self.current = 0
self._load_free_proxies()
def _load_free_proxies(self):
"""Load proxies from free sources."""
sources = [
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
"https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
]
for url in sources:
try:
r = requests.get(url, timeout=5)
for line in r.text.strip().split("\n"):
line = line.strip()
if ":" in line and not line.startswith("#"):
parts = line.split(":")
if len(parts) == 2:
self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"})
except:
pass
# Deduplicate
seen = set()
unique = []
for p in self.proxies:
if p["http"] not in seen:
seen.add(p["http"])
unique.append(p)
self.proxies = unique
print(f" ProxyRotator: loaded {len(self.proxies)} free proxies")
def get_proxy(self) -> Optional[dict]:
if not self.proxies:
return None
self.current = (self.current + 1) % len(self.proxies)
return self.proxies[self.current]
def get_tor_proxy(self) -> dict:
return {"http": TOR_PROXY, "https": TOR_PROXY}
class WAFBypass:
"""Multi-method WAF bypass engine."""
def __init__(self):
self.cloudscraper = cloudscraper.create_scraper(
browser={"browser": "chrome", "platform": "windows", "mobile": False}
)
self.proxy = ProxyRotator()
self.session_pool = {}
self.curl_methods = {
"chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15),
"chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15),
"edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15),
"safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15),
}
def try_flaresolverr(self, url: str) -> Optional[str]:
"""Bypass CF via FlareSolverr Docker container."""
if not USE_FLARESOLVERR:
return None
try:
payload = {
"cmd": "request.get",
"url": url,
"maxTimeout": 60000,
"session": f"session_{hash(url) % 10000}",
}
r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65)
data = r.json()
if data.get("status") == "ok":
solution = data.get("solution", {})
return solution.get("response", "")
except:
pass
return None
def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]:
"""Route through Tor network."""
if not USE_TOR:
return None
try:
proxies = self.proxy.get_tor_proxy()
r = requests.get(url, headers=headers, proxies=proxies, timeout=30)
return r
except:
return None
def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try all curl_cffi impersonations in order."""
for name, fn in self.curl_methods.items():
try:
headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124")
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try cloudscraper with different configs."""
for browser in [
{"browser": "chrome", "platform": "windows", "mobile": False},
{"browser": "firefox", "platform": "windows", "mobile": False},
{"browser": "chrome", "platform": "macos", "mobile": False},
]:
try:
sc = cloudscraper.create_scraper(browser=browser)
sc.headers.update(rotate_headers(country))
r = sc.get(url, timeout=15)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_playwright(self, url: str) -> Optional[str]:
"""Use Playwright headless browser as last resort."""
try:
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
context = browser.new_context(
user_agent=random.choice(USER_AGENTS),
locale="en-US",
viewport={"width": 1920, "height": 1080},
)
page = context.new_page()
page.goto(url, timeout=30000)
content = page.content()
browser.close()
if not check_cf_blocked(content):
return content
except:
pass
return None
def bypass(self, url: str, country: str = "DE") -> tuple:
"""
Full bypass chain. Returns (content: str, method: str, elapsed: float).
Chain order:
1. curl_cffi chrome124 (fastest, 0.1-0.3s)
2. cloudscraper (basic CF, 0.3-0.5s)
3. curl_cffi edge99 (alt fingerprint)
4. curl_cffi safari17 (Eniro bypass)
5. Tor proxy rotation (anonymous, 2-5s)
6. FlareSolverr (heavy CF, 5-15s)
7. Playwright (full browser, 10-30s)
"""
start = time.time()
# Level 1: curl_cffi TLS fingerprint
r = self.try_curl(url, country)
if r and r.status_code == 200:
return (r.text, "curl_cffi", time.time() - start)
# Level 2: cloudscraper
r = self.try_cloudscraper(url, country)
if r and r.status_code == 200:
return (r.text, "cloudscraper", time.time() - start)
# Level 3: Tor
if USE_TOR:
r = self.try_tor(url, rotate_headers(country))
if r and r.status_code == 200:
return (r.text, "tor", time.time() - start)
# Level 4: FlareSolverr
content = self.try_flaresolverr(url)
if content:
return (content, "flaresolverr", time.time() - start)
# Level 5: Playwright (last resort)
content = self.try_playwright(url)
if content:
return (content, "playwright", time.time() - start)
return (None, "all_failed", time.time() - start)
def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple:
"""Bypass with proxy rotation for each method."""
start = time.time()
proxy = self.proxy.get_proxy()
headers = rotate_headers(country)
for name, fn in self.curl_methods.items():
try:
if proxy:
r = fn(url, headers)
else:
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return (r.text, f"curl_cffi+proxy_{name}", time.time() - start)
except:
continue
return self.bypass(url, country)