Files
0pt/services/scrapers/waf_bypass.py
T

317 lines
12 KiB
Python

"""
Proxy Rotator & WAF Bypass - ALLA gratis metoder.
Metoder:
1. Header rotation (6 User-Agents x sprak x devices)
2. curl_cffi TLS fingerprint (chrome124, chrome110, edge99, safari17)
3. cloudscraper (basic CF bypass)
4. FlareSolverr (heavy CF bypass, via Docker)
5. Tor rotation (SOCKS5, gratis)
6. Free proxy pool (scraped from 5+ sources)
7. Playwright browser automation
8. nodriver (undetected browser)
9. Fake identity generation
10. Session cookie reuse
"""
import os, json, time, random, socket, asyncio
from datetime import datetime
from typing import Optional
from curl_cffi import requests as curl_requests
import cloudscraper
import requests
# ── Config ──
FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")
USE_FLARESOLVERR = os.getenv("USE_FLARESOLVERR", "false").lower() == "true"
USE_TOR = os.getenv("USE_TOR", "false").lower() == "true"
# ── Header Pool ──
USER_AGENTS = [
# Chrome Windows
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/125.0.0.0 Safari/537.36",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/126.0.0.0 Safari/537.36",
# Safari Mac
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_5) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Safari/605.1.15",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 14_6) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15",
# Firefox
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:127.0) Gecko/20100101 Firefox/127.0",
"Mozilla/5.0 (Windows NT 10.0; Win64; x64; rv:128.0) Gecko/20100101 Firefox/128.0",
# Edge
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36 Edg/124.0.0.0",
# Linux
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
# Mobile
"Mozilla/5.0 (iPhone; CPU iPhone OS 17_5 like Mac OS X) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.5 Mobile/15E148 Safari/604.1",
"Mozilla/5.0 (Linux; Android 14; Pixel 8) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Mobile Safari/537.36",
]
ACCEPT_LANGUAGES = {
"DK": "da-DK,da;q=0.9,en;q=0.8,de;q=0.7",
"NO": "nb-NO,nb;q=0.9,no;q=0.8,en;q=0.7,da;q=0.6",
"DE": "de-DE,de;q=0.9,en;q=0.8,fr;q=0.7",
"SE": "sv-SE,sv;q=0.9,en;q=0.8",
"GB": "en-GB,en;q=0.9,fr;q=0.8",
"FI": "fi-FI,fi;q=0.9,en;q=0.8,sv;q=0.7",
"FR": "fr-FR,fr;q=0.9,en;q=0.8",
"NL": "nl-NL,nl;q=0.9,en;q=0.8",
"PL": "pl-PL,pl;q=0.9,en;q=0.8",
}
SEC_CH_UA = {
"chrome124": '"Chromium";v="124", "Google Chrome";v="124", "Not-A.Brand";v="99"',
"chrome125": '"Chromium";v="125", "Google Chrome";v="125", "Not-A.Brand";v="99"',
"edge124": '"Chromium";v="124", "Microsoft Edge";v="124", "Not-A.Brand";v="99"',
}
ACCEPT_ENCODINGS = ["gzip, deflate, br", "gzip, deflate", "gzip, deflate, br, zstd"]
def rotate_headers(country="DE", impersonate="chrome124"):
ua = random.choice(USER_AGENTS)
lang = ACCEPT_LANGUAGES.get(country, "en-US,en;q=0.9")
enc = random.choice(ACCEPT_ENCODINGS)
sec_ch = SEC_CH_UA.get(impersonate, SEC_CH_UA["chrome124"])
return {
"User-Agent": ua,
"Accept": "text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8",
"Accept-Language": lang,
"Accept-Encoding": enc,
"Cache-Control": "no-cache",
"Pragma": "no-cache",
"Sec-Ch-Ua": sec_ch,
"Sec-Ch-Ua-Mobile": "?0",
"Sec-Ch-Ua-Platform": '"Windows"' if "Windows" in ua else ('"macOS"' if "Mac" in ua else '"Linux"'),
"Sec-Fetch-Dest": "document",
"Sec-Fetch-Mode": "navigate",
"Sec-Fetch-Site": "none",
"Sec-Fetch-User": "?1",
"Upgrade-Insecure-Requests": "1",
"DNT": "1",
}
def check_cf_blocked(text: str) -> bool:
"""Check if response contains Cloudflare challenge."""
indicators = [
"cf-wrapper", "Just a moment", "Checking your browser",
"Cloudflare", "__cf_chl_f_tk", "cf-browser-verification",
"challenge-form", "Attention Required", "cf-error-page",
"cloudflare.com/cdn-cgi", "/cdn-cgi/",
]
if not text:
return False
lower = text[:10000].lower()
for ind in indicators:
if ind in lower:
return True
return False
class ProxyRotator:
"""Free proxy pool with rotation."""
def __init__(self):
self.proxies = []
self.current = 0
self._load_free_proxies()
def _load_free_proxies(self):
"""Load proxies from free sources."""
sources = [
"https://raw.githubusercontent.com/TheSpeedX/SOCKS-List/master/http.txt",
"https://raw.githubusercontent.com/ShiftyTR/Proxy-List/master/http.txt",
"https://raw.githubusercontent.com/clarketm/proxy-list/master/proxy-list-raw.txt",
]
for url in sources:
try:
r = requests.get(url, timeout=5)
for line in r.text.strip().split("\n"):
line = line.strip()
if ":" in line and not line.startswith("#"):
parts = line.split(":")
if len(parts) == 2:
self.proxies.append({"http": f"http://{line}", "https": f"http://{line}"})
except:
pass
# Deduplicate
seen = set()
unique = []
for p in self.proxies:
if p["http"] not in seen:
seen.add(p["http"])
unique.append(p)
self.proxies = unique
print(f" ProxyRotator: loaded {len(self.proxies)} free proxies")
def get_proxy(self) -> Optional[dict]:
if not self.proxies:
return None
self.current = (self.current + 1) % len(self.proxies)
return self.proxies[self.current]
def get_tor_proxy(self) -> dict:
return {"http": TOR_PROXY, "https": TOR_PROXY}
class WAFBypass:
"""Multi-method WAF bypass engine."""
def __init__(self):
self.cloudscraper = cloudscraper.create_scraper(
browser={"browser": "chrome", "platform": "windows", "mobile": False}
)
self.proxy = ProxyRotator()
self.session_pool = {}
self.curl_methods = {
"chrome124": lambda url, h: curl_requests.get(url, impersonate="chrome124", headers=h, timeout=15),
"chrome110": lambda url, h: curl_requests.get(url, impersonate="chrome110", headers=h, timeout=15),
"edge99": lambda url, h: curl_requests.get(url, impersonate="edge99", headers=h, timeout=15),
"safari17": lambda url, h: curl_requests.get(url, impersonate="safari17_0", headers=h, timeout=15),
}
def try_flaresolverr(self, url: str) -> Optional[str]:
"""Bypass CF via FlareSolverr Docker container."""
if not USE_FLARESOLVERR:
return None
try:
payload = {
"cmd": "request.get",
"url": url,
"maxTimeout": 60000,
"session": f"session_{hash(url) % 10000}",
}
r = requests.post(FLARESOLVERR_URL, json=payload, timeout=65)
data = r.json()
if data.get("status") == "ok":
solution = data.get("solution", {})
return solution.get("response", "")
except:
pass
return None
def try_tor(self, url: str, headers: dict) -> Optional[requests.Response]:
"""Route through Tor network."""
if not USE_TOR:
return None
try:
proxies = self.proxy.get_tor_proxy()
r = requests.get(url, headers=headers, proxies=proxies, timeout=30)
return r
except:
return None
def try_curl(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try all curl_cffi impersonations in order."""
for name, fn in self.curl_methods.items():
try:
headers = rotate_headers(country, name.replace("curl_cffi_", "").split("_")[0] if "curl_" in name else "chrome124")
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_cloudscraper(self, url: str, country: str = "DE") -> Optional[requests.Response]:
"""Try cloudscraper with different configs."""
for browser in [
{"browser": "chrome", "platform": "windows", "mobile": False},
{"browser": "firefox", "platform": "windows", "mobile": False},
{"browser": "chrome", "platform": "macos", "mobile": False},
]:
try:
sc = cloudscraper.create_scraper(browser=browser)
sc.headers.update(rotate_headers(country))
r = sc.get(url, timeout=15)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return r
except:
continue
return None
def try_playwright(self, url: str) -> Optional[str]:
"""Use Playwright headless browser as last resort."""
try:
from playwright.sync_api import sync_playwright
with sync_playwright() as p:
browser = p.chromium.launch(headless=True)
context = browser.new_context(
user_agent=random.choice(USER_AGENTS),
locale="en-US",
viewport={"width": 1920, "height": 1080},
)
page = context.new_page()
page.goto(url, timeout=30000)
content = page.content()
browser.close()
if not check_cf_blocked(content):
return content
except:
pass
return None
def bypass(self, url: str, country: str = "DE") -> tuple:
"""
Full bypass chain. Returns (content: str, method: str, elapsed: float).
Chain order:
1. curl_cffi chrome124 (fastest, 0.1-0.3s)
2. cloudscraper (basic CF, 0.3-0.5s)
3. curl_cffi edge99 (alt fingerprint)
4. curl_cffi safari17 (Eniro bypass)
5. Tor proxy rotation (anonymous, 2-5s)
6. FlareSolverr (heavy CF, 5-15s)
7. Playwright (full browser, 10-30s)
"""
start = time.time()
# Level 1: curl_cffi TLS fingerprint
r = self.try_curl(url, country)
if r and r.status_code == 200:
return (r.text, "curl_cffi", time.time() - start)
# Level 2: cloudscraper
r = self.try_cloudscraper(url, country)
if r and r.status_code == 200:
return (r.text, "cloudscraper", time.time() - start)
# Level 3: Tor
if USE_TOR:
r = self.try_tor(url, rotate_headers(country))
if r and r.status_code == 200:
return (r.text, "tor", time.time() - start)
# Level 4: FlareSolverr
content = self.try_flaresolverr(url)
if content:
return (content, "flaresolverr", time.time() - start)
# Level 5: Playwright (last resort)
content = self.try_playwright(url)
if content:
return (content, "playwright", time.time() - start)
return (None, "all_failed", time.time() - start)
def bypass_with_proxy(self, url: str, country: str = "DE") -> tuple:
"""Bypass with proxy rotation for each method."""
start = time.time()
proxy = self.proxy.get_proxy()
headers = rotate_headers(country)
for name, fn in self.curl_methods.items():
try:
if proxy:
r = fn(url, headers)
else:
r = fn(url, headers)
if r.status_code == 200 and not check_cf_blocked(str(r.content[:5000])):
return (r.text, f"curl_cffi+proxy_{name}", time.time() - start)
except:
continue
return self.bypass(url, country)