chore(salvage): bevara ocommittat arbete vid konsolidering 2026-09-25
CI (SIAX Cloud) / security (push) Successful in 12s
Security Scan (SIAX Cloud) / security (push) Successful in 14s
CI (SIAX Cloud) / quality (push) Failing after 45s
CI (SIAX Cloud) / integration (push) Successful in 1m59s
CI (SIAX Cloud) / security (pull_request) Successful in 20s
CI (SIAX Cloud) / quality (pull_request) Failing after 26s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 33s

Automatiskt commitat innan lokala dubbletter rensades.

Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
This commit is contained in:
Claude Code
2026-09-25 01:55:43 +02:00
co-authored by Claude Opus 5
parent dc6aaa0e10
commit 148a32b577
12 changed files with 1362 additions and 2 deletions
+6
View File
@@ -7,6 +7,7 @@ services:
- COUNTRY=NO
- SOURCE=brreg
- MAX_CONCURRENCY=100
- SCRAPING_ENABLED=false
- S3_ENDPOINT=${S3_ENDPOINT}
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
@@ -21,6 +22,7 @@ services:
- COUNTRY=NO
- SOURCE=brreg_under
- MAX_CONCURRENCY=100
- SCRAPING_ENABLED=false
volumes:
- scraper-data:/data
restart: unless-stopped
@@ -32,6 +34,7 @@ services:
- COUNTRY=DK
- SOURCE=cvrapi
- MAX_CONCURRENCY=30
- SCRAPING_ENABLED=false
volumes:
- scraper-data:/data
restart: unless-stopped
@@ -43,6 +46,7 @@ services:
- COUNTRY=DE
- SOURCE=northdata
- MAX_CONCURRENCY=20
- SCRAPING_ENABLED=false
volumes:
- scraper-data:/data
restart: unless-stopped
@@ -54,6 +58,7 @@ services:
- COUNTRY=DE
- SOURCE=handelsregister
- MAX_CONCURRENCY=20
- SCRAPING_ENABLED=false
volumes:
- scraper-data:/data
restart: unless-stopped
@@ -68,6 +73,7 @@ services:
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
- S3_SECRET_KEY=${S3_SECRET_KEY}
- S3_BUCKET=opt-scraped-data
- SCRAPING_ENABLED=false
volumes:
- scraper-data:/data
command: ["python", "-m", "scrapers.s3_sync"]
+7 -1
View File
@@ -13,7 +13,7 @@ Metoder (i prioritetsordning):
6. FlareSolverr - heavy CF bypass (kraver Docker)
"""
import json, time, random, os, asyncio
import json, time, random, os, asyncio, sys
from datetime import datetime
from typing import Optional
from dataclasses import dataclass, field, asdict
@@ -24,6 +24,12 @@ import httpx
from curl_cffi import requests as curl_requests
import cloudscraper
# ── Master switch: scraping ONLY runs when explicitly enabled ──
SCRAPING_ENABLED = os.getenv("SCRAPING_ENABLED", "false").lower() in ("1", "true", "yes")
if not SCRAPING_ENABLED:
print("[SCRAPER] SCRAPING_ENABLED not set — engine is in standby. Set SCRAPING_ENABLED=true to activate.")
sys.exit(0)
# ── Config ──
OUTPUT_DIR = Path(__file__).parent / "output"
HEADER_POOL = {
+6
View File
@@ -23,6 +23,12 @@ import httpx
from curl_cffi import requests as curl_requests
import cloudscraper
# ── Master switch: scraping ONLY runs when explicitly enabled ──
SCRAPING_ENABLED = os.getenv("SCRAPING_ENABLED", "false").lower() in ("1", "true", "yes")
if not SCRAPING_ENABLED:
print("[SCRAPER] SCRAPING_ENABLED not set — engine v2 is in standby. Set SCRAPING_ENABLED=true to activate.")
sys.exit(0)
# ── Config from env ──
S3_ENDPOINT = os.getenv("S3_ENDPOINT", "https://s3.siax.io")
S3_ACCESS_KEY = os.getenv("S3_ACCESS_KEY", "")
+7 -1
View File
@@ -13,13 +13,19 @@ Metoder:
9. Fake identity generation
10. Session cookie reuse
"""
import os, json, time, random, socket, asyncio
import os, json, time, random, socket, asyncio, sys
from datetime import datetime
from typing import Optional
from curl_cffi import requests as curl_requests
import cloudscraper
import requests
# ── Master switch: scraping ONLY runs when explicitly enabled ──
SCRAPING_ENABLED = os.getenv("SCRAPING_ENABLED", "false").lower() in ("1", "true", "yes")
if not SCRAPING_ENABLED:
print("[WAF_BYPASS] SCRAPING_ENABLED not set — waf_bypass is in standby. Set SCRAPING_ENABLED=true to activate.")
sys.exit(0)
# ── Config ──
FLARESOLVERR_URL = os.getenv("FLARESOLVERR_URL", "http://localhost:8191/v1")
TOR_PROXY = os.getenv("TOR_PROXY", "socks5://localhost:9050")