CI (SIAX Cloud) / security (push) Successful in 12s
Security Scan (SIAX Cloud) / security (push) Successful in 14s
CI (SIAX Cloud) / quality (push) Failing after 45s
CI (SIAX Cloud) / integration (push) Successful in 1m59s
CI (SIAX Cloud) / security (pull_request) Successful in 20s
CI (SIAX Cloud) / quality (pull_request) Failing after 26s
CI (SIAX Cloud) / integration (pull_request) Successful in 39s
Security Scan (SIAX Cloud) / security (pull_request) Successful in 33s
Automatiskt commitat innan lokala dubbletter rensades. Co-Authored-By: Claude Opus 5 <noreply@anthropic.com>
84 lines
2.2 KiB
YAML
84 lines
2.2 KiB
YAML
version: "3.8"
|
|
|
|
services:
|
|
scraper-brreg-no:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=NO
|
|
- SOURCE=brreg
|
|
- MAX_CONCURRENCY=100
|
|
- SCRAPING_ENABLED=false
|
|
- S3_ENDPOINT=${S3_ENDPOINT}
|
|
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
|
|
- S3_SECRET_KEY=${S3_SECRET_KEY}
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg"]
|
|
|
|
scraper-brreg-under:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=NO
|
|
- SOURCE=brreg_under
|
|
- MAX_CONCURRENCY=100
|
|
- SCRAPING_ENABLED=false
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg_under"]
|
|
|
|
scraper-dk-cvrapi:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DK
|
|
- SOURCE=cvrapi
|
|
- MAX_CONCURRENCY=30
|
|
- SCRAPING_ENABLED=false
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DK", "--source", "cvrapi"]
|
|
|
|
scraper-de-northdata:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DE
|
|
- SOURCE=northdata
|
|
- MAX_CONCURRENCY=20
|
|
- SCRAPING_ENABLED=false
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "northdata"]
|
|
|
|
scraper-de-handelsregister:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DE
|
|
- SOURCE=handelsregister
|
|
- MAX_CONCURRENCY=20
|
|
- SCRAPING_ENABLED=false
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "handelsregister"]
|
|
|
|
s3-sync:
|
|
image: scraper-engine:latest
|
|
depends_on:
|
|
- scraper-brreg-no
|
|
environment:
|
|
- S3_ENDPOINT=${S3_ENDPOINT}
|
|
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
|
|
- S3_SECRET_KEY=${S3_SECRET_KEY}
|
|
- S3_BUCKET=opt-scraped-data
|
|
- SCRAPING_ENABLED=false
|
|
volumes:
|
|
- scraper-data:/data
|
|
command: ["python", "-m", "scrapers.s3_sync"]
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
scraper-data:
|