78 lines
2.1 KiB
YAML
78 lines
2.1 KiB
YAML
version: "3.8"
|
|
|
|
services:
|
|
scraper-brreg-no:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=NO
|
|
- SOURCE=brreg
|
|
- MAX_CONCURRENCY=100
|
|
- S3_ENDPOINT=${S3_ENDPOINT}
|
|
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
|
|
- S3_SECRET_KEY=${S3_SECRET_KEY}
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg"]
|
|
|
|
scraper-brreg-under:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=NO
|
|
- SOURCE=brreg_under
|
|
- MAX_CONCURRENCY=100
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg_under"]
|
|
|
|
scraper-dk-cvrapi:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DK
|
|
- SOURCE=cvrapi
|
|
- MAX_CONCURRENCY=30
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DK", "--source", "cvrapi"]
|
|
|
|
scraper-de-northdata:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DE
|
|
- SOURCE=northdata
|
|
- MAX_CONCURRENCY=20
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "northdata"]
|
|
|
|
scraper-de-handelsregister:
|
|
image: scraper-engine:latest
|
|
environment:
|
|
- COUNTRY=DE
|
|
- SOURCE=handelsregister
|
|
- MAX_CONCURRENCY=20
|
|
volumes:
|
|
- scraper-data:/data
|
|
restart: unless-stopped
|
|
command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "handelsregister"]
|
|
|
|
s3-sync:
|
|
image: scraper-engine:latest
|
|
depends_on:
|
|
- scraper-brreg-no
|
|
environment:
|
|
- S3_ENDPOINT=${S3_ENDPOINT}
|
|
- S3_ACCESS_KEY=${S3_ACCESS_KEY}
|
|
- S3_SECRET_KEY=${S3_SECRET_KEY}
|
|
- S3_BUCKET=opt-scraped-data
|
|
volumes:
|
|
- scraper-data:/data
|
|
command: ["python", "-m", "scrapers.s3_sync"]
|
|
restart: unless-stopped
|
|
|
|
volumes:
|
|
scraper-data:
|