version: "3.8" services: scraper-brreg-no: image: scraper-engine:latest environment: - COUNTRY=NO - SOURCE=brreg - MAX_CONCURRENCY=100 - S3_ENDPOINT=${S3_ENDPOINT} - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} volumes: - scraper-data:/data restart: unless-stopped command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg"] scraper-brreg-under: image: scraper-engine:latest environment: - COUNTRY=NO - SOURCE=brreg_under - MAX_CONCURRENCY=100 volumes: - scraper-data:/data restart: unless-stopped command: ["python", "-m", "scrapers.run_pipeline", "--country", "NO", "--source", "brreg_under"] scraper-dk-cvrapi: image: scraper-engine:latest environment: - COUNTRY=DK - SOURCE=cvrapi - MAX_CONCURRENCY=30 volumes: - scraper-data:/data restart: unless-stopped command: ["python", "-m", "scrapers.run_pipeline", "--country", "DK", "--source", "cvrapi"] scraper-de-northdata: image: scraper-engine:latest environment: - COUNTRY=DE - SOURCE=northdata - MAX_CONCURRENCY=20 volumes: - scraper-data:/data restart: unless-stopped command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "northdata"] scraper-de-handelsregister: image: scraper-engine:latest environment: - COUNTRY=DE - SOURCE=handelsregister - MAX_CONCURRENCY=20 volumes: - scraper-data:/data restart: unless-stopped command: ["python", "-m", "scrapers.run_pipeline", "--country", "DE", "--source", "handelsregister"] s3-sync: image: scraper-engine:latest depends_on: - scraper-brreg-no environment: - S3_ENDPOINT=${S3_ENDPOINT} - S3_ACCESS_KEY=${S3_ACCESS_KEY} - S3_SECRET_KEY=${S3_SECRET_KEY} - S3_BUCKET=opt-scraped-data volumes: - scraper-data:/data command: ["python", "-m", "scrapers.s3_sync"] restart: unless-stopped volumes: scraper-data: