108 lines
3.0 KiB
YAML
108 lines
3.0 KiB
YAML
services:
|
|
|
|
# ── Redis (state + job queue) ─────────────────────────────────────────────────
|
|
redis:
|
|
image: redis:7-alpine
|
|
container_name: seed-redis
|
|
restart: unless-stopped
|
|
volumes:
|
|
- redis-data:/data
|
|
healthcheck:
|
|
test: ["CMD", "redis-cli", "ping"]
|
|
interval: 10s
|
|
timeout: 3s
|
|
retries: 5
|
|
|
|
# ── Admin panel (FastAPI) ─────────────────────────────────────────────────────
|
|
admin:
|
|
build: .
|
|
image: seed:latest
|
|
hostname: crawler-admin
|
|
container_name: seed-admin
|
|
restart: unless-stopped
|
|
command: python main.py admin
|
|
ports:
|
|
- "${COMPOSE_ADMIN_PORT:-127.0.0.0:8800}:8000"
|
|
env_file:
|
|
- .env
|
|
environment:
|
|
ADMIN_HOST: "0.0.0.0"
|
|
DB_PATH: /app/data/tracker.db
|
|
LOG_FILE: /app/logs/seed.log
|
|
REDIS_URL: "redis://redis:6379/0"
|
|
SKIP_INTERNAL_WORKER: "true"
|
|
HEADLESS: "true"
|
|
CHROME_BINARY: /usr/bin/chromium
|
|
CHROMEDRIVER_PATH: /usr/bin/chromedriver
|
|
RECORDINGS_DIR: /app/recordings
|
|
volumes:
|
|
- seed-data:/app/data
|
|
- seed-logs:/app/logs
|
|
- seed-recordings:/app/recordings
|
|
depends_on:
|
|
redis:
|
|
condition: service_healthy
|
|
healthcheck:
|
|
test: ["CMD", "curl", "-f", "http://localhost:8000/"]
|
|
interval: 30s
|
|
timeout: 5s
|
|
retries: 3
|
|
start_period: 10s
|
|
networks:
|
|
- proxy-network
|
|
- default
|
|
|
|
# ── RQ worker (Chrome crawler, runs batch jobs) ───────────────────────────────
|
|
worker:
|
|
build: .
|
|
image: seed:latest
|
|
restart: unless-stopped
|
|
deploy:
|
|
replicas: 3
|
|
command: ["rq", "worker", "--url", "redis://redis:6379/0", "batch"]
|
|
env_file:
|
|
- .env
|
|
environment:
|
|
DB_PATH: /app/data/tracker.db
|
|
LOG_FILE: /app/logs/seed.log
|
|
REDIS_URL: "redis://redis:6379/0"
|
|
HEADLESS: "true"
|
|
CHROME_BINARY: /usr/bin/chromium
|
|
CHROMEDRIVER_PATH: /usr/bin/chromedriver
|
|
RECORD_SESSIONS: "true"
|
|
RECORDING_FPS: "12"
|
|
RECORDING_MAX_FILES: "3"
|
|
RECORDINGS_DIR: /app/recordings
|
|
volumes:
|
|
- seed-data:/app/data
|
|
- seed-logs:/app/logs
|
|
- seed-recordings:/app/recordings
|
|
depends_on:
|
|
redis:
|
|
condition: service_healthy
|
|
# Chrome needs extra shared memory to avoid OOM crashes
|
|
# Headful Chromium recording needs substantially more shared memory than
|
|
# headless mode. This is a limit, not an up-front RAM allocation.
|
|
shm_size: "2gb"
|
|
cap_add:
|
|
- SYS_ADMIN
|
|
security_opt:
|
|
- seccomp:unconfined
|
|
|
|
|
|
networks:
|
|
proxy-network:
|
|
external: true
|
|
default:
|
|
name: crawle-snapp_default
|
|
|
|
volumes:
|
|
seed-data:
|
|
driver: local
|
|
seed-logs:
|
|
driver: local
|
|
seed-recordings:
|
|
driver: local
|
|
redis-data:
|
|
driver: local
|