diff --git a/Dockerfile b/Dockerfile index 82f2c53..23c115a 100644 --- a/Dockerfile +++ b/Dockerfile @@ -49,7 +49,7 @@ ENV PATH="/app/.venv/bin:$PATH" \ # ── Application code ────────────────────────────────────────────────────────── COPY . . -RUN mkdir -p data +RUN mkdir -p data logs # Non-root user for safety RUN useradd -m -u 1001 seed && chown -R seed:seed /app diff --git a/admin/main.py b/admin/main.py index 2406570..3560eb7 100644 --- a/admin/main.py +++ b/admin/main.py @@ -27,6 +27,8 @@ _rq_worker: subprocess.Popen[bytes] | None = None def on_startup() -> None: global _rq_worker init_db() + if os.environ.get("SKIP_INTERNAL_WORKER", "").lower() in ("1", "true", "yes"): + return redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0") _rq_worker = subprocess.Popen( [sys.executable, "-m", "rq", "worker", "--url", redis_url, "batch"], diff --git a/docker-compose.yml b/docker-compose.yml index 3bc3023..d9cdc20 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -1,6 +1,19 @@ services: - # ── Admin panel (FastAPI) ──────────────────────────────────────────────────── + # ── Redis (state + job queue) ───────────────────────────────────────────────── + redis: + image: redis:7-alpine + container_name: seed-redis + restart: unless-stopped + volumes: + - redis-data:/data + healthcheck: + test: ["CMD", "redis-cli", "ping"] + interval: 10s + timeout: 3s + retries: 5 + + # ── Admin panel (FastAPI) ───────────────────────────────────────────────────── admin: build: . image: seed:latest @@ -12,12 +25,18 @@ services: env_file: - .env environment: - # Override host so FastAPI binds on all interfaces inside the container ADMIN_HOST: "0.0.0.0" DB_PATH: /app/data/tracker.db + LOG_FILE: /app/logs/seed.log + REDIS_URL: "redis://redis:6379/0" + SKIP_INTERNAL_WORKER: "true" HEADLESS: "true" volumes: - seed-data:/app/data + - seed-logs:/app/logs + depends_on: + redis: + condition: service_healthy healthcheck: test: ["CMD", "curl", "-f", "http://localhost:8000/"] interval: 30s @@ -25,32 +44,38 @@ services: retries: 3 start_period: 10s - # ── Crawler (one-shot, started manually or via make docker-otp / docker-fresh) - crawler: + # ── RQ worker (Chrome crawler, runs batch jobs) ─────────────────────────────── + worker: build: . image: seed:latest - container_name: seed-crawler - # No restart — run on demand - restart: "no" - profiles: - - crawler # docker compose --profile crawler run crawler python main.py otp + container_name: seed-worker + restart: unless-stopped + command: python -m rq worker --url redis://redis:6379/0 batch env_file: - .env environment: DB_PATH: /app/data/tracker.db + LOG_FILE: /app/logs/seed.log + REDIS_URL: "redis://redis:6379/0" HEADLESS: "true" CHROME_BINARY: /usr/bin/google-chrome-stable volumes: - seed-data:/app/data - # Chrome needs /dev/shm enlarged to avoid OOM crashes + - seed-logs:/app/logs + depends_on: + redis: + condition: service_healthy + # Chrome needs extra shared memory to avoid OOM crashes shm_size: "256mb" - # Required capabilities for Chrome sandbox inside Docker cap_add: - SYS_ADMIN security_opt: - seccomp:unconfined volumes: - # Named volume — SQLite file persists across container restarts and rebuilds seed-data: driver: local + seed-logs: + driver: local + redis-data: + driver: local