From 3982b03f5caf4b5b4c6ed3b42bc33634560dcc01 Mon Sep 17 00:00:00 2001 From: nfel Date: Tue, 28 Jul 2026 22:15:14 +0330 Subject: [PATCH] Use packaged Chromium driver in Docker --- Dockerfile | 23 ++++++----------------- crawler/driver.py | 6 +++++- docker-compose.yml | 5 ++++- scripts/patch_driver.py | 37 +++++++++++++++++++++++++++++++++++-- 4 files changed, 50 insertions(+), 21 deletions(-) diff --git a/Dockerfile b/Dockerfile index 50e8678..1b77109 100644 --- a/Dockerfile +++ b/Dockerfile @@ -15,20 +15,9 @@ RUN uv sync --frozen --no-dev --no-install-project # ── Stage 2: runtime ────────────────────────────────────────────────────────── FROM python:3.11-slim AS runtime -# ── Chrome + system deps ────────────────────────────────────────────────────── +# ── Chromium + matching system ChromeDriver ─────────────────────────────────── RUN apt-get update && apt-get install -y --no-install-recommends \ - wget gnupg ca-certificates curl unzip \ - # X11 / rendering libs needed even in headless mode - libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \ - libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \ - libgbm1 libasound2 libpango-1.0-0 libcairo2 libx11-xcb1 \ - && wget -qO- https://dl.google.com/linux/linux_signing_key.pub \ - | gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \ - && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] \ - http://dl.google.com/linux/chrome/deb/ stable main" \ - > /etc/apt/sources.list.d/google-chrome.list \ - && apt-get update \ - && apt-get install -y --no-install-recommends google-chrome-stable \ + ca-certificates curl chromium chromium-driver \ && rm -rf /var/lib/apt/lists/* # ── uv + venv from builder ──────────────────────────────────────────────────── @@ -41,8 +30,8 @@ ENV PATH="/app/.venv/bin:$PATH" \ PYTHONUNBUFFERED=1 \ PYTHONDONTWRITEBYTECODE=1 \ # Tell undetected-chromedriver where Chrome lives - CHROME_BINARY=/usr/bin/google-chrome-stable \ - CHROMEDRIVER_PATH=/app/drivers/chromedriver \ + CHROME_BINARY=/usr/bin/chromium \ + CHROMEDRIVER_PATH=/usr/bin/chromedriver \ # Always run headless inside Docker HEADLESS=true @@ -52,8 +41,8 @@ COPY . . # Copy this after the application so a host venv can never replace it. COPY --from=builder /app/.venv /app/.venv -# Download and patch the matching driver at build time. Runtime jobs must not -# depend on undetected-chromedriver's network downloader. +# Patch Debian's version-matched driver at build time. Runtime jobs require no +# ChromeDriver network download. RUN PYTHONPATH=. python scripts/patch_driver.py && mkdir -p data logs # Non-root user for safety diff --git a/crawler/driver.py b/crawler/driver.py index 80862fe..43571a9 100644 --- a/crawler/driver.py +++ b/crawler/driver.py @@ -4,6 +4,7 @@ from __future__ import annotations import os import random +import re import shutil import subprocess import sys @@ -56,7 +57,10 @@ def _detect_chrome() -> tuple[str, int]: out = subprocess.check_output( [resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5 ).strip() - major = int(out.split()[-1].split(".")[0]) + match = re.search(r"\b(\d+(?:\.\d+){3})\b", out) + if match is None: + continue + major = int(match.group(1).split(".")[0]) log.debug("Detected Chrome %d via '%s'", major, resolved) return resolved, major except Exception: diff --git a/docker-compose.yml b/docker-compose.yml index 5749b17..3a8e633 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -31,6 +31,8 @@ services: REDIS_URL: "redis://redis:6379/0" SKIP_INTERNAL_WORKER: "true" HEADLESS: "true" + CHROME_BINARY: /usr/bin/chromium + CHROMEDRIVER_PATH: /usr/bin/chromedriver volumes: - seed-data:/app/data - seed-logs:/app/logs @@ -58,7 +60,8 @@ services: LOG_FILE: /app/logs/seed.log REDIS_URL: "redis://redis:6379/0" HEADLESS: "true" - CHROME_BINARY: /usr/bin/google-chrome-stable + CHROME_BINARY: /usr/bin/chromium + CHROMEDRIVER_PATH: /usr/bin/chromedriver volumes: - seed-data:/app/data - seed-logs:/app/logs diff --git a/scripts/patch_driver.py b/scripts/patch_driver.py index 0215ecd..f825b94 100644 --- a/scripts/patch_driver.py +++ b/scripts/patch_driver.py @@ -7,6 +7,7 @@ version matching the installed Chrome, then patches it with undetected_chromedri from __future__ import annotations import io +import re import shutil import stat import subprocess @@ -42,7 +43,10 @@ def _get_chrome_full_version() -> tuple[int, str]: out = subprocess.check_output( [binary, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5 ).strip() - full = out.split()[-1] + match = re.search(r"\b(\d+(?:\.\d+){3})\b", out) + if match is None: + continue + full = match.group(1) major = int(full.split(".")[0]) log.info("Detected Chrome %s via '%s'", full, binary) return major, full @@ -97,13 +101,42 @@ def _patch(dest: Path) -> None: log.info("Patch applied successfully") +def _validate_existing_driver(dest: Path, chrome_major: int) -> bool: + if not dest.exists(): + return False + + try: + out = subprocess.check_output( + [str(dest), "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5 + ).strip() + driver_major = int(out.split()[1].split(".")[0]) + except Exception as exc: + log.critical("Could not inspect ChromeDriver at %s: %s", dest, exc) + sys.exit(1) + + if driver_major != chrome_major: + log.critical( + "ChromeDriver major %d does not match Chrome major %d", + driver_major, + chrome_major, + ) + sys.exit(1) + + log.info("Using packaged ChromeDriver %s", out.split()[1]) + return True + + def main() -> None: dest = Path(config.chromedriver_path) dest.parent.mkdir(parents=True, exist_ok=True) major, _ = _get_chrome_full_version() - cd_version = _latest_chromedriver_version(major) + if _validate_existing_driver(dest, major): + _patch(dest) + log.info("ChromeDriver ready at %s", dest) + return + cd_version = _latest_chromedriver_version(major) _download_chromedriver(cd_version, dest) _patch(dest)