diff --git a/crawler/driver.py b/crawler/driver.py index 9e96159..3e329a8 100644 --- a/crawler/driver.py +++ b/crawler/driver.py @@ -91,7 +91,6 @@ def _build_options( chrome_binary: str, profile_dir: str | None = None, display: str | None = None, - initial_url: str | None = None, ) -> Options: opts = Options() opts.binary_location = chrome_binary @@ -108,14 +107,17 @@ def _build_options( opts.add_argument(f"--user-agent={user_agent}") opts.add_argument("--no-sandbox") opts.add_argument("--disable-gpu") + opts.add_argument("--disable-logging") + opts.add_argument("--no-first-run") + opts.add_argument("--no-default-browser-check") + opts.add_argument("--disable-background-networking") + opts.add_argument("--disable-component-update") + opts.add_argument("--disable-default-apps") + opts.add_argument("--disable-sync") opts.add_argument("--renderer-process-limit=2") opts.add_argument("--disable-blink-features=AutomationControlled") opts.add_argument("--disable-infobars") opts.add_argument("--window-size=1920,1080") - if initial_url and initial_url.startswith(("http://", "https://")): - # Opening the configured page as a Chrome process argument avoids - # waiting for the initial New Tab renderer before navigation begins. - opts.add_argument(initial_url) return opts @@ -135,7 +137,6 @@ def make_driver( chrome_binary, profile_dir, display, - initial_url, ) log.info( @@ -150,6 +151,10 @@ def make_driver( driver_executable_path=_local_driver_path(), version_main=major, use_subprocess=True, + # undetected-chromedriver captures Chrome stdout/stderr in pipes but + # does not drain them. Fatal-only logging prevents a full pipe from + # freezing Chrome during renderer startup in Docker. + log_level=3, ) log.info("ChromeDriver connected") diff --git a/docker-compose.yml b/docker-compose.yml index 2794a33..2c5c03b 100644 --- a/docker-compose.yml +++ b/docker-compose.yml @@ -77,7 +77,9 @@ services: redis: condition: service_healthy # Chrome needs extra shared memory to avoid OOM crashes - shm_size: "512mb" + # Headful Chromium recording needs substantially more shared memory than + # headless mode. This is a limit, not an up-front RAM allocation. + shm_size: "2gb" cap_add: - SYS_ADMIN security_opt: