Recover from slow browser navigation

crawler/driver.py

@@ -98,8 +98,8 @@ def _build_options(
         opts.add_argument(f"--user-data-dir={profile_dir}")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
-    opts.add_argument("--disable-dev-shm-usage")
     opts.add_argument("--disable-gpu")
+    opts.add_argument("--renderer-process-limit=2")
     opts.add_argument("--disable-blink-features=AutomationControlled")
     opts.add_argument("--disable-infobars")
     opts.add_argument("--window-size=1920,1080")

crawler/scenarios/fresh_session.py

@@ -2,9 +2,13 @@
 from __future__ import annotations
 
 import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 class FreshSessionScenario:
@@ -34,13 +38,24 @@ class FreshSessionScenario:
         if not self.url:
             return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
 
-        self.driver.get(self.url)
+        try:
+            self.driver.get(self.url)
+        except TimeoutException:
+            log.warning(
+                "Page load exceeded the browser timeout; stopping navigation and continuing"
+            )
+            self.driver.execute_cdp_cmd("Page.stopLoading", {})
         human_delay(2.0, 4.0)
 
-        # Wait for the page to reach a ready state
+        # Interactive is sufficient for the dynamic flow and avoids waiting on
-        self.wait.until(
+        # analytics, ads, or other background resources indefinitely.
-            lambda d: d.execute_script("return document.readyState") == "complete"
+        try:
-        )
+            self.wait.until(
+                lambda d: d.execute_script("return document.readyState")
+                in ("interactive", "complete")
+            )
+        except TimeoutException:
+            log.warning("Document did not report a ready state; continuing with the current DOM")
 
         title = self.driver.title
         current_url = self.driver.current_url

docker-compose.yml

@@ -69,7 +69,7 @@ services:
       redis:
         condition: service_healthy
     # Chrome needs extra shared memory to avoid OOM crashes
-    shm_size: "256mb"
+    shm_size: "512mb"
     cap_add:
       - SYS_ADMIN
     security_opt: