Recover from slow browser navigation

crawler/scenarios/fresh_session.py

@@ -2,9 +2,13 @@
 from __future__ import annotations
 
 import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 class FreshSessionScenario:
@@ -34,13 +38,24 @@ class FreshSessionScenario:
         if not self.url:
             return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
 
-        self.driver.get(self.url)
+        try:
+            self.driver.get(self.url)
+        except TimeoutException:
+            log.warning(
+                "Page load exceeded the browser timeout; stopping navigation and continuing"
+            )
+            self.driver.execute_cdp_cmd("Page.stopLoading", {})
         human_delay(2.0, 4.0)
 
-        # Wait for the page to reach a ready state
-        self.wait.until(
-            lambda d: d.execute_script("return document.readyState") == "complete"
-        )
+        # Interactive is sufficient for the dynamic flow and avoids waiting on
+        # analytics, ads, or other background resources indefinitely.
+        try:
+            self.wait.until(
+                lambda d: d.execute_script("return document.readyState")
+                in ("interactive", "complete")
+            )
+        except TimeoutException:
+            log.warning("Document did not report a ready state; continuing with the current DOM")
 
         title = self.driver.title
         current_url = self.driver.current_url