From 601b8ec69e85b0469983cbb9d22104ed895f5b38 Mon Sep 17 00:00:00 2001
From: nfel <nfilsaraee@gmail.com>
Date: Tue, 28 Jul 2026 23:47:15 +0330
Subject: [PATCH] Recover from slow browser navigation

---
 crawler/driver.py                  |  2 +-
 crawler/scenarios/fresh_session.py | 25 ++++++++++++++++++++-----
 docker-compose.yml                 |  2 +-
 3 files changed, 22 insertions(+), 7 deletions(-)

diff --git a/crawler/driver.py b/crawler/driver.py
index 32e6ed0..8e37fc0 100644
--- a/crawler/driver.py
+++ b/crawler/driver.py
@@ -98,8 +98,8 @@ def _build_options(
         opts.add_argument(f"--user-data-dir={profile_dir}")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
-    opts.add_argument("--disable-dev-shm-usage")
     opts.add_argument("--disable-gpu")
+    opts.add_argument("--renderer-process-limit=2")
     opts.add_argument("--disable-blink-features=AutomationControlled")
     opts.add_argument("--disable-infobars")
     opts.add_argument("--window-size=1920,1080")
diff --git a/crawler/scenarios/fresh_session.py b/crawler/scenarios/fresh_session.py
index c59a46c..a48f019 100644
--- a/crawler/scenarios/fresh_session.py
+++ b/crawler/scenarios/fresh_session.py
@@ -2,9 +2,13 @@
 from __future__ import annotations
 
 import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 class FreshSessionScenario:
@@ -34,13 +38,24 @@ class FreshSessionScenario:
         if not self.url:
             return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
 
-        self.driver.get(self.url)
+        try:
+            self.driver.get(self.url)
+        except TimeoutException:
+            log.warning(
+                "Page load exceeded the browser timeout; stopping navigation and continuing"
+            )
+            self.driver.execute_cdp_cmd("Page.stopLoading", {})
         human_delay(2.0, 4.0)
 
-        # Wait for the page to reach a ready state
-        self.wait.until(
-            lambda d: d.execute_script("return document.readyState") == "complete"
-        )
+        # Interactive is sufficient for the dynamic flow and avoids waiting on
+        # analytics, ads, or other background resources indefinitely.
+        try:
+            self.wait.until(
+                lambda d: d.execute_script("return document.readyState")
+                in ("interactive", "complete")
+            )
+        except TimeoutException:
+            log.warning("Document did not report a ready state; continuing with the current DOM")
 
         title = self.driver.title
         current_url = self.driver.current_url
diff --git a/docker-compose.yml b/docker-compose.yml
index 3a8e633..6084c13 100644
--- a/docker-compose.yml
+++ b/docker-compose.yml
@@ -69,7 +69,7 @@ services:
       redis:
         condition: service_healthy
     # Chrome needs extra shared memory to avoid OOM crashes
-    shm_size: "256mb"
+    shm_size: "512mb"
     cap_add:
       - SYS_ADMIN
     security_opt: