Harden browser session cleanup

crawler/driver.py

@@ -6,8 +6,10 @@ import os
 import random
 import re
 import shutil
+import signal
 import subprocess
 import sys
+import tempfile
 import time
 from contextlib import contextmanager
 from pathlib import Path
@@ -81,11 +83,19 @@ def _local_driver_path() -> str | None:
     return None
 
 
-def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
+def _build_options(
+    user_agent: str,
+    headless: bool,
+    chrome_binary: str,
+    profile_dir: str | None = None,
+) -> Options:
     opts = Options()
     opts.binary_location = chrome_binary
+    opts.page_load_strategy = "eager"
     if headless:
         opts.add_argument("--headless=new")
+    if profile_dir:
+        opts.add_argument(f"--user-data-dir={profile_dir}")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
     opts.add_argument("--disable-dev-shm-usage")
@@ -96,12 +106,15 @@ def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Optio
     return opts
 
 
-def make_driver(headless: bool | None = None) -> uc.Chrome:
+def make_driver(
+    headless: bool | None = None,
+    profile_dir: str | None = None,
+) -> uc.Chrome:
     """Return a stealthed undetected Chrome instance."""
     use_headless = config.headless if headless is None else headless
     chrome_binary, major = _detect_chrome()
     ua = random.choice(_USER_AGENTS)
-    opts = _build_options(ua, use_headless, chrome_binary)
+    opts = _build_options(ua, use_headless, chrome_binary, profile_dir)
 
     log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
 
@@ -116,6 +129,7 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
         driver.maximize_window()
     else:
         driver.set_window_size(1920, 1080)
+    driver.set_page_load_timeout(45)
 
     stealth(
         driver,
@@ -139,13 +153,48 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
     return driver
 
 
+def _profile_processes(profile_dir: str) -> list[int]:
+    marker = profile_dir.encode()
+    pids: list[int] = []
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        try:
+            if marker in (entry / "cmdline").read_bytes():
+                pids.append(int(entry.name))
+        except (FileNotFoundError, PermissionError, ProcessLookupError):
+            continue
+    return pids
+
+
+def _terminate_profile_processes(profile_dir: str) -> None:
+    for sig in (signal.SIGTERM, signal.SIGKILL):
+        pids = _profile_processes(profile_dir)
+        for pid in pids:
+            try:
+                os.kill(pid, sig)
+            except ProcessLookupError:
+                pass
+        if sig == signal.SIGTERM and pids:
+            time.sleep(0.5)
+
+
 @contextmanager
 def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
-    driver = make_driver(headless)
+    profile_dir = tempfile.mkdtemp(prefix="seed-chrome-")
+    driver: uc.Chrome | None = None
     try:
+        driver = make_driver(headless, profile_dir=profile_dir)
         yield driver
     finally:
-        driver.quit()
+        _terminate_profile_processes(profile_dir)
+        if driver is not None:
+            try:
+                driver.quit()
+            except Exception as exc:
+                log.warning("Driver shutdown error: %s", exc)
+        _terminate_profile_processes(profile_dir)
+        shutil.rmtree(profile_dir, ignore_errors=True)
         log.debug("Driver session closed")