diff --git a/crawler/driver.py b/crawler/driver.py index 43571a9..32e6ed0 100644 --- a/crawler/driver.py +++ b/crawler/driver.py @@ -6,8 +6,10 @@ import os import random import re import shutil +import signal import subprocess import sys +import tempfile import time from contextlib import contextmanager from pathlib import Path @@ -81,11 +83,19 @@ def _local_driver_path() -> str | None: return None -def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options: +def _build_options( + user_agent: str, + headless: bool, + chrome_binary: str, + profile_dir: str | None = None, +) -> Options: opts = Options() opts.binary_location = chrome_binary + opts.page_load_strategy = "eager" if headless: opts.add_argument("--headless=new") + if profile_dir: + opts.add_argument(f"--user-data-dir={profile_dir}") opts.add_argument(f"--user-agent={user_agent}") opts.add_argument("--no-sandbox") opts.add_argument("--disable-dev-shm-usage") @@ -96,12 +106,15 @@ def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Optio return opts -def make_driver(headless: bool | None = None) -> uc.Chrome: +def make_driver( + headless: bool | None = None, + profile_dir: str | None = None, +) -> uc.Chrome: """Return a stealthed undetected Chrome instance.""" use_headless = config.headless if headless is None else headless chrome_binary, major = _detect_chrome() ua = random.choice(_USER_AGENTS) - opts = _build_options(ua, use_headless, chrome_binary) + opts = _build_options(ua, use_headless, chrome_binary, profile_dir) log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major) @@ -116,6 +129,7 @@ def make_driver(headless: bool | None = None) -> uc.Chrome: driver.maximize_window() else: driver.set_window_size(1920, 1080) + driver.set_page_load_timeout(45) stealth( driver, @@ -139,13 +153,48 @@ def make_driver(headless: bool | None = None) -> uc.Chrome: return driver +def _profile_processes(profile_dir: str) -> list[int]: + marker = profile_dir.encode() + pids: list[int] = [] + for entry in Path("/proc").iterdir(): + if not entry.name.isdigit(): + continue + try: + if marker in (entry / "cmdline").read_bytes(): + pids.append(int(entry.name)) + except (FileNotFoundError, PermissionError, ProcessLookupError): + continue + return pids + + +def _terminate_profile_processes(profile_dir: str) -> None: + for sig in (signal.SIGTERM, signal.SIGKILL): + pids = _profile_processes(profile_dir) + for pid in pids: + try: + os.kill(pid, sig) + except ProcessLookupError: + pass + if sig == signal.SIGTERM and pids: + time.sleep(0.5) + + @contextmanager def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]: - driver = make_driver(headless) + profile_dir = tempfile.mkdtemp(prefix="seed-chrome-") + driver: uc.Chrome | None = None try: + driver = make_driver(headless, profile_dir=profile_dir) yield driver finally: - driver.quit() + _terminate_profile_processes(profile_dir) + if driver is not None: + try: + driver.quit() + except Exception as exc: + log.warning("Driver shutdown error: %s", exc) + _terminate_profile_processes(profile_dir) + shutil.rmtree(profile_dir, ignore_errors=True) log.debug("Driver session closed")