wip

crawler/batch.py

@@ -0,0 +1,117 @@
+"""Batch runner — spawn N parallel Chrome workers to execute the active flow config."""
+from __future__ import annotations
+
+import asyncio
+import threading
+import time
+import uuid
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Any, Callable
+
+from logger import get_logger
+
+log = get_logger(__name__)
+
+
+def _single_run(
+    run_index: int,
+    batch_id: str,
+    cfg: dict[str, Any],
+    headless: bool,
+) -> tuple[bool, str | None]:
+    """One isolated Chrome session. Returns (success, failure_reason)."""
+    from crawler.driver import driver_session
+    from crawler.dynamic_flow import DynamicFlow
+    from crawler.scenarios.fresh_session import FreshSessionScenario
+    from db.repositories.runs import runs_repo
+
+    identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
+    t0 = time.monotonic()
+    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+    try:
+        target_url: str = cfg.get("target_url") or ""
+        with driver_session(headless=headless) as driver:
+            FreshSessionScenario(driver, target_url).run()
+            result = DynamicFlow(driver, cfg).run() if cfg else None
+
+        total_ms = int((time.monotonic() - t0) * 1000)
+        if result:
+            steps_data = [
+                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
+                for s in result.steps
+            ]
+            asyncio.run(runs_repo.insert(
+                "batch", identifier, result.success,
+                result.failure_reason, steps_data, total_ms,
+            ))
+            log.info("[batch:%s] Run %d done — success=%s (%d ms)", batch_id[:8], run_index, result.success, total_ms)
+            return result.success, result.failure_reason
+        else:
+            asyncio.run(runs_repo.insert("batch", identifier, True, None, [], total_ms))
+            log.info("[batch:%s] Run %d done (no flow config) — %d ms", batch_id[:8], run_index, total_ms)
+            return True, None
+
+    except Exception as exc:
+        total_ms = int((time.monotonic() - t0) * 1000)
+        asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
+        log.error("[batch:%s] Run %d failed — %s (%d ms)", batch_id[:8], run_index, exc, total_ms)
+        return False, str(exc)
+
+
+def run_batch(
+    cfg: dict[str, Any],
+    workers: int,
+    total_runs: int,
+    headless: bool = True,
+    stagger_ms: int = 0,
+    on_progress: Callable[[int, int, int], None] | None = None,
+) -> dict[str, Any]:
+    """
+    Run `total_runs` crawler sessions with up to `workers` parallel Chrome processes.
+
+    `stagger_ms` delays each successive submission to the pool, spreading
+    Chrome startup load and reducing bot-detection fingerprint clustering.
+
+    `on_progress(completed, succeeded, failed)` is called after every finished run.
+
+    Returns {"batch_id", "total", "succeeded", "failed"}.
+    """
+    batch_id = uuid.uuid4().hex
+    log.info(
+        "Batch %s — %d run(s), %d worker(s), stagger=%d ms, headless=%s",
+        batch_id[:8], total_runs, workers, stagger_ms, headless,
+    )
+
+    succeeded = 0
+    failed = 0
+    lock = threading.Lock()
+
+    with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=f"batch-{batch_id[:8]}") as pool:
+        futures: list[Future[tuple[bool, str | None]]] = []
+        for i in range(total_runs):
+            futures.append(pool.submit(_single_run, i, batch_id, cfg, headless))
+            if stagger_ms > 0 and i < total_runs - 1:
+                time.sleep(stagger_ms / 1000)
+
+        for fut in as_completed(futures):
+            try:
+                success, _ = fut.result()
+            except Exception as exc:
+                log.error("[batch:%s] Unhandled thread exception: %s", batch_id[:8], exc)
+                success = False
+
+            with lock:
+                if success:
+                    succeeded += 1
+                else:
+                    failed += 1
+                completed = succeeded + failed
+
+            if on_progress:
+                on_progress(completed, succeeded, failed)
+
+    log.info(
+        "Batch %s finished — %d/%d succeeded, %d failed",
+        batch_id[:8], succeeded, total_runs, failed,
+    )
+    return {"batch_id": batch_id, "total": total_runs, "succeeded": succeeded, "failed": failed}

crawler/driver.py

@@ -1,9 +1,14 @@
 """Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
 from __future__ import annotations
 
+import os
 import random
+import shutil
+import subprocess
+import sys
 import time
 from contextlib import contextmanager
+from pathlib import Path
 from typing import Generator
 
 import undetected_chromedriver as uc
@@ -11,6 +16,16 @@ from selenium.webdriver.chrome.options import Options
 from selenium_stealth import stealth
 
 from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
+
+# Selenium talks to ChromeDriver over localhost — must not go through any system proxy.
+for _var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
+    os.environ.pop(_var, None)
+_no_proxy = "localhost,127.0.0.1,::1"
+os.environ.setdefault("no_proxy", _no_proxy)
+os.environ.setdefault("NO_PROXY", _no_proxy)
 
 _USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@@ -18,31 +33,79 @@ _USER_AGENTS = [
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
 ]
 
+_CHROME_CANDIDATES = [
+    "google-chrome-stable",
+    "google-chrome",
+    "chromium",
+    "chromium-browser",
+]
 
-def _build_options(user_agent: str, headless: bool) -> Options:
+
+def _detect_chrome() -> tuple[str, int]:
+    """Return (binary_path, major_version) for the first Chrome found."""
+    candidates = list(_CHROME_CANDIDATES)
+    if config.chrome_binary:
+        candidates.insert(0, config.chrome_binary)
+
+    for binary in candidates:
+        resolved = shutil.which(binary) or (binary if Path(binary).exists() else None)
+        if not resolved:
+            continue
+        try:
+            out = subprocess.check_output(
+                [resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
+            ).strip()
+            major = int(out.split()[-1].split(".")[0])
+            log.debug("Detected Chrome %d via '%s'", major, resolved)
+            return resolved, major
+        except Exception:
+            continue
+
+    log.critical("No Chrome binary found. Install Chrome or set CHROME_BINARY in .env")
+    sys.exit(1)
+
+
+def _local_driver_path() -> str | None:
+    path = Path(config.chromedriver_path)
+    if path.exists():
+        return str(path)
+    log.warning(
+        "No patched ChromeDriver at '%s'. Run 'make patch-driver' once. Falling back to auto-download.",
+        path,
+    )
+    return None
+
+
+def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
     opts = Options()
+    opts.binary_location = chrome_binary
     if headless:
         opts.add_argument("--headless=new")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
     opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
     opts.add_argument("--disable-blink-features=AutomationControlled")
     opts.add_argument("--disable-infobars")
     opts.add_argument("--window-size=1920,1080")
-    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
-    opts.add_experimental_option("useAutomationExtension", False)
-    if config.chrome_binary:
-        opts.binary_location = config.chrome_binary
     return opts
 
 
 def make_driver(headless: bool | None = None) -> uc.Chrome:
     """Return a stealthed undetected Chrome instance."""
     use_headless = config.headless if headless is None else headless
+    chrome_binary, major = _detect_chrome()
     ua = random.choice(_USER_AGENTS)
-    opts = _build_options(ua, use_headless)
+    opts = _build_options(ua, use_headless, chrome_binary)
 
-    driver = uc.Chrome(options=opts, use_subprocess=True)
+    log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
+
+    driver = uc.Chrome(
+        options=opts,
+        driver_executable_path=_local_driver_path(),
+        version_main=major,
+        use_subprocess=True,
+    )
 
     stealth(
         driver,
@@ -54,7 +117,6 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
         fix_hairline=True,
     )
 
-    # Mask navigator.webdriver via CDP
     driver.execute_cdp_cmd(
         "Page.addScriptToEvaluateOnNewDocument",
         {
@@ -65,19 +127,19 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
             """
         },
     )
+    log.info("Driver ready (Chrome %d)", major)
     return driver
 
 
 @contextmanager
 def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
-    """Context manager that guarantees driver.quit() on exit."""
     driver = make_driver(headless)
     try:
         yield driver
     finally:
         driver.quit()
+        log.debug("Driver session closed")
 
 
 def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
-    """Sleep for a random human-like interval."""
     time.sleep(random.uniform(lo, hi))

crawler/dynamic_flow.py

@@ -4,15 +4,17 @@ Dynamic flow executor — reads the active flow config from DB at runtime and ru
   2. scroll_step  : infinite-scroll until target item is visible or max_scrolls reached
   3. click_step   : click each target item id
 """
+
 from __future__ import annotations
 
-import asyncio
+import math
+import random
 import time
 from dataclasses import dataclass, field
 from typing import Any
 
 import undetected_chromedriver as uc
-from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -21,6 +23,9 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 class DynamicFlowError(Exception):
@@ -71,10 +76,30 @@ def _wait_for(
     selector: str,
     selector_type: str,
     timeout: float = 15.0,
+    label: str = "",
 ) -> WebElement:
-    return WebDriverWait(driver, timeout).until(
-        EC.presence_of_element_located((_by(selector_type), selector))
+    log.debug(
+        "Waiting for %s [%s:%s] (timeout=%ss)",
+        label or "element",
+        selector_type,
+        selector,
+        timeout,
     )
+    try:
+        el = WebDriverWait(driver, timeout).until(
+            EC.presence_of_element_located((_by(selector_type), selector))
+        )
+        log.debug("Found %s [%s:%s]", label or "element", selector_type, selector)
+        return el
+    except Exception as exc:
+        log.warning(
+            "Selector not found — %s [%s:%s]: %s",
+            label or "element",
+            selector_type,
+            selector,
+            exc,
+        )
+        raise
 
 
 class DynamicFlow:
@@ -87,6 +112,7 @@ class DynamicFlow:
             config_id=self.cfg["id"],
             config_name=self.cfg["name"],
         )
+        log.info("Starting flow '%s' (id=%s)", self.cfg["name"], self.cfg["id"])
 
         search_texts: list[str] = self.cfg.get("search_texts_json") or []
         item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
@@ -95,6 +121,7 @@ class DynamicFlow:
             step = self._run_search(text)
             result.steps.append(step)
             if not step.success:
+                log.error("Search step failed for '%s' — aborting flow", text)
                 return result
             human_delay(0.8, 1.5)
 
@@ -102,12 +129,21 @@ class DynamicFlow:
                 scroll_step = self._run_scroll_and_find(item_id)
                 result.steps.append(scroll_step)
                 if not scroll_step.success:
-                    continue  # try next item_id, don't abort the whole flow
+                    log.warning(
+                        "Scroll/find failed for item '%s' — skipping click", item_id
+                    )
+                    continue
 
                 click_step = self._run_click(item_id, scroll_step.data.get("element"))
                 result.steps.append(click_step)
                 human_delay(1.0, 2.5)
 
+        log.info(
+            "Flow '%s' finished — success=%s, steps=%d",
+            self.cfg["name"],
+            result.success,
+            len(result.steps),
+        )
         return result
 
     # ------------------------------------------------------------------
@@ -116,44 +152,70 @@ class DynamicFlow:
     def _run_search(self, text: str) -> StepResult:
         t0 = time.monotonic()
         name = f"search:{text}"
+        log.info("[%s] Searching for '%s'", name, text)
         try:
             sel = self.cfg["search_box_selector"]
             sel_type = self.cfg["search_box_selector_type"]
             if not sel:
                 raise DynamicFlowError("search_box_selector is not configured")
 
-            box = _wait_for(self.driver, sel, sel_type)
+            box = _wait_for(self.driver, sel, sel_type, label="search box")
             box.clear()
             human_delay(0.3, 0.6)
 
-            # Type character by character for a human feel
             for ch in text:
                 box.send_keys(ch)
                 time.sleep(0.04)
+            log.debug("[%s] Typed search text", name)
 
             human_delay(0.4, 0.8)
 
-            # Submit: explicit button or Enter
             submit_sel = self.cfg.get("search_submit_selector", "")
             if submit_sel:
                 submit_type = self.cfg.get("search_submit_selector_type", "css")
-                btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
+                log.debug(
+                    "[%s] Clicking submit button [%s:%s]", name, submit_type, submit_sel
+                )
+                btn = _wait_for(
+                    self.driver,
+                    submit_sel,
+                    submit_type,
+                    timeout=5.0,
+                    label="submit button",
+                )
                 btn.click()
             else:
+                log.debug("[%s] Submitting via Enter key", name)
                 box.send_keys(Keys.RETURN)
 
-            # Wait for results container if configured
             results_sel = self.cfg.get("search_results_selector", "")
+            results_sel_type = self.cfg.get("search_results_selector_type", "css")
             if results_sel:
-                _wait_for(self.driver, results_sel, "css", timeout=15.0)
+                log.debug(
+                    "[%s] Waiting for results container [%s:%s]",
+                    name,
+                    results_sel_type,
+                    results_sel,
+                )
+                _wait_for(
+                    self.driver,
+                    results_sel,
+                    results_sel_type,
+                    timeout=15.0,
+                    label="results container",
+                )
 
             human_delay(1.0, 2.0)
-            return StepResult(name=name, success=True, data={"text": text},
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            ms = int((time.monotonic() - t0) * 1000)
+            log.info("[%s] Search OK — %d ms", name, ms)
+            return StepResult(
+                name=name, success=True, data={"text": text}, duration_ms=ms
+            )
 
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            ms = int((time.monotonic() - t0) * 1000)
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
 
     # ------------------------------------------------------------------
     # Step: scroll (infinite-scroll) until item selector matches
@@ -161,56 +223,270 @@ class DynamicFlow:
     def _run_scroll_and_find(self, item_id: str) -> StepResult:
         t0 = time.monotonic()
         name = f"scroll_find:{item_id}"
+        log.info("[%s] Scrolling to find item '%s'", name, item_id)
         try:
-            item_sel = self._build_item_selector(item_id)
-            item_sel_type = self.cfg.get("item_selector_type", "css")
+            item_sel, item_sel_type = self._build_item_selector(item_id)
             scroll_container = self.cfg.get("scroll_container_selector", "").strip()
+            scroll_container_type = self.cfg.get(
+                "scroll_container_selector_type", "css"
+            )
             max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
             pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
             no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
+            pagination_sel: str = self.cfg.get("pagination_selector", "").strip()
+            pagination_sel_type: str = self.cfg.get("pagination_selector_type", "css")
+            max_pages: int = int(self.cfg.get("max_pages", 10))
+            pagination_wait_ms: int = int(self.cfg.get("pagination_wait_ms", 1500))
+
+            log.debug(
+                "[%s] Selector [%s:%s], container=[%s:'%s'], max_scrolls=%d, pagination=%s",
+                name,
+                item_sel_type,
+                item_sel,
+                scroll_container_type,
+                scroll_container or "window",
+                max_scrolls,
+                (
+                    f"[{pagination_sel_type}:{pagination_sel}] max_pages={max_pages}"
+                    if pagination_sel
+                    else "off"
+                ),
+            )
 
-            # Check if already visible before scrolling
             found = self._find_item_visible(item_sel, item_sel_type)
             if found:
-                return StepResult(name=name, success=True,
-                                  data={"item_id": item_id, "scrolls": 0, "element": found},
-                                  duration_ms=int((time.monotonic() - t0) * 1000))
+                log.info("[%s] Item visible immediately (no scroll needed)", name)
+                return StepResult(
+                    name=name,
+                    success=True,
+                    data={
+                        "item_id": item_id,
+                        "scrolls": 0,
+                        "pages": 0,
+                        "element": found,
+                    },
+                    duration_ms=int((time.monotonic() - t0) * 1000),
+                )
 
-            last_height = self._get_scroll_height(scroll_container)
+            last_height = self._get_scroll_height(
+                scroll_container, scroll_container_type
+            )
             stale_since: float | None = None
+            total_scrolls = 0
+            page_num = 0
+            scroll_on_page = 0
 
-            for scroll_num in range(1, max_scrolls + 1):
-                self._scroll_down(scroll_container)
-                time.sleep(pause_ms / 1000)
-
+            while scroll_on_page < max_scrolls:
+                # Check for the item BEFORE scrolling so items near the top of a
+                # freshly-paginated page are seen before we jump to the bottom.
                 found = self._find_item_visible(item_sel, item_sel_type)
                 if found:
-                    return StepResult(name=name, success=True,
-                                      data={"item_id": item_id, "scrolls": scroll_num, "element": found},
-                                      duration_ms=int((time.monotonic() - t0) * 1000))
+                    ms = int((time.monotonic() - t0) * 1000)
+                    log.info(
+                        "[%s] Found after %d scroll(s) on page %d — %d ms",
+                        name,
+                        total_scrolls,
+                        page_num + 1,
+                        ms,
+                    )
+                    return StepResult(
+                        name=name,
+                        success=True,
+                        data={
+                            "item_id": item_id,
+                            "scrolls": total_scrolls,
+                            "pages": page_num,
+                            "element": found,
+                        },
+                        duration_ms=ms,
+                    )
 
-                new_height = self._get_scroll_height(scroll_container)
+                scroll_on_page += 1
+                total_scrolls += 1
+                self._scroll_down(scroll_container, scroll_container_type)
+                time.sleep(pause_ms / 1000)
+
+                new_height = self._get_scroll_height(
+                    scroll_container, scroll_container_type
+                )
                 if new_height == last_height:
                     if stale_since is None:
                         stale_since = time.monotonic()
-                    elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
-                        raise DynamicFlowError(
-                            f"No new content after {no_new_ms} ms — reached end of page "
-                            f"without finding item '{item_id}'"
+                        log.debug(
+                            "[%s] Page height unchanged at scroll %d — starting stale timer",
+                            name,
+                            scroll_on_page,
                         )
+                    elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
+                        # Infinite scroll exhausted — try pagination if configured
+                        if pagination_sel and page_num < max_pages:
+                            if self._try_paginate(
+                                name,
+                                pagination_sel,
+                                pagination_sel_type,
+                                pagination_wait_ms,
+                            ):
+                                page_num += 1
+                                scroll_on_page = 0
+                                stale_since = None
+                                last_height = self._get_scroll_height(
+                                    scroll_container, scroll_container_type
+                                )
+                                log.info(
+                                    "[%s] Advanced to pagination page %d",
+                                    name,
+                                    page_num + 1,
+                                )
+                                # Loop continues — item check happens at top of next iteration
+                            else:
+                                raise DynamicFlowError(
+                                    f"Pagination button [{pagination_sel_type}:{pagination_sel}] "
+                                    f"not visible on page {page_num + 1}"
+                                )
+                        elif pagination_sel and page_num >= max_pages:
+                            raise DynamicFlowError(
+                                f"Item '{item_id}' not found after {page_num} pagination page(s)"
+                            )
+                        else:
+                            raise DynamicFlowError(
+                                f"No new content after {no_new_ms} ms — reached end of page "
+                                f"without finding item '{item_id}'"
+                            )
                 else:
                     stale_since = None
                     last_height = new_height
+                    log.debug(
+                        "[%s] Scroll %d (page %d) — new height %d px",
+                        name,
+                        scroll_on_page,
+                        page_num + 1,
+                        new_height,
+                    )
 
             raise DynamicFlowError(
-                f"Item '{item_id}' not found after {max_scrolls} scrolls"
+                f"Item '{item_id}' not found after {total_scrolls} scrolls"
             )
 
-        except DynamicFlowError:
-            raise
+        except DynamicFlowError as exc:
+            ms = int((time.monotonic() - t0) * 1000)
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            ms = int((time.monotonic() - t0) * 1000)
+            log.error("[%s] Unexpected error — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
+
+    # ------------------------------------------------------------------
+    # Human mouse simulation
+    # ------------------------------------------------------------------
+    def _human_click(self, el: WebElement, name: str = "") -> None:
+        """
+        Simulate a human-like mouse approach and click:
+          1. Smooth scroll element into viewport
+          2. Dispatch JS mousemove events along a curved path
+          3. Trigger mouseenter / mouseover on the element
+          4. Click via ActionChains with random sub-element offset
+          5. Fallback: synthetic JS mouse event sequence
+        """
+        # Step 1 — bring element into view smoothly
+        self.driver.execute_script(
+            "arguments[0].scrollIntoView({block:'center', behavior:'smooth'});", el
+        )
+        time.sleep(random.uniform(0.35, 0.7))
+
+        # Step 2 — get element centre in viewport coordinates
+        rect: dict[str, float] = self.driver.execute_script(
+            """
+            const r = arguments[0].getBoundingClientRect();
+            return {x: r.left, y: r.top, w: r.width, h: r.height,
+                    vw: window.innerWidth, vh: window.innerHeight};
+            """,
+            el,
+        )
+        cx = rect["x"] + rect["w"] / 2
+        cy = rect["y"] + rect["h"] / 2
+        vw = rect["vw"]
+        vh = rect["vh"]
+
+        # Step 3 — build a curved path from a random off-element start
+        sx = random.uniform(vw * 0.1, vw * 0.5)
+        sy = random.uniform(vh * 0.1, vh * 0.5)
+        # Control point for quadratic Bézier — pulled perpendicular to the straight line
+        dx, dy = cx - sx, cy - sy
+        perp_x = -dy * random.uniform(0.15, 0.4)
+        perp_y = dx * random.uniform(0.15, 0.4)
+        qx = sx + dx / 2 + perp_x
+        qy = sy + dy / 2 + perp_y
+
+        steps = random.randint(10, 18)
+        prev_mx, prev_my = sx, sy
+        for i in range(1, steps + 1):
+            t = i / steps
+            # Quadratic Bézier point
+            bx = (1 - t) ** 2 * sx + 2 * (1 - t) * t * qx + t**2 * cx
+            by = (1 - t) ** 2 * sy + 2 * (1 - t) * t * qy + t**2 * cy
+            # Add small Gaussian jitter that fades near the target
+            jitter = max(0.0, 1.0 - t) * 4.0
+            mx = bx + random.gauss(0, jitter)
+            my = by + random.gauss(0, jitter)
+
+            self.driver.execute_script(
+                """
+                document.dispatchEvent(new MouseEvent('mousemove', {
+                    bubbles: true, cancelable: true,
+                    clientX: arguments[0], clientY: arguments[1]
+                }));
+                """,
+                int(mx),
+                int(my),
+            )
+            # Variable inter-step pause — faster in the middle, slower near target
+            speed = 0.5 + 0.5 * math.sin(math.pi * t)
+            time.sleep(random.uniform(0.008, 0.025) / max(speed, 0.3))
+            prev_mx, prev_my = mx, my
+
+        # Step 4 — hover events on the element itself
+        self.driver.execute_script(
+            """
+            arguments[0].dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
+            arguments[0].dispatchEvent(new MouseEvent('mouseover',  {bubbles: true}));
+            """,
+            el,
+        )
+        time.sleep(random.uniform(0.1, 0.28))
+
+        # Step 5 — ActionChains click with random offset from centre
+        off_x = int(random.uniform(-rect["w"] * 0.28, rect["w"] * 0.28))
+        off_y = int(random.uniform(-rect["h"] * 0.28, rect["h"] * 0.28))
+        log.debug(
+            "[%s] Human click — target=(%.0f,%.0f) offset=(%d,%d)",
+            name,
+            cx,
+            cy,
+            off_x,
+            off_y,
+        )
+        try:
+            ActionChains(self.driver).move_to_element_with_offset(
+                el, off_x, off_y
+            ).pause(random.uniform(0.05, 0.14)).click().perform()
+            log.debug("[%s] ActionChains click OK", name)
+        except Exception as ac_exc:
+            log.warning(
+                "[%s] ActionChains failed (%s) — synthetic JS click", name, ac_exc
+            )
+            self.driver.execute_script(
+                """
+                const opts = {bubbles: true, cancelable: true,
+                              clientX: arguments[1], clientY: arguments[2]};
+                arguments[0].dispatchEvent(new MouseEvent('mousedown', opts));
+                arguments[0].dispatchEvent(new MouseEvent('mouseup',   opts));
+                arguments[0].dispatchEvent(new MouseEvent('click',     opts));
+                """,
+                el,
+                int(cx + off_x),
+                int(cy + off_y),
+            )
 
     # ------------------------------------------------------------------
     # Step: click the found element
@@ -218,65 +494,205 @@ class DynamicFlow:
     def _run_click(self, item_id: str, element: Any) -> StepResult:
         t0 = time.monotonic()
         name = f"click:{item_id}"
+        log.info("[%s] Clicking item '%s'", name, item_id)
         try:
             if element is None:
                 raise DynamicFlowError("No element reference from scroll step")
 
-            el: WebElement = element
-            # Scroll element into view and click
-            self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
-            human_delay(0.3, 0.7)
-            try:
-                el.click()
-            except Exception:
-                # Fallback: JS click
-                self.driver.execute_script("arguments[0].click();", el)
+            self._human_click(element, name)
 
             human_delay(0.8, 1.5)
-            return StepResult(name=name, success=True,
-                              data={"item_id": item_id, "url_after": self.driver.current_url},
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            ms = int((time.monotonic() - t0) * 1000)
+            log.info(
+                "[%s] Click OK — landed on %s — %d ms",
+                name,
+                self.driver.current_url,
+                ms,
+            )
+            return StepResult(
+                name=name,
+                success=True,
+                data={"item_id": item_id, "url_after": self.driver.current_url},
+                duration_ms=ms,
+            )
 
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            ms = int((time.monotonic() - t0) * 1000)
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
+
+    # ------------------------------------------------------------------
+    # Pagination helper
+    # ------------------------------------------------------------------
+    def _try_paginate(
+        self, step_name: str, sel: str, sel_type: str, wait_ms: int
+    ) -> bool:
+        """
+        Find the next-page button and click it reliably.
+
+        Uses a layered approach rather than full human-mouse simulation because
+        pagination buttons are small and offset-based clicking is fragile:
+          1. Native Selenium click (triggers real browser events)
+          2. JS click() on the element
+          3. Synthetic pointer + mouse + click event dispatch
+        """
+        log.debug("[%s] Looking for pagination button [%s:%s]", step_name, sel_type, sel)
+
+        # Search all matching elements, not just the first visible one,
+        # in case the button is present but momentarily off-screen.
+        try:
+            candidates = _find_all(self.driver, sel, sel_type)
+        except Exception as exc:
+            log.warning("[%s] Pagination selector failed — %s", step_name, exc)
+            return False
+
+        btn: WebElement | None = None
+        for el in candidates:
+            try:
+                if el.is_displayed() and el.is_enabled():
+                    btn = el
+                    break
+            except Exception:
+                continue
+
+        if btn is None:
+            log.debug("[%s] Pagination button not found / not interactable", step_name)
+            return False
+
+        log.debug("[%s] Pagination button found — attempting click", step_name)
+        try:
+            self.driver.execute_script(
+                "arguments[0].scrollIntoView({block:'nearest', behavior:'instant'});", btn
+            )
+            time.sleep(random.uniform(0.15, 0.35))
+
+            # Layer 1: native Selenium click
+            try:
+                btn.click()
+                log.debug("[%s] Pagination: native click OK", step_name)
+            except Exception as e1:
+                log.debug("[%s] Pagination: native click failed (%s) — trying JS", step_name, e1)
+                # Layer 2: JS .click()
+                try:
+                    self.driver.execute_script("arguments[0].click();", btn)
+                    log.debug("[%s] Pagination: JS click OK", step_name)
+                except Exception as e2:
+                    log.debug("[%s] Pagination: JS click failed (%s) — dispatching events", step_name, e2)
+                    # Layer 3: synthetic pointer + mouse + click events
+                    self.driver.execute_script(
+                        """
+                        const el = arguments[0];
+                        const r = el.getBoundingClientRect();
+                        const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
+                        const opts = {bubbles: true, cancelable: true, clientX: cx, clientY: cy};
+                        ['pointerdown','mousedown','pointerup','mouseup','click']
+                            .forEach(t => el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, opts)));
+                        """,
+                        btn,
+                    )
+                    log.debug("[%s] Pagination: synthetic events dispatched", step_name)
+
+            log.info("[%s] Pagination button clicked — waiting %d ms", step_name, wait_ms)
+            time.sleep(wait_ms / 1000)
+            return True
+
+        except Exception as exc:
+            log.warning("[%s] Pagination click failed — %s", step_name, exc)
+            return False
 
     # ------------------------------------------------------------------
     # Helpers
     # ------------------------------------------------------------------
-    def _build_item_selector(self, item_id: str) -> str:
+    def _build_item_selector(self, item_id: str) -> tuple[str, str]:
+        """
+        Return (selector, effective_type).
+
+        Priority:
+          1. item_url_template — renders the URL pattern and uses XPath
+             contains(@href, ...) to find the matching <a> element.
+          2. item_selector_type == "a-link" — same href-contains approach but
+             driven by item_selector_template instead.
+          3. CSS / XPath — item_selector_template used verbatim.
+
+        Href-based searches always use XPath so that arbitrary URL characters
+        (colons, brackets, dots, slashes) never break the selector.
+        """
+        url_tpl: str = self.cfg.get("item_url_template", "").strip()
+        if url_tpl:
+            href = url_tpl.replace("{item_id}", item_id)
+            log.debug("item_url_template resolved to href pattern '%s'", href)
+            return f"//a[contains(@href, '{href}')]", "xpath"
+
+        sel_type: str = self.cfg.get("item_selector_type", "css")
         template: str = self.cfg.get("item_selector_template", "")
+
+        if sel_type == "a-link":
+            href_fragment = (
+                template.replace("{item_id}", item_id) if template else item_id
+            )
+            return f"//a[contains(@href, '{href_fragment}')]", "xpath"
+
         if not template:
             raise DynamicFlowError("item_selector_template is not configured")
-        return template.replace("{item_id}", item_id)
+        return template.replace("{item_id}", item_id), sel_type
 
-    def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
+    def _find_item_visible(
+        self, selector: str, selector_type: str
+    ) -> WebElement | None:
         try:
             els = _find_all(self.driver, selector, selector_type)
-            for el in els:
+        except Exception as exc:
+            log.debug("Selector error [%s:%s]: %s", selector_type, selector, exc)
+            return None
+        for el in els:
+            try:
                 if el.is_displayed():
                     return el
-        except NoSuchElementException:
-            pass
+            except Exception:
+                continue
         return None
 
-    def _get_scroll_height(self, container_sel: str) -> int:
+    def _get_scroll_height(
+        self, container_sel: str, container_type: str = "css"
+    ) -> int:
+        body_h = int(self.driver.execute_script(
+            "return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)"
+        ))
         if container_sel:
             try:
-                el = _find(self.driver, container_sel, "css")
-                return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
-            except Exception:
-                pass
-        return int(self.driver.execute_script("return document.body.scrollHeight"))
-
-    def _scroll_down(self, container_sel: str) -> None:
-        if container_sel:
-            try:
-                el = _find(self.driver, container_sel, "css")
-                self.driver.execute_script(
-                    "arguments[0].scrollTop = arguments[0].scrollHeight", el
+                el = _find(self.driver, container_sel, container_type)
+                container_h = int(self.driver.execute_script("return arguments[0].scrollHeight", el))
+                return max(body_h, container_h)
+            except Exception as exc:
+                log.warning(
+                    "Could not read scrollHeight from container [%s:'%s']: %s — using body",
+                    container_type, container_sel, exc,
                 )
-                return
-            except Exception:
-                pass
-        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+        return body_h
+
+    def _scroll_down(self, container_sel: str, container_type: str = "css") -> None:
+        if container_sel:
+            try:
+                el = _find(self.driver, container_sel, container_type)
+                self.driver.execute_script(
+                    """
+                    const el = arguments[0];
+                    el.scrollTop = el.scrollHeight;
+                    el.dispatchEvent(new Event('scroll', {bubbles: true}));
+                    """,
+                    el,
+                )
+            except Exception as exc:
+                log.warning(
+                    "Could not scroll container [%s:'%s']: %s — falling through to window",
+                    container_type, container_sel, exc,
+                )
+        # Always advance the window too — most infinite-scroll triggers
+        # listen to window scroll events, not container scroll events.
+        self.driver.execute_script(
+            """
+            window.scrollTo(0, document.body.scrollHeight);
+            window.dispatchEvent(new Event('scroll', {bubbles: true}));
+            document.dispatchEvent(new Event('scroll', {bubbles: true}));
+            """
+        )

crawler/scenarios/fresh_session.py

@@ -2,11 +2,9 @@
 from __future__ import annotations
 
 import undetected_chromedriver as uc
-from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 
-from config import config
-from crawler.driver import human_delay, make_driver
+from crawler.driver import human_delay
 
 
 class FreshSessionScenario:
@@ -19,8 +17,9 @@ class FreshSessionScenario:
     explicitly delete all cookies after loading to be safe.
     """
 
-    def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
+    def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
         self.driver = driver
+        self.url = url
         self.wait = WebDriverWait(driver, timeout)
 
     def run(self) -> dict[str, object]:
@@ -32,7 +31,7 @@ class FreshSessionScenario:
             "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
         )
 
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
         human_delay(2.0, 4.0)
 
         # Wait for the page to reach a ready state

crawler/scenarios/otp_login.py

@@ -10,7 +10,6 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
-from config import config
 from crawler.driver import human_delay
 
 
@@ -32,11 +31,13 @@ class OTPLoginScenario:
         driver: uc.Chrome,
         phone: str,
         otp_resolver: Callable[[str], str],
+        url: str,
         timeout: int = 30,
     ) -> None:
         self.driver = driver
         self.phone = phone
         self.otp_resolver = otp_resolver
+        self.url = url
         self.wait = WebDriverWait(driver, timeout)
 
     # ------------------------------------------------------------------
@@ -50,7 +51,7 @@ class OTPLoginScenario:
 
     def run(self) -> dict[str, object]:
         """Execute the OTP login. Returns a result dict."""
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
         human_delay(1.5, 3.0)
 
         try:

crawler/tasks.py

@@ -0,0 +1,214 @@
+"""Huey task definitions and shared batch state for the admin panel."""
+from __future__ import annotations
+
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from huey import MemoryHuey
+
+huey = MemoryHuey("seed", immediate=False)
+
+# ── Shared state (all accessed under _lock) ───────────────────────────────────
+_lock = threading.Lock()
+
+_current_batch_id: str | None = None
+_current_result: Any = None          # Huey Result — kept for revocation
+
+_batch_meta: dict[str, Any] = {}     # batch_id → aggregate metadata
+_worker_slots: dict[str, dict] = {}  # f"{batch_id}:{slot:04d}" → worker info
+_stop_events: dict[str, threading.Event] = {}
+
+
+# ── Public helpers ─────────────────────────────────────────────────────────────
+
+def get_current_batch_id() -> str | None:
+    with _lock:
+        return _current_batch_id
+
+
+def set_current(batch_id: str, result: Any) -> None:
+    global _current_batch_id, _current_result
+    with _lock:
+        _current_batch_id = batch_id
+        _current_result = result
+
+
+def stop_current() -> bool:
+    """Signal the running batch to stop. Returns True if there was something to stop."""
+    with _lock:
+        bid = _current_batch_id
+        result = _current_result
+
+    if bid is None:
+        return False
+
+    # Revoke task if it hasn't started yet (still pending in Huey queue)
+    if result is not None:
+        try:
+            result.revoke(revoke_once=True)
+        except Exception:
+            pass
+
+    # Signal running workers to stop after their current step
+    ev = _stop_events.get(bid)
+    if ev:
+        ev.set()
+
+    with _lock:
+        if bid in _batch_meta:
+            _batch_meta[bid]["stopping"] = True
+
+    return True
+
+
+def batch_status() -> dict[str, Any]:
+    with _lock:
+        bid = _current_batch_id
+        if bid is None:
+            return {"batch_id": None, "running": False}
+
+        meta = dict(_batch_meta.get(bid, {}))
+        prefix = f"{bid}:"
+        workers = [dict(v) for k, v in _worker_slots.items() if k.startswith(prefix)]
+
+    meta["batch_id"] = bid
+    meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0))
+    meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running")
+    return meta
+
+
+# ── Huey task ─────────────────────────────────────────────────────────────────
+
+@huey.task(name="run_batch")
+def run_batch_task(
+    batch_id: str,
+    cfg: dict[str, Any],
+    workers: int,
+    total_runs: int,
+    headless: bool,
+    stagger_ms: int,
+) -> dict[str, Any]:
+    import asyncio
+
+    from crawler.driver import driver_session
+    from crawler.dynamic_flow import DynamicFlow
+    from crawler.scenarios.fresh_session import FreshSessionScenario
+    from db.repositories.runs import runs_repo
+    from logger import get_logger
+
+    log = get_logger(__name__)
+
+    stop_ev = threading.Event()
+    with _lock:
+        _stop_events[batch_id] = stop_ev
+        _batch_meta[batch_id] = {
+            "running": True,
+            "stopping": False,
+            "total": total_runs,
+            "completed": 0,
+            "succeeded": 0,
+            "failed": 0,
+            "configured_workers": workers,
+            "started_at": time.time(),
+            "finished_at": None,
+            "error": None,
+        }
+
+    def _run_one(slot: int) -> bool:
+        slot_key = f"{batch_id}:{slot:04d}"
+
+        if stop_ev.is_set():
+            with _lock:
+                _worker_slots[slot_key] = {
+                    "slot": slot, "status": "skipped",
+                    "started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped",
+                }
+            return False
+
+        with _lock:
+            _worker_slots[slot_key] = {
+                "slot": slot, "status": "running",
+                "started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None,
+            }
+
+        t0 = time.monotonic()
+        identifier = f"batch-{batch_id[:8]}-{slot:04d}"
+        log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
+
+        try:
+            target_url: str = cfg.get("target_url") or ""
+            with driver_session(headless=headless) as driver:
+                FreshSessionScenario(driver, target_url).run()
+                result = DynamicFlow(driver, cfg).run() if cfg else None
+
+            total_ms = int((time.monotonic() - t0) * 1000)
+            success = result.success if result else True
+            reason = result.failure_reason if result else None
+            steps = [
+                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
+                for s in (result.steps if result else [])
+            ]
+            asyncio.run(runs_repo.insert("batch", identifier, success, reason, steps, total_ms))
+
+            status = "ok" if success else "failed"
+            with _lock:
+                _worker_slots[slot_key].update({
+                    "status": status, "finished_at": time.time(), "duration_ms": total_ms,
+                })
+            log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms)
+            return success
+
+        except Exception as exc:
+            total_ms = int((time.monotonic() - t0) * 1000)
+            asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
+            with _lock:
+                _worker_slots[slot_key].update({
+                    "status": "error", "error": str(exc),
+                    "finished_at": time.time(), "duration_ms": total_ms,
+                })
+            log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc)
+            return False
+
+    succeeded = 0
+    failed = 0
+
+    try:
+        with ThreadPoolExecutor(
+            max_workers=workers,
+            thread_name_prefix=f"batch-{batch_id[:8]}",
+        ) as pool:
+            futures = []
+            for i in range(total_runs):
+                if stop_ev.is_set():
+                    break
+                futures.append(pool.submit(_run_one, i))
+                if stagger_ms > 0 and i < total_runs - 1:
+                    if stop_ev.wait(timeout=stagger_ms / 1000):
+                        break  # stop was signalled during the stagger delay
+
+            for fut in as_completed(futures):
+                try:
+                    ok = fut.result()
+                except Exception:
+                    ok = False
+                if ok:
+                    succeeded += 1
+                else:
+                    failed += 1
+                with _lock:
+                    _batch_meta[batch_id]["completed"] = succeeded + failed
+                    _batch_meta[batch_id]["succeeded"] = succeeded
+                    _batch_meta[batch_id]["failed"] = failed
+
+    except Exception as exc:
+        with _lock:
+            _batch_meta[batch_id]["error"] = str(exc)
+
+    finally:
+        with _lock:
+            _batch_meta[batch_id]["running"] = False
+            _batch_meta[batch_id]["finished_at"] = time.time()
+
+    return {"batch_id": batch_id, "succeeded": succeeded, "failed": failed}