init

crawler/driver.py

@@ -0,0 +1,83 @@
+"""Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
+from __future__ import annotations
+
+import random
+import time
+from contextlib import contextmanager
+from typing import Generator
+
+import undetected_chromedriver as uc
+from selenium.webdriver.chrome.options import Options
+from selenium_stealth import stealth
+
+from config import config
+
+_USER_AGENTS = [
+    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
+]
+
+
+def _build_options(user_agent: str, headless: bool) -> Options:
+    opts = Options()
+    if headless:
+        opts.add_argument("--headless=new")
+    opts.add_argument(f"--user-agent={user_agent}")
+    opts.add_argument("--no-sandbox")
+    opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-blink-features=AutomationControlled")
+    opts.add_argument("--disable-infobars")
+    opts.add_argument("--window-size=1920,1080")
+    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
+    opts.add_experimental_option("useAutomationExtension", False)
+    if config.chrome_binary:
+        opts.binary_location = config.chrome_binary
+    return opts
+
+
+def make_driver(headless: bool | None = None) -> uc.Chrome:
+    """Return a stealthed undetected Chrome instance."""
+    use_headless = config.headless if headless is None else headless
+    ua = random.choice(_USER_AGENTS)
+    opts = _build_options(ua, use_headless)
+
+    driver = uc.Chrome(options=opts, use_subprocess=True)
+
+    stealth(
+        driver,
+        languages=["en-US", "en"],
+        vendor="Google Inc.",
+        platform="Win32",
+        webgl_vendor="Intel Inc.",
+        renderer="Intel Iris OpenGL Engine",
+        fix_hairline=True,
+    )
+
+    # Mask navigator.webdriver via CDP
+    driver.execute_cdp_cmd(
+        "Page.addScriptToEvaluateOnNewDocument",
+        {
+            "source": """
+                Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
+                Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
+                Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en']});
+            """
+        },
+    )
+    return driver
+
+
+@contextmanager
+def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
+    """Context manager that guarantees driver.quit() on exit."""
+    driver = make_driver(headless)
+    try:
+        yield driver
+    finally:
+        driver.quit()
+
+
+def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
+    """Sleep for a random human-like interval."""
+    time.sleep(random.uniform(lo, hi))

crawler/dynamic_flow.py

@@ -0,0 +1,282 @@
+"""
+Dynamic flow executor — reads the active flow config from DB at runtime and runs:
+  1. search_step  : type each search text into the search box
+  2. scroll_step  : infinite-scroll until target item is visible or max_scrolls reached
+  3. click_step   : click each target item id
+"""
+from __future__ import annotations
+
+import asyncio
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import undetected_chromedriver as uc
+from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.webdriver.common.action_chains import ActionChains
+from selenium.webdriver.common.by import By
+from selenium.webdriver.common.keys import Keys
+from selenium.webdriver.remote.webelement import WebElement
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+from crawler.driver import human_delay
+
+
+class DynamicFlowError(Exception):
+    pass
+
+
+@dataclass
+class StepResult:
+    name: str
+    success: bool
+    data: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+    duration_ms: int = 0
+
+
+@dataclass
+class DynamicFlowResult:
+    config_id: int
+    config_name: str
+    steps: list[StepResult] = field(default_factory=list)
+
+    @property
+    def success(self) -> bool:
+        return all(s.success for s in self.steps)
+
+    @property
+    def failure_reason(self) -> str | None:
+        for s in self.steps:
+            if not s.success:
+                return f"{s.name}: {s.error}"
+        return None
+
+
+def _by(selector_type: str) -> str:
+    return By.XPATH if selector_type.lower() == "xpath" else By.CSS_SELECTOR
+
+
+def _find(driver: uc.Chrome, selector: str, selector_type: str) -> WebElement:
+    return driver.find_element(_by(selector_type), selector)
+
+
+def _find_all(driver: uc.Chrome, selector: str, selector_type: str) -> list[WebElement]:
+    return driver.find_elements(_by(selector_type), selector)
+
+
+def _wait_for(
+    driver: uc.Chrome,
+    selector: str,
+    selector_type: str,
+    timeout: float = 15.0,
+) -> WebElement:
+    return WebDriverWait(driver, timeout).until(
+        EC.presence_of_element_located((_by(selector_type), selector))
+    )
+
+
+class DynamicFlow:
+    def __init__(self, driver: uc.Chrome, cfg: dict[str, Any]) -> None:
+        self.driver = driver
+        self.cfg = cfg
+
+    def run(self) -> DynamicFlowResult:
+        result = DynamicFlowResult(
+            config_id=self.cfg["id"],
+            config_name=self.cfg["name"],
+        )
+
+        search_texts: list[str] = self.cfg.get("search_texts_json") or []
+        item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
+
+        for text in search_texts:
+            step = self._run_search(text)
+            result.steps.append(step)
+            if not step.success:
+                return result
+            human_delay(0.8, 1.5)
+
+            for item_id in item_ids:
+                scroll_step = self._run_scroll_and_find(item_id)
+                result.steps.append(scroll_step)
+                if not scroll_step.success:
+                    continue  # try next item_id, don't abort the whole flow
+
+                click_step = self._run_click(item_id, scroll_step.data.get("element"))
+                result.steps.append(click_step)
+                human_delay(1.0, 2.5)
+
+        return result
+
+    # ------------------------------------------------------------------
+    # Step: type into search box
+    # ------------------------------------------------------------------
+    def _run_search(self, text: str) -> StepResult:
+        t0 = time.monotonic()
+        name = f"search:{text}"
+        try:
+            sel = self.cfg["search_box_selector"]
+            sel_type = self.cfg["search_box_selector_type"]
+            if not sel:
+                raise DynamicFlowError("search_box_selector is not configured")
+
+            box = _wait_for(self.driver, sel, sel_type)
+            box.clear()
+            human_delay(0.3, 0.6)
+
+            # Type character by character for a human feel
+            for ch in text:
+                box.send_keys(ch)
+                time.sleep(0.04)
+
+            human_delay(0.4, 0.8)
+
+            # Submit: explicit button or Enter
+            submit_sel = self.cfg.get("search_submit_selector", "")
+            if submit_sel:
+                submit_type = self.cfg.get("search_submit_selector_type", "css")
+                btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
+                btn.click()
+            else:
+                box.send_keys(Keys.RETURN)
+
+            # Wait for results container if configured
+            results_sel = self.cfg.get("search_results_selector", "")
+            if results_sel:
+                _wait_for(self.driver, results_sel, "css", timeout=15.0)
+
+            human_delay(1.0, 2.0)
+            return StepResult(name=name, success=True, data={"text": text},
+                              duration_ms=int((time.monotonic() - t0) * 1000))
+
+        except Exception as exc:
+            return StepResult(name=name, success=False, error=str(exc),
+                              duration_ms=int((time.monotonic() - t0) * 1000))
+
+    # ------------------------------------------------------------------
+    # Step: scroll (infinite-scroll) until item selector matches
+    # ------------------------------------------------------------------
+    def _run_scroll_and_find(self, item_id: str) -> StepResult:
+        t0 = time.monotonic()
+        name = f"scroll_find:{item_id}"
+        try:
+            item_sel = self._build_item_selector(item_id)
+            item_sel_type = self.cfg.get("item_selector_type", "css")
+            scroll_container = self.cfg.get("scroll_container_selector", "").strip()
+            max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
+            pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
+            no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
+
+            # Check if already visible before scrolling
+            found = self._find_item_visible(item_sel, item_sel_type)
+            if found:
+                return StepResult(name=name, success=True,
+                                  data={"item_id": item_id, "scrolls": 0, "element": found},
+                                  duration_ms=int((time.monotonic() - t0) * 1000))
+
+            last_height = self._get_scroll_height(scroll_container)
+            stale_since: float | None = None
+
+            for scroll_num in range(1, max_scrolls + 1):
+                self._scroll_down(scroll_container)
+                time.sleep(pause_ms / 1000)
+
+                found = self._find_item_visible(item_sel, item_sel_type)
+                if found:
+                    return StepResult(name=name, success=True,
+                                      data={"item_id": item_id, "scrolls": scroll_num, "element": found},
+                                      duration_ms=int((time.monotonic() - t0) * 1000))
+
+                new_height = self._get_scroll_height(scroll_container)
+                if new_height == last_height:
+                    if stale_since is None:
+                        stale_since = time.monotonic()
+                    elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
+                        raise DynamicFlowError(
+                            f"No new content after {no_new_ms} ms — reached end of page "
+                            f"without finding item '{item_id}'"
+                        )
+                else:
+                    stale_since = None
+                    last_height = new_height
+
+            raise DynamicFlowError(
+                f"Item '{item_id}' not found after {max_scrolls} scrolls"
+            )
+
+        except DynamicFlowError:
+            raise
+        except Exception as exc:
+            return StepResult(name=name, success=False, error=str(exc),
+                              duration_ms=int((time.monotonic() - t0) * 1000))
+
+    # ------------------------------------------------------------------
+    # Step: click the found element
+    # ------------------------------------------------------------------
+    def _run_click(self, item_id: str, element: Any) -> StepResult:
+        t0 = time.monotonic()
+        name = f"click:{item_id}"
+        try:
+            if element is None:
+                raise DynamicFlowError("No element reference from scroll step")
+
+            el: WebElement = element
+            # Scroll element into view and click
+            self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
+            human_delay(0.3, 0.7)
+            try:
+                el.click()
+            except Exception:
+                # Fallback: JS click
+                self.driver.execute_script("arguments[0].click();", el)
+
+            human_delay(0.8, 1.5)
+            return StepResult(name=name, success=True,
+                              data={"item_id": item_id, "url_after": self.driver.current_url},
+                              duration_ms=int((time.monotonic() - t0) * 1000))
+
+        except Exception as exc:
+            return StepResult(name=name, success=False, error=str(exc),
+                              duration_ms=int((time.monotonic() - t0) * 1000))
+
+    # ------------------------------------------------------------------
+    # Helpers
+    # ------------------------------------------------------------------
+    def _build_item_selector(self, item_id: str) -> str:
+        template: str = self.cfg.get("item_selector_template", "")
+        if not template:
+            raise DynamicFlowError("item_selector_template is not configured")
+        return template.replace("{item_id}", item_id)
+
+    def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
+        try:
+            els = _find_all(self.driver, selector, selector_type)
+            for el in els:
+                if el.is_displayed():
+                    return el
+        except NoSuchElementException:
+            pass
+        return None
+
+    def _get_scroll_height(self, container_sel: str) -> int:
+        if container_sel:
+            try:
+                el = _find(self.driver, container_sel, "css")
+                return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
+            except Exception:
+                pass
+        return int(self.driver.execute_script("return document.body.scrollHeight"))
+
+    def _scroll_down(self, container_sel: str) -> None:
+        if container_sel:
+            try:
+                el = _find(self.driver, container_sel, "css")
+                self.driver.execute_script(
+                    "arguments[0].scrollTop = arguments[0].scrollHeight", el
+                )
+                return
+            except Exception:
+                pass
+        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")

crawler/flow_runner.py

@@ -0,0 +1,104 @@
+"""Executes a pre-defined sequence of flow steps and records results."""
+from __future__ import annotations
+
+import time
+from dataclasses import dataclass, field
+from typing import Callable
+
+import undetected_chromedriver as uc
+
+from config import config
+from crawler.driver import human_delay
+
+# A flow step is a callable that receives the driver and returns arbitrary data.
+StepFn = Callable[[uc.Chrome], dict[str, object]]
+
+# Registry — add your custom step functions here.
+# Keys must match the step names in config.flow_steps.
+STEP_REGISTRY: dict[str, StepFn] = {}
+
+
+def register_step(name: str) -> Callable[[StepFn], StepFn]:
+    """Decorator to register a function as a named flow step."""
+    def decorator(fn: StepFn) -> StepFn:
+        STEP_REGISTRY[name] = fn
+        return fn
+    return decorator
+
+
+@dataclass
+class StepResult:
+    name: str
+    success: bool
+    data: dict[str, object] = field(default_factory=dict)
+    error: str | None = None
+    duration_ms: int = 0
+
+
+@dataclass
+class FlowResult:
+    scenario: str
+    identifier: str          # phone number or "fresh"
+    steps: list[StepResult] = field(default_factory=list)
+
+    @property
+    def success(self) -> bool:
+        return all(s.success for s in self.steps)
+
+    @property
+    def failure_reason(self) -> str | None:
+        for s in self.steps:
+            if not s.success:
+                return f"{s.name}: {s.error}"
+        return None
+
+
+class FlowRunner:
+    def __init__(self, driver: uc.Chrome) -> None:
+        self.driver = driver
+
+    def run(self, scenario: str, identifier: str) -> FlowResult:
+        result = FlowResult(scenario=scenario, identifier=identifier)
+
+        for step_name in config.flow_steps:
+            fn = STEP_REGISTRY.get(step_name)
+            if fn is None:
+                result.steps.append(
+                    StepResult(
+                        name=step_name,
+                        success=False,
+                        error=f"Step '{step_name}' not found in registry",
+                    )
+                )
+                break
+
+            t0 = time.monotonic()
+            try:
+                data = fn(self.driver)
+                duration = int((time.monotonic() - t0) * 1000)
+                result.steps.append(StepResult(name=step_name, success=True, data=data, duration_ms=duration))
+                human_delay(0.5, 1.5)
+            except Exception as exc:
+                duration = int((time.monotonic() - t0) * 1000)
+                result.steps.append(
+                    StepResult(name=step_name, success=False, error=str(exc), duration_ms=duration)
+                )
+                break  # abort remaining steps on first failure
+
+        return result
+
+
+# ---------------------------------------------------------------------------
+# Example steps — replace with your actual flow logic
+# ---------------------------------------------------------------------------
+
+@register_step("step_home")
+def step_home(driver: uc.Chrome) -> dict[str, object]:
+    return {"url": driver.current_url, "title": driver.title}
+
+
+@register_step("step_browse")
+def step_browse(driver: uc.Chrome) -> dict[str, object]:
+    # Example: navigate to a listing page
+    human_delay(1.0, 2.0)
+    return {"url": driver.current_url}

crawler/scenarios/fresh_session.py

@@ -0,0 +1,52 @@
+"""Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
+from __future__ import annotations
+
+import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.support.ui import WebDriverWait
+
+from config import config
+from crawler.driver import human_delay, make_driver
+
+
+class FreshSessionScenario:
+    """
+    Opens the target URL in a brand-new browser profile (no stored cookies,
+    localStorage, or cache) so each run appears as a first-time visitor.
+
+    Because undetected-chromedriver already creates an isolated temp profile
+    per instance, simply spinning up a new driver is sufficient — but we also
+    explicitly delete all cookies after loading to be safe.
+    """
+
+    def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
+        self.driver = driver
+        self.wait = WebDriverWait(driver, timeout)
+
+    def run(self) -> dict[str, object]:
+        """Navigate to the target, clear any residual state, return page info."""
+        self.driver.delete_all_cookies()
+
+        # Clear localStorage / sessionStorage via JS
+        self.driver.execute_script(
+            "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
+        )
+
+        self.driver.get(config.target_url)
+        human_delay(2.0, 4.0)
+
+        # Wait for the page to reach a ready state
+        self.wait.until(
+            lambda d: d.execute_script("return document.readyState") == "complete"
+        )
+
+        title = self.driver.title
+        current_url = self.driver.current_url
+        cookies = self.driver.get_cookies()
+
+        return {
+            "title": title,
+            "url": current_url,
+            "cookie_count": len(cookies),
+            "cookies": cookies,
+        }

crawler/scenarios/otp_login.py

@@ -0,0 +1,106 @@
+"""Scenario 1 — SMS-OTP login with a batch of phone numbers."""
+from __future__ import annotations
+
+import time
+from typing import Callable
+
+import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException
+from selenium.webdriver.common.by import By
+from selenium.webdriver.support import expected_conditions as EC
+from selenium.webdriver.support.ui import WebDriverWait
+
+from config import config
+from crawler.driver import human_delay
+
+
+class OTPLoginError(Exception):
+    pass
+
+
+class OTPLoginScenario:
+    """
+    Drives an SMS-OTP login flow.
+
+    The caller supplies an `otp_resolver` — a callable that receives the
+    phone number and returns the OTP string.  Wire it up to your SMS
+    gateway / SIM-management API.
+    """
+
+    def __init__(
+        self,
+        driver: uc.Chrome,
+        phone: str,
+        otp_resolver: Callable[[str], str],
+        timeout: int = 30,
+    ) -> None:
+        self.driver = driver
+        self.phone = phone
+        self.otp_resolver = otp_resolver
+        self.wait = WebDriverWait(driver, timeout)
+
+    # ------------------------------------------------------------------
+    # Override these selectors to match the target site's actual HTML.
+    # ------------------------------------------------------------------
+    _PHONE_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[type='tel'], input[name='phone']")
+    _SEND_OTP_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
+    _OTP_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[name='otp'], input[autocomplete='one-time-code']")
+    _CONFIRM_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
+    _SUCCESS_INDICATOR = (By.CSS_SELECTOR, "[data-testid='home'], .dashboard, #main-content")
+
+    def run(self) -> dict[str, object]:
+        """Execute the OTP login. Returns a result dict."""
+        self.driver.get(config.target_url)
+        human_delay(1.5, 3.0)
+
+        try:
+            self._enter_phone()
+            self._request_otp()
+            otp = self._resolve_otp()
+            self._enter_otp(otp)
+            self._confirm_login()
+            self._wait_for_success()
+        except TimeoutException as exc:
+            raise OTPLoginError(f"Timeout during OTP login for {self.phone}") from exc
+
+        return {"phone": self.phone, "cookies": self.driver.get_cookies()}
+
+    def _enter_phone(self) -> None:
+        field = self.wait.until(EC.element_to_be_clickable(self._PHONE_FIELD_SELECTOR))
+        field.clear()
+        human_delay(0.3, 0.7)
+        for char in self.phone:
+            field.send_keys(char)
+            time.sleep(0.05)
+
+    def _request_otp(self) -> None:
+        btn = self.wait.until(EC.element_to_be_clickable(self._SEND_OTP_BTN_SELECTOR))
+        human_delay(0.5, 1.2)
+        btn.click()
+
+    def _resolve_otp(self) -> str:
+        # Poll for the OTP from the external resolver (SMS gateway / webhook)
+        for attempt in range(12):
+            human_delay(5.0, 8.0)
+            otp = self.otp_resolver(self.phone)
+            if otp:
+                return otp
+            if attempt == 11:
+                raise OTPLoginError(f"OTP not received for {self.phone} after 12 attempts")
+        return ""  # unreachable but satisfies type checker
+
+    def _enter_otp(self, otp: str) -> None:
+        field = self.wait.until(EC.element_to_be_clickable(self._OTP_FIELD_SELECTOR))
+        field.clear()
+        human_delay(0.3, 0.7)
+        for char in otp:
+            field.send_keys(char)
+            time.sleep(0.08)
+
+    def _confirm_login(self) -> None:
+        btn = self.wait.until(EC.element_to_be_clickable(self._CONFIRM_BTN_SELECTOR))
+        human_delay(0.5, 1.0)
+        btn.click()
+
+    def _wait_for_success(self) -> None:
+        self.wait.until(EC.presence_of_element_located(self._SUCCESS_INDICATOR))