nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

wip

Browse Source
This commit is contained in:
Navid Filsaraee
2026-06-16 16:56:27 +03:30
parent a5d09be46e
commit 24155ad6bf
22 changed files with 2485 additions and 762 deletions
Download Patch File Download Diff File Expand all files Collapse all files
crawler/dynamic_flow.py
+492 -76
View File
@@ -4,15 +4,17 @@ Dynamic flow executor — reads the active flow config from DB at runtime and ru
2. scroll_step : infinite-scroll until target item is visible or max_scrolls reached
3. click_step : click each target item id
"""
from __future__ import annotations
import asyncio
import math
import random
import time
from dataclasses import dataclass, field
from typing import Any
import undetected_chromedriver as uc
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.common.exceptions import NoSuchElementException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
@@ -21,6 +23,9 @@ from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
from logger import get_logger
log = get_logger(__name__)
class DynamicFlowError(Exception):
@@ -71,10 +76,30 @@ def _wait_for(
selector: str,
selector_type: str,
timeout: float = 15.0,
label: str = "",
) -> WebElement:
return WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((_by(selector_type), selector))
log.debug(
"Waiting for %s [%s:%s] (timeout=%ss)",
label or "element",
selector_type,
selector,
timeout,
)
try:
el = WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((_by(selector_type), selector))
)
log.debug("Found %s [%s:%s]", label or "element", selector_type, selector)
return el
except Exception as exc:
log.warning(
"Selector not found — %s [%s:%s]: %s",
label or "element",
selector_type,
selector,
exc,
)
raise
class DynamicFlow:
@@ -87,6 +112,7 @@ class DynamicFlow:
config_id=self.cfg["id"],
config_name=self.cfg["name"],
)
log.info("Starting flow '%s' (id=%s)", self.cfg["name"], self.cfg["id"])
search_texts: list[str] = self.cfg.get("search_texts_json") or []
item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
@@ -95,6 +121,7 @@ class DynamicFlow:
step = self._run_search(text)
result.steps.append(step)
if not step.success:
log.error("Search step failed for '%s' — aborting flow", text)
return result
human_delay(0.8, 1.5)
@@ -102,12 +129,21 @@ class DynamicFlow:
scroll_step = self._run_scroll_and_find(item_id)
result.steps.append(scroll_step)
if not scroll_step.success:
continue # try next item_id, don't abort the whole flow
log.warning(
"Scroll/find failed for item '%s' — skipping click", item_id
)
continue
click_step = self._run_click(item_id, scroll_step.data.get("element"))
result.steps.append(click_step)
human_delay(1.0, 2.5)
log.info(
"Flow '%s' finished — success=%s, steps=%d",
self.cfg["name"],
result.success,
len(result.steps),
)
return result
# ------------------------------------------------------------------
@@ -116,44 +152,70 @@ class DynamicFlow:
def _run_search(self, text: str) -> StepResult:
t0 = time.monotonic()
name = f"search:{text}"
log.info("[%s] Searching for '%s'", name, text)
try:
sel = self.cfg["search_box_selector"]
sel_type = self.cfg["search_box_selector_type"]
if not sel:
raise DynamicFlowError("search_box_selector is not configured")
box = _wait_for(self.driver, sel, sel_type)
box = _wait_for(self.driver, sel, sel_type, label="search box")
box.clear()
human_delay(0.3, 0.6)
# Type character by character for a human feel
for ch in text:
box.send_keys(ch)
time.sleep(0.04)
log.debug("[%s] Typed search text", name)
human_delay(0.4, 0.8)
# Submit: explicit button or Enter
submit_sel = self.cfg.get("search_submit_selector", "")
if submit_sel:
submit_type = self.cfg.get("search_submit_selector_type", "css")
btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
log.debug(
"[%s] Clicking submit button [%s:%s]", name, submit_type, submit_sel
)
btn = _wait_for(
self.driver,
submit_sel,
submit_type,
timeout=5.0,
label="submit button",
)
btn.click()
else:
log.debug("[%s] Submitting via Enter key", name)
box.send_keys(Keys.RETURN)
# Wait for results container if configured
results_sel = self.cfg.get("search_results_selector", "")
results_sel_type = self.cfg.get("search_results_selector_type", "css")
if results_sel:
_wait_for(self.driver, results_sel, "css", timeout=15.0)
log.debug(
"[%s] Waiting for results container [%s:%s]",
name,
results_sel_type,
results_sel,
)
_wait_for(
self.driver,
results_sel,
results_sel_type,
timeout=15.0,
label="results container",
)
human_delay(1.0, 2.0)
return StepResult(name=name, success=True, data={"text": text},
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.info("[%s] Search OK — %d ms", name, ms)
return StepResult(
name=name, success=True, data={"text": text}, duration_ms=ms
)
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Failed — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
# ------------------------------------------------------------------
# Step: scroll (infinite-scroll) until item selector matches
@@ -161,56 +223,270 @@ class DynamicFlow:
def _run_scroll_and_find(self, item_id: str) -> StepResult:
t0 = time.monotonic()
name = f"scroll_find:{item_id}"
log.info("[%s] Scrolling to find item '%s'", name, item_id)
try:
item_sel = self._build_item_selector(item_id)
item_sel_type = self.cfg.get("item_selector_type", "css")
item_sel, item_sel_type = self._build_item_selector(item_id)
scroll_container = self.cfg.get("scroll_container_selector", "").strip()
scroll_container_type = self.cfg.get(
"scroll_container_selector_type", "css"
)
max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
pagination_sel: str = self.cfg.get("pagination_selector", "").strip()
pagination_sel_type: str = self.cfg.get("pagination_selector_type", "css")
max_pages: int = int(self.cfg.get("max_pages", 10))
pagination_wait_ms: int = int(self.cfg.get("pagination_wait_ms", 1500))
log.debug(
"[%s] Selector [%s:%s], container=[%s:'%s'], max_scrolls=%d, pagination=%s",
name,
item_sel_type,
item_sel,
scroll_container_type,
scroll_container or "window",
max_scrolls,
(
f"[{pagination_sel_type}:{pagination_sel}] max_pages={max_pages}"
if pagination_sel
else "off"
),
)
# Check if already visible before scrolling
found = self._find_item_visible(item_sel, item_sel_type)
if found:
return StepResult(name=name, success=True,
data={"item_id": item_id, "scrolls": 0, "element": found},
duration_ms=int((time.monotonic() - t0) * 1000))
log.info("[%s] Item visible immediately (no scroll needed)", name)
return StepResult(
name=name,
success=True,
data={
"item_id": item_id,
"scrolls": 0,
"pages": 0,
"element": found,
},
duration_ms=int((time.monotonic() - t0) * 1000),
)
last_height = self._get_scroll_height(scroll_container)
last_height = self._get_scroll_height(
scroll_container, scroll_container_type
)
stale_since: float | None = None
total_scrolls = 0
page_num = 0
scroll_on_page = 0
for scroll_num in range(1, max_scrolls + 1):
self._scroll_down(scroll_container)
time.sleep(pause_ms / 1000)
while scroll_on_page < max_scrolls:
# Check for the item BEFORE scrolling so items near the top of a
# freshly-paginated page are seen before we jump to the bottom.
found = self._find_item_visible(item_sel, item_sel_type)
if found:
return StepResult(name=name, success=True,
data={"item_id": item_id, "scrolls": scroll_num, "element": found},
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.info(
"[%s] Found after %d scroll(s) on page %d — %d ms",
name,
total_scrolls,
page_num + 1,
ms,
)
return StepResult(
name=name,
success=True,
data={
"item_id": item_id,
"scrolls": total_scrolls,
"pages": page_num,
"element": found,
},
duration_ms=ms,
)
new_height = self._get_scroll_height(scroll_container)
scroll_on_page += 1
total_scrolls += 1
self._scroll_down(scroll_container, scroll_container_type)
time.sleep(pause_ms / 1000)
new_height = self._get_scroll_height(
scroll_container, scroll_container_type
)
if new_height == last_height:
if stale_since is None:
stale_since = time.monotonic()
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
raise DynamicFlowError(
f"No new content after {no_new_ms} ms — reached end of page "
f"without finding item '{item_id}'"
log.debug(
"[%s] Page height unchanged at scroll %d — starting stale timer",
name,
scroll_on_page,
)
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
# Infinite scroll exhausted — try pagination if configured
if pagination_sel and page_num < max_pages:
if self._try_paginate(
name,
pagination_sel,
pagination_sel_type,
pagination_wait_ms,
):
page_num += 1
scroll_on_page = 0
stale_since = None
last_height = self._get_scroll_height(
scroll_container, scroll_container_type
)
log.info(
"[%s] Advanced to pagination page %d",
name,
page_num + 1,
)
# Loop continues — item check happens at top of next iteration
else:
raise DynamicFlowError(
f"Pagination button [{pagination_sel_type}:{pagination_sel}] "
f"not visible on page {page_num + 1}"
)
elif pagination_sel and page_num >= max_pages:
raise DynamicFlowError(
f"Item '{item_id}' not found after {page_num} pagination page(s)"
)
else:
raise DynamicFlowError(
f"No new content after {no_new_ms} ms — reached end of page "
f"without finding item '{item_id}'"
)
else:
stale_since = None
last_height = new_height
log.debug(
"[%s] Scroll %d (page %d) — new height %d px",
name,
scroll_on_page,
page_num + 1,
new_height,
)
raise DynamicFlowError(
f"Item '{item_id}' not found after {max_scrolls} scrolls"
f"Item '{item_id}' not found after {total_scrolls} scrolls"
)
except DynamicFlowError:
raise
except DynamicFlowError as exc:
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Failed — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Unexpected error — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
# ------------------------------------------------------------------
# Human mouse simulation
# ------------------------------------------------------------------
def _human_click(self, el: WebElement, name: str = "") -> None:
"""
Simulate a human-like mouse approach and click:
1. Smooth scroll element into viewport
2. Dispatch JS mousemove events along a curved path
3. Trigger mouseenter / mouseover on the element
4. Click via ActionChains with random sub-element offset
5. Fallback: synthetic JS mouse event sequence
"""
# Step 1 — bring element into view smoothly
self.driver.execute_script(
"arguments[0].scrollIntoView({block:'center', behavior:'smooth'});", el
)
time.sleep(random.uniform(0.35, 0.7))
# Step 2 — get element centre in viewport coordinates
rect: dict[str, float] = self.driver.execute_script(
"""
const r = arguments[0].getBoundingClientRect();
return {x: r.left, y: r.top, w: r.width, h: r.height,
vw: window.innerWidth, vh: window.innerHeight};
""",
el,
)
cx = rect["x"] + rect["w"] / 2
cy = rect["y"] + rect["h"] / 2
vw = rect["vw"]
vh = rect["vh"]
# Step 3 — build a curved path from a random off-element start
sx = random.uniform(vw * 0.1, vw * 0.5)
sy = random.uniform(vh * 0.1, vh * 0.5)
# Control point for quadratic Bézier — pulled perpendicular to the straight line
dx, dy = cx - sx, cy - sy
perp_x = -dy * random.uniform(0.15, 0.4)
perp_y = dx * random.uniform(0.15, 0.4)
qx = sx + dx / 2 + perp_x
qy = sy + dy / 2 + perp_y
steps = random.randint(10, 18)
prev_mx, prev_my = sx, sy
for i in range(1, steps + 1):
t = i / steps
# Quadratic Bézier point
bx = (1 - t) ** 2 * sx + 2 * (1 - t) * t * qx + t**2 * cx
by = (1 - t) ** 2 * sy + 2 * (1 - t) * t * qy + t**2 * cy
# Add small Gaussian jitter that fades near the target
jitter = max(0.0, 1.0 - t) * 4.0
mx = bx + random.gauss(0, jitter)
my = by + random.gauss(0, jitter)
self.driver.execute_script(
"""
document.dispatchEvent(new MouseEvent('mousemove', {
bubbles: true, cancelable: true,
clientX: arguments[0], clientY: arguments[1]
}));
""",
int(mx),
int(my),
)
# Variable inter-step pause — faster in the middle, slower near target
speed = 0.5 + 0.5 * math.sin(math.pi * t)
time.sleep(random.uniform(0.008, 0.025) / max(speed, 0.3))
prev_mx, prev_my = mx, my
# Step 4 — hover events on the element itself
self.driver.execute_script(
"""
arguments[0].dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
arguments[0].dispatchEvent(new MouseEvent('mouseover', {bubbles: true}));
""",
el,
)
time.sleep(random.uniform(0.1, 0.28))
# Step 5 — ActionChains click with random offset from centre
off_x = int(random.uniform(-rect["w"] * 0.28, rect["w"] * 0.28))
off_y = int(random.uniform(-rect["h"] * 0.28, rect["h"] * 0.28))
log.debug(
"[%s] Human click — target=(%.0f,%.0f) offset=(%d,%d)",
name,
cx,
cy,
off_x,
off_y,
)
try:
ActionChains(self.driver).move_to_element_with_offset(
el, off_x, off_y
).pause(random.uniform(0.05, 0.14)).click().perform()
log.debug("[%s] ActionChains click OK", name)
except Exception as ac_exc:
log.warning(
"[%s] ActionChains failed (%s) — synthetic JS click", name, ac_exc
)
self.driver.execute_script(
"""
const opts = {bubbles: true, cancelable: true,
clientX: arguments[1], clientY: arguments[2]};
arguments[0].dispatchEvent(new MouseEvent('mousedown', opts));
arguments[0].dispatchEvent(new MouseEvent('mouseup', opts));
arguments[0].dispatchEvent(new MouseEvent('click', opts));
""",
el,
int(cx + off_x),
int(cy + off_y),
)
# ------------------------------------------------------------------
# Step: click the found element
@@ -218,65 +494,205 @@ class DynamicFlow:
def _run_click(self, item_id: str, element: Any) -> StepResult:
t0 = time.monotonic()
name = f"click:{item_id}"
log.info("[%s] Clicking item '%s'", name, item_id)
try:
if element is None:
raise DynamicFlowError("No element reference from scroll step")
el: WebElement = element
# Scroll element into view and click
self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
human_delay(0.3, 0.7)
try:
el.click()
except Exception:
# Fallback: JS click
self.driver.execute_script("arguments[0].click();", el)
self._human_click(element, name)
human_delay(0.8, 1.5)
return StepResult(name=name, success=True,
data={"item_id": item_id, "url_after": self.driver.current_url},
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.info(
"[%s] Click OK — landed on %s — %d ms",
name,
self.driver.current_url,
ms,
)
return StepResult(
name=name,
success=True,
data={"item_id": item_id, "url_after": self.driver.current_url},
duration_ms=ms,
)
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Failed — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
# ------------------------------------------------------------------
# Pagination helper
# ------------------------------------------------------------------
def _try_paginate(
self, step_name: str, sel: str, sel_type: str, wait_ms: int
) -> bool:
"""
Find the next-page button and click it reliably.
Uses a layered approach rather than full human-mouse simulation because
pagination buttons are small and offset-based clicking is fragile:
1. Native Selenium click (triggers real browser events)
2. JS click() on the element
3. Synthetic pointer + mouse + click event dispatch
"""
log.debug("[%s] Looking for pagination button [%s:%s]", step_name, sel_type, sel)
# Search all matching elements, not just the first visible one,
# in case the button is present but momentarily off-screen.
try:
candidates = _find_all(self.driver, sel, sel_type)
except Exception as exc:
log.warning("[%s] Pagination selector failed — %s", step_name, exc)
return False
btn: WebElement | None = None
for el in candidates:
try:
if el.is_displayed() and el.is_enabled():
btn = el
break
except Exception:
continue
if btn is None:
log.debug("[%s] Pagination button not found / not interactable", step_name)
return False
log.debug("[%s] Pagination button found — attempting click", step_name)
try:
self.driver.execute_script(
"arguments[0].scrollIntoView({block:'nearest', behavior:'instant'});", btn
)
time.sleep(random.uniform(0.15, 0.35))
# Layer 1: native Selenium click
try:
btn.click()
log.debug("[%s] Pagination: native click OK", step_name)
except Exception as e1:
log.debug("[%s] Pagination: native click failed (%s) — trying JS", step_name, e1)
# Layer 2: JS .click()
try:
self.driver.execute_script("arguments[0].click();", btn)
log.debug("[%s] Pagination: JS click OK", step_name)
except Exception as e2:
log.debug("[%s] Pagination: JS click failed (%s) — dispatching events", step_name, e2)
# Layer 3: synthetic pointer + mouse + click events
self.driver.execute_script(
"""
const el = arguments[0];
const r = el.getBoundingClientRect();
const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
const opts = {bubbles: true, cancelable: true, clientX: cx, clientY: cy};
['pointerdown','mousedown','pointerup','mouseup','click']
.forEach(t => el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, opts)));
""",
btn,
)
log.debug("[%s] Pagination: synthetic events dispatched", step_name)
log.info("[%s] Pagination button clicked — waiting %d ms", step_name, wait_ms)
time.sleep(wait_ms / 1000)
return True
except Exception as exc:
log.warning("[%s] Pagination click failed — %s", step_name, exc)
return False
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _build_item_selector(self, item_id: str) -> str:
def _build_item_selector(self, item_id: str) -> tuple[str, str]:
"""
Return (selector, effective_type).
Priority:
1. item_url_template — renders the URL pattern and uses XPath
contains(@href, ...) to find the matching <a> element.
2. item_selector_type == "a-link" — same href-contains approach but
driven by item_selector_template instead.
3. CSS / XPath — item_selector_template used verbatim.
Href-based searches always use XPath so that arbitrary URL characters
(colons, brackets, dots, slashes) never break the selector.
"""
url_tpl: str = self.cfg.get("item_url_template", "").strip()
if url_tpl:
href = url_tpl.replace("{item_id}", item_id)
log.debug("item_url_template resolved to href pattern '%s'", href)
return f"//a[contains(@href, '{href}')]", "xpath"
sel_type: str = self.cfg.get("item_selector_type", "css")
template: str = self.cfg.get("item_selector_template", "")
if sel_type == "a-link":
href_fragment = (
template.replace("{item_id}", item_id) if template else item_id
)
return f"//a[contains(@href, '{href_fragment}')]", "xpath"
if not template:
raise DynamicFlowError("item_selector_template is not configured")
return template.replace("{item_id}", item_id)
return template.replace("{item_id}", item_id), sel_type
def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
def _find_item_visible(
self, selector: str, selector_type: str
) -> WebElement | None:
try:
els = _find_all(self.driver, selector, selector_type)
for el in els:
except Exception as exc:
log.debug("Selector error [%s:%s]: %s", selector_type, selector, exc)
return None
for el in els:
try:
if el.is_displayed():
return el
except NoSuchElementException:
pass
except Exception:
continue
return None
def _get_scroll_height(self, container_sel: str) -> int:
def _get_scroll_height(
self, container_sel: str, container_type: str = "css"
) -> int:
body_h = int(self.driver.execute_script(
"return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)"
))
if container_sel:
try:
el = _find(self.driver, container_sel, "css")
return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
except Exception:
pass
return int(self.driver.execute_script("return document.body.scrollHeight"))
def _scroll_down(self, container_sel: str) -> None:
if container_sel:
try:
el = _find(self.driver, container_sel, "css")
self.driver.execute_script(
"arguments[0].scrollTop = arguments[0].scrollHeight", el
el = _find(self.driver, container_sel, container_type)
container_h = int(self.driver.execute_script("return arguments[0].scrollHeight", el))
return max(body_h, container_h)
except Exception as exc:
log.warning(
"Could not read scrollHeight from container [%s:'%s']: %s — using body",
container_type, container_sel, exc,
)
return
except Exception:
pass
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
return body_h
def _scroll_down(self, container_sel: str, container_type: str = "css") -> None:
if container_sel:
try:
el = _find(self.driver, container_sel, container_type)
self.driver.execute_script(
"""
const el = arguments[0];
el.scrollTop = el.scrollHeight;
el.dispatchEvent(new Event('scroll', {bubbles: true}));
""",
el,
)
except Exception as exc:
log.warning(
"Could not scroll container [%s:'%s']: %s — falling through to window",
container_type, container_sel, exc,
)
# Always advance the window too — most infinite-scroll triggers
# listen to window scroll events, not container scroll events.
self.driver.execute_script(
"""
window.scrollTo(0, document.body.scrollHeight);
window.dispatchEvent(new Event('scroll', {bubbles: true}));
document.dispatchEvent(new Event('scroll', {bubbles: true}));
"""
)