diff --git a/admin/static/dashboard.html b/admin/static/dashboard.html index 1ac44bf..2052851 100644 --- a/admin/static/dashboard.html +++ b/admin/static/dashboard.html @@ -204,8 +204,8 @@ -

Paste URL fragment from goToProduct('https://…') — e.g. roshdbook.ir/product/شیمی-دهم/

- +

Paste URL fragment from goToProduct('https://…')— e.g. roshdbook.ir/product/شیمی-دهم/

+
@@ -821,7 +821,8 @@ function renderConfigs() { #${c.id}
- Searches: ${(c.search_texts_json || []).length} text(s)  ·  + ${esc(c.scenario || 'dynamic')} +  ·  Searches: ${(c.search_texts_json || []).length} text(s)  ·  Items: ${(c.target_item_ids_json || []).length} ID(s)  ·  Max scrolls: ${c.max_scrolls}  ·  Search box: ${esc(c.search_box_selector || '—')} @@ -846,7 +847,7 @@ function _refreshBatchConfigSelect() { const sel = document.getElementById('batch-config-select'); const cur = sel.value; sel.innerHTML = '' + - _configs.map(c => ``).join(''); + _configs.map(c => ``).join(''); } async function deleteConfig(id) { diff --git a/crawler/driver.py b/crawler/driver.py index a9878e8..80862fe 100644 --- a/crawler/driver.py +++ b/crawler/driver.py @@ -1,4 +1,5 @@ """Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection.""" + from __future__ import annotations import os @@ -107,6 +108,11 @@ def make_driver(headless: bool | None = None) -> uc.Chrome: use_subprocess=True, ) + if not use_headless: + driver.maximize_window() + else: + driver.set_window_size(1920, 1080) + stealth( driver, languages=["en-US", "en"], @@ -119,13 +125,11 @@ def make_driver(headless: bool | None = None) -> uc.Chrome: driver.execute_cdp_cmd( "Page.addScriptToEvaluateOnNewDocument", - { - "source": """ + {"source": """ Object.defineProperty(navigator, 'webdriver', {get: () => undefined}); Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]}); Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en']}); - """ - }, + """}, ) log.info("Driver ready (Chrome %d)", major) return driver diff --git a/crawler/scenarios/digipay.py b/crawler/scenarios/digipay.py index 14d241f..7c3072f 100644 --- a/crawler/scenarios/digipay.py +++ b/crawler/scenarios/digipay.py @@ -36,13 +36,41 @@ _SEARCH_TRIGGER_CANDIDATES = [ ] +_CONTAINER_FIX_JS = """ +(function() { + function inject() { + if (document.getElementById('__seed_fix')) return; + var s = document.createElement('style'); + s.id = '__seed_fix'; + s.textContent = '.container{margin-left:0!important;margin-right:0!important}'; + document.head.appendChild(s); + } + inject(); + new MutationObserver(inject).observe(document.head, {childList: true}); + new MutationObserver(inject).observe(document.documentElement, {childList: true}); +})(); +""" + + +def _inject_container_fix(driver: uc.Chrome) -> None: + driver.execute_script(_CONTAINER_FIX_JS) + + def _wait_spa_ready(driver: uc.Chrome, timeout: float = 20.0) -> None: """Wait for DOM ready + Vue/React hydration (no pending network requests).""" + # Force desktop viewport — --window-size sets outer chrome, not inner viewport + driver.execute_cdp_cmd("Emulation.setDeviceMetricsOverride", { + "width": 1920, + "height": 1080, + "deviceScaleFactor": 1, + "mobile": False, + }) WebDriverWait(driver, timeout).until( lambda d: d.execute_script("return document.readyState") == "complete" ) # Extra pause for SPA JS to hydrate time.sleep(1.5) + _inject_container_fix(driver) def _find_search_input(driver: uc.Chrome, wait: WebDriverWait) -> Any: @@ -106,6 +134,7 @@ class DigiPayFlow(DynamicFlow): wait_long.until( EC.presence_of_element_located((By.CSS_SELECTOR, _RESULTS_WAIT_CSS)) ) + _inject_container_fix(self.driver) human_delay(1.0, 2.0) ms = int((time.monotonic() - t0) * 1000)