nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Digipay and speedtest added

Browse Source
This commit is contained in:
Navid Filsaraee
2026-06-27 16:18:33 +03:30
parent 64886b2b10
commit 744550deaa
8 changed files with 421 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/routes.py
+66 -6
View File
@@ -1,9 +1,11 @@
"""FastAPI routes — dashboard API + auth."""
from __future__ import annotations
import secrets
from typing import Annotated, Any
import speedtest # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel
@@ -24,9 +26,15 @@ router = APIRouter()
security = HTTPBasic()
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
def require_auth(
credentials: Annotated[HTTPBasicCredentials, Depends(security)],
) -> str:
ok_user = secrets.compare_digest(
credentials.username.encode(), config.admin_username.encode()
)
ok_pass = secrets.compare_digest(
credentials.password.encode(), config.admin_password.encode()
)
if not (ok_user and ok_pass):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
@@ -43,6 +51,7 @@ Auth = Annotated[str, Depends(require_auth)]
# Stats
# ---------------------------------------------------------------------------
@router.get("/api/stats")
async def stats(_: Auth) -> dict[str, object]:
return await get_stats()
@@ -52,6 +61,7 @@ async def stats(_: Auth) -> dict[str, object]:
# Flow configs
# ---------------------------------------------------------------------------
class FlowConfigIn(BaseModel):
name: str
search_texts: list[str] = []
@@ -75,6 +85,7 @@ class FlowConfigIn(BaseModel):
item_url_template: str = ""
target_item_ids: list[str] = []
target_url: str = ""
scenario: str = "dynamic"
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -101,6 +112,7 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
"item_url_template": body.item_url_template,
"target_item_ids_json": body.target_item_ids,
"target_url": body.target_url,
"scenario": body.scenario,
}
@@ -146,6 +158,7 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
# Batch runs (managed via Huey task queue)
# ---------------------------------------------------------------------------
class BatchTaskIn(BaseModel):
config_id: int
workers: int = 3
@@ -160,17 +173,25 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
cfg = await get_flow_config(body.config_id)
if not cfg:
raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found")
raise HTTPException(
status_code=404, detail=f"Flow config {body.config_id} not found"
)
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
log.info("Task enqueued — config=%d runs=%d workers=%d status=%s",
body.config_id, body.total_runs, body.workers, result["status"])
log.info(
"Task enqueued — config=%d runs=%d workers=%d status=%s",
body.config_id,
body.total_runs,
body.workers,
result["status"],
)
return result
@router.delete("/api/batch-queue/{queue_id}", status_code=200)
async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
from crawler.tasks import dequeue
removed = dequeue(queue_id)
if not removed:
raise HTTPException(status_code=404, detail="Queue item not found")
@@ -180,6 +201,7 @@ async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
@router.post("/api/batch-queue/clear", status_code=200)
async def clear_batch_queue(_: Auth) -> dict[str, object]:
from crawler.tasks import clear_queue
count = clear_queue()
return {"cleared": count}
@@ -187,6 +209,7 @@ async def clear_batch_queue(_: Auth) -> dict[str, object]:
@router.post("/api/stop-batch", status_code=200)
async def stop_batch(_: Auth) -> dict[str, object]:
from crawler.tasks import stop_current
stopped = stop_current()
if not stopped:
raise HTTPException(status_code=404, detail="No active batch to stop")
@@ -196,12 +219,14 @@ async def stop_batch(_: Auth) -> dict[str, object]:
@router.get("/api/batch-status")
async def batch_status(_: Auth) -> dict[str, object]:
from crawler.tasks import batch_status as _status
return _status() # type: ignore[return-value]
@router.get("/api/batch-runs")
async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
from db.repositories.batch_runs import batch_runs_repo
return await batch_runs_repo.list() # type: ignore[return-value]
@@ -209,6 +234,7 @@ async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
# Log viewer
# ---------------------------------------------------------------------------
@router.get("/api/logs")
def get_logs(
_: Auth,
@@ -217,6 +243,7 @@ def get_logs(
q: str = "",
) -> dict[str, object]:
import os
from logger import _LOG_FILE
path = _LOG_FILE
@@ -245,3 +272,36 @@ def get_logs(
break
return {"lines": results, "file": path}
# ---------------------------------------------------------------------------
# Speed test
# ---------------------------------------------------------------------------
@router.post("/api/speedtest")
async def run_speedtest(_: Auth) -> dict[str, object]:
import asyncio
def _run() -> dict[str, object]:
try:
log.info("running speedtest")
st = speedtest.Speedtest(secure=True)
st.get_best_server()
download_bps: float = st.download()
upload_bps: float = st.upload()
r = st.results.dict()
server: dict[str, object] = r.get("server") or {}
client: dict[str, object] = r.get("client") or {}
return {
"download_mbps": round(download_bps / 1_000_000, 2),
"upload_mbps": round(upload_bps / 1_000_000, 2),
"ping_ms": round(float(r.get("ping") or 0), 1),
"server": f"{server.get('name', '')} ({server.get('country', '')})",
"isp": str(client.get("isp", "")),
}
except Exception as e:
log.error(e)
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, _run)
admin/static/dashboard.html
+198 -7
View File
@@ -4,6 +4,9 @@
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Seed — Admin</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Vazirmatn:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
@@ -11,7 +14,7 @@
--border: #2a2d3e; --text: #e2e8f0; --muted: #8892a4;
--accent: #6366f1; --green: #22c55e; --red: #ef4444; --yellow: #eab308;
}
body { background: var(--bg); color: var(--text); font-family: 'Inter', system-ui, sans-serif; font-size: 14px; }
body { background: var(--bg); color: var(--text); font-family: 'Vazirmatn', 'Inter', system-ui, sans-serif; font-size: 14px; }
/* ── Header ─────────────────────────────────── */
header {
@@ -120,7 +123,7 @@
width: 100%; background: var(--bg); border: 1px solid var(--border); border-radius: 7px;
padding: 9px 11px; color: var(--text); font-size: 13px; font-family: inherit;
}
.fg textarea { resize: vertical; min-height: 64px; font-family: monospace; }
.fg textarea { resize: vertical; min-height: 64px; font-family: 'Vazirmatn', 'Inter', system-ui, sans-serif; }
.fg input:focus, .fg select:focus, .fg textarea:focus { outline: none; border-color: var(--accent); }
.fg .hint { color: var(--muted); font-size: 11px; margin-top: 4px; }
.form-footer { display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; }
@@ -168,6 +171,16 @@
<input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
</div>
<!-- Scenario selector -->
<div class="fg">
<label>Scenario</label>
<select id="cfg-scenario" style="width:100%">
<option value="dynamic">Generic / DynamicFlow</option>
<option value="digipay">DigiPay (mydigipay.com)</option>
<option value="snappmarket">Snapp Market (snappshop.ir)</option>
</select>
</div>
<div class="fg">
<label>Target URL</label>
<input id="cfg-target-url" type="url" placeholder="https://example.com" />
@@ -179,11 +192,34 @@
</div>
<div class="fg">
<label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
<label id="cfg-item-ids-label">Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
<textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
</div>
<!-- Advanced -->
<!-- DigiPay-specific fields -->
<div id="digipay-fields" style="display:none">
<div class="fg">
<label>Item Match Mode</label>
<select id="cfg-digipay-match" style="width:100%">
<option value="onclick">URL match — onclick goToProduct URL fragment</option>
<option value="text">Text match — product title text</option>
</select>
<p class="hint" id="digipay-onclick-hint">Paste URL fragment from <code>goToProduct('https://…')</code> — e.g. <code>roshdbook.ir/product/شیمی-دهم/</code></p>
<p class="hint" id="digipay-text-hint" style="display:none">Paste the visible Persian product title — e.g. <code>شیمی دهم تک جلدی انتشارات مبتکران</code></p>
</div>
<div class="form-row">
<div class="fg">
<label>Max Scrolls</label>
<input id="cfg-dp-max-scrolls" type="number" min="1" max="500" value="20" />
</div>
<div class="fg">
<label>Scroll Pause (ms)</label>
<input id="cfg-dp-scroll-pause" type="number" min="200" max="10000" value="1500" />
</div>
</div>
</div>
<!-- Advanced (generic scenario only) -->
<details class="adv" id="adv-details">
<summary>Advanced Settings</summary>
<div class="adv-body">
@@ -321,6 +357,45 @@
<div class="card"><div class="label">Success Rate</div><div class="value" id="c-rate">–</div></div>
</div>
<!-- Speed Test -->
<div class="panel" style="margin-bottom:20px">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">
<div class="section-title" style="margin:0">Network Speed</div>
<button class="btn btn-primary" id="speedtest-btn" style="padding:5px 16px;font-size:12px" onclick="runSpeedTest()">▶ Run Test</button>
</div>
<div id="speedtest-status" style="color:var(--muted);font-size:12px;margin-bottom:12px;display:none">
<span id="speedtest-msg">Testing… this takes ~20 seconds</span>
</div>
<div id="speedtest-results" style="display:none">
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(130px,1fr));gap:12px">
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Download</div>
<div id="st-download" style="font-size:24px;font-weight:700;color:var(--green)">–</div>
<div style="color:var(--muted);font-size:11px">Mbps</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Upload</div>
<div id="st-upload" style="font-size:24px;font-weight:700;color:var(--accent)">–</div>
<div style="color:var(--muted);font-size:11px">Mbps</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Ping</div>
<div id="st-ping" style="font-size:24px;font-weight:700;color:var(--yellow)">–</div>
<div style="color:var(--muted);font-size:11px">ms</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px;grid-column:span 2">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Server / ISP</div>
<div id="st-server" style="font-size:13px;font-weight:600">–</div>
<div id="st-isp" style="color:var(--muted);font-size:12px;margin-top:2px">–</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Tested At</div>
<div id="st-time" style="font-size:13px;font-weight:600">–</div>
</div>
</div>
</div>
</div>
<div class="grid-2">
<div class="panel">
<div class="section-title">Scenario Breakdown</div>
@@ -637,6 +712,34 @@ document.querySelectorAll('.tab').forEach(tab => {
});
});
/* ─────────────────────────── Speed Test ─────────────────────── */
async function runSpeedTest() {
const btn = document.getElementById('speedtest-btn');
btn.disabled = true;
btn.textContent = '⏳ Testing…';
document.getElementById('speedtest-status').style.display = 'block';
document.getElementById('speedtest-results').style.display = 'none';
const res = await apiFetch('/api/speedtest', { method: 'POST' });
btn.disabled = false;
btn.textContent = '▶ Run Test';
document.getElementById('speedtest-status').style.display = 'none';
if (!res || !res.ok) {
document.getElementById('speedtest-msg').textContent = 'Speed test failed.';
document.getElementById('speedtest-status').style.display = 'block';
return;
}
const d = await res.json();
document.getElementById('st-download').textContent = d.download_mbps;
document.getElementById('st-upload').textContent = d.upload_mbps;
document.getElementById('st-ping').textContent = d.ping_ms;
document.getElementById('st-server').textContent = d.server || '–';
document.getElementById('st-isp').textContent = d.isp || '–';
document.getElementById('st-time').textContent = new Date().toLocaleTimeString();
document.getElementById('speedtest-results').style.display = 'block';
}
/* ─────────────────────────── Dashboard ──────────────────────── */
async function loadStats() {
const res = await apiFetch('/api/stats');
@@ -779,6 +882,7 @@ function exportConfig(id) {
item_url_template: c.item_url_template,
target_item_ids: c.target_item_ids_json || [],
target_url: c.target_url || '',
scenario: c.scenario || 'dynamic',
};
const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
const url = URL.createObjectURL(blob);
@@ -813,6 +917,7 @@ async function duplicateConfig(id) {
item_url_template: c.item_url_template || '',
target_item_ids: c.target_item_ids_json || [],
target_url: c.target_url || '',
scenario: c.scenario || 'dynamic',
};
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
if (res && res.ok) await loadConfigs();
@@ -850,12 +955,42 @@ document.getElementById('import-file-input').addEventListener('change', async (e
item_url_template: data.item_url_template || '',
target_item_ids: data.target_item_ids || [],
target_url: data.target_url || '',
scenario: data.scenario || 'dynamic',
};
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
if (res && res.ok) await loadConfigs();
});
/* ─── Form ─── */
function _applyScenario(scenario) {
const isDigi = scenario === 'digipay';
const isSnapp = scenario === 'snappmarket';
document.getElementById('digipay-fields').style.display = isDigi ? '' : 'none';
document.getElementById('adv-details').style.display = isDigi ? 'none' : '';
if (isDigi) {
if (!document.getElementById('cfg-target-url').value)
document.getElementById('cfg-target-url').value = 'https://www.mydigipay.com/stores/all-stores/';
document.getElementById('cfg-target-url').placeholder = 'https://www.mydigipay.com/stores/all-stores/';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line — URL fragments or product titles)</span>';
} else if (isSnapp) {
if (!document.getElementById('cfg-target-url').value)
document.getElementById('cfg-target-url').value = 'https://snappshop.ir/';
document.getElementById('cfg-target-url').placeholder = 'https://snappshop.ir/';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line — href URL fragments)</span>';
// Pre-set item selector to a-link for href-based matching
document.getElementById('cfg-item-sel-type').value = 'a-link';
document.getElementById('alink-hint').style.display = '';
} else {
document.getElementById('cfg-target-url').placeholder = 'https://example.com';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line)</span>';
}
}
function _resetAdvanced() {
document.getElementById('cfg-search-sel').value = '';
document.getElementById('cfg-search-sel-type').value = 'css';
@@ -876,29 +1011,40 @@ function _resetAdvanced() {
document.getElementById('cfg-item-sel-type').value = 'css';
document.getElementById('alink-hint').style.display = 'none';
document.getElementById('cfg-item-url-tpl').value = '';
// DigiPay defaults
document.getElementById('cfg-digipay-match').value = 'onclick';
document.getElementById('cfg-dp-max-scrolls').value = '20';
document.getElementById('cfg-dp-scroll-pause').value = '1500';
document.getElementById('digipay-onclick-hint').style.display = '';
document.getElementById('digipay-text-hint').style.display = 'none';
}
function openNewForm() {
document.getElementById('form-title').textContent = 'New Flow Config';
document.getElementById('cfg-id').value = '';
document.getElementById('cfg-name').value = '';
document.getElementById('cfg-scenario').value = 'dynamic';
document.getElementById('cfg-target-url').value = '';
document.getElementById('cfg-search-texts').value = '';
document.getElementById('cfg-item-ids').value = '';
_resetAdvanced();
document.getElementById('adv-details').removeAttribute('open');
_applyScenario('dynamic');
document.getElementById('config-overlay').classList.remove('hidden');
}
function openEditForm(id) {
const c = _configs.find(x => x.id === id);
if (!c) return;
const scenario = c.scenario || 'dynamic';
document.getElementById('form-title').textContent = 'Edit Flow Config';
document.getElementById('cfg-id').value = id;
document.getElementById('cfg-name').value = c.name || '';
document.getElementById('cfg-scenario').value = scenario;
document.getElementById('cfg-target-url').value = c.target_url || '';
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
// Generic fields
document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
@@ -919,7 +1065,15 @@ function openEditForm(id) {
document.getElementById('cfg-item-sel-type').value = _ist;
document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
// DigiPay fields
const dpMatch = (scenario === 'digipay') ? (_ist || 'onclick') : 'onclick';
document.getElementById('cfg-digipay-match').value = dpMatch;
document.getElementById('cfg-dp-max-scrolls').value = c.max_scrolls ?? 20;
document.getElementById('cfg-dp-scroll-pause').value = c.scroll_pause_ms ?? 1500;
document.getElementById('digipay-onclick-hint').style.display = dpMatch === 'onclick' ? '' : 'none';
document.getElementById('digipay-text-hint').style.display = dpMatch === 'text' ? '' : 'none';
document.getElementById('adv-details').setAttribute('open', '');
_applyScenario(scenario);
document.getElementById('config-overlay').classList.remove('hidden');
}
@@ -928,9 +1082,41 @@ function closeForm() {
}
function collectForm() {
return {
const scenario = document.getElementById('cfg-scenario').value;
const base = {
name: document.getElementById('cfg-name').value.trim(),
scenario,
target_url: document.getElementById('cfg-target-url').value.trim(),
search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean),
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
};
if (scenario === 'digipay') {
const matchMode = document.getElementById('cfg-digipay-match').value;
return {
...base,
// Fixed DigiPay selectors (hardcoded in DigiPayFlow, stored for reference)
search_box_selector: '/html/body/header/header/div[2]',
search_box_selector_type: 'xpath',
search_results_selector: "a[onclick*='goToProduct']",
search_results_selector_type: 'css',
search_submit_selector: '',
search_submit_selector_type: 'css',
scroll_container_selector: '',
scroll_container_selector_type: 'css',
item_selector_type: matchMode,
item_selector_template: '',
item_url_template: '',
max_scrolls: parseInt(document.getElementById('cfg-dp-max-scrolls').value) || 20,
scroll_pause_ms: parseInt(document.getElementById('cfg-dp-scroll-pause').value) || 1500,
no_new_content_timeout_ms: 3000,
pagination_selector: '', pagination_selector_type: 'css',
max_pages: 10, pagination_wait_ms: 1500,
};
}
return {
...base,
search_box_selector: document.getElementById('cfg-search-sel').value.trim(),
search_box_selector_type: document.getElementById('cfg-search-sel-type').value,
search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
@@ -949,8 +1135,6 @@ function collectForm() {
item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
item_selector_type: document.getElementById('cfg-item-sel-type').value,
item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
target_url: document.getElementById('cfg-target-url').value.trim(),
};
}
@@ -960,6 +1144,13 @@ document.getElementById('config-overlay').addEventListener('click', e => { if (e
document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
});
document.getElementById('cfg-scenario').addEventListener('change', e => {
_applyScenario(e.target.value);
});
document.getElementById('cfg-digipay-match').addEventListener('change', e => {
document.getElementById('digipay-onclick-hint').style.display = e.target.value === 'onclick' ? '' : 'none';
document.getElementById('digipay-text-hint').style.display = e.target.value === 'text' ? '' : 'none';
});
document.getElementById('form-save').addEventListener('click', async () => {
const body = collectForm();
crawler/scenarios/digipay.py
+133
View File
@@ -0,0 +1,133 @@
"""DigiPay-specific flow — search + find via onclick goToProduct URL."""
from __future__ import annotations
import time
from typing import Any
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
from crawler.dynamic_flow import DynamicFlow, StepResult, _wait_for
from logger import get_logger
log = get_logger(__name__)
_RESULTS_WAIT_CSS = "a[onclick*='goToProduct']"
# Ordered list of selectors tried to locate the search input
_SEARCH_INPUT_CANDIDATES = [
(By.XPATH, "/html/body/header/header/div[2]//input"),
(By.XPATH, "//header//input[@type='search']"),
(By.XPATH, "//header//input[contains(@class,'search')]"),
(By.XPATH, "//input[@type='search']"),
(By.CSS_SELECTOR, "input[type='search']"),
(By.XPATH, "//input[contains(@placeholder,'جست')]"), # 'جست' = search (Farsi prefix)
]
# Selectors tried to trigger / open the search box before locating the input
_SEARCH_TRIGGER_CANDIDATES = [
(By.XPATH, "/html/body/header/header/div[2]"),
(By.XPATH, "//header//*[contains(@class,'search')]"),
(By.CSS_SELECTOR, "header [class*='search']"),
]
def _wait_spa_ready(driver: uc.Chrome, timeout: float = 20.0) -> None:
"""Wait for DOM ready + Vue/React hydration (no pending network requests)."""
WebDriverWait(driver, timeout).until(
lambda d: d.execute_script("return document.readyState") == "complete"
)
# Extra pause for SPA JS to hydrate
time.sleep(1.5)
def _find_search_input(driver: uc.Chrome, wait: WebDriverWait) -> Any:
"""Try each candidate selector until one is clickable."""
for by, sel in _SEARCH_INPUT_CANDIDATES:
try:
el = wait.until(EC.element_to_be_clickable((by, sel)))
log.debug("Search input found via [%s:%s]", by, sel)
return el
except Exception:
continue
raise RuntimeError("Search input not found — tried all candidate selectors")
class DigiPayFlow(DynamicFlow):
"""
DigiPay-specific flow:
- Waits for SPA hydration before interacting.
- Tries multiple selectors to find/open search input.
- Waits for `a[onclick*='goToProduct']` cards after submit.
- Item matching via onclick URL fragment or product title text.
"""
def _run_search(self, text: str) -> StepResult:
t0 = time.monotonic()
name = f"search:{text}"
log.info("[%s] DigiPay search for '%s'", name, text)
try:
wait_long = WebDriverWait(self.driver, 25)
wait_short = WebDriverWait(self.driver, 5)
# Wait for SPA to fully hydrate
_wait_spa_ready(self.driver)
log.debug("[%s] SPA ready", name)
# Try to click a trigger element to open/reveal the search input
for by, sel in _SEARCH_TRIGGER_CANDIDATES:
try:
trigger = wait_short.until(EC.element_to_be_clickable((by, sel)))
trigger.click()
log.debug("[%s] Clicked trigger [%s:%s]", name, by, sel)
human_delay(0.4, 0.7)
break
except Exception:
continue
# Locate the actual input (may now be in an overlay)
box = _find_search_input(self.driver, wait_long)
box.clear()
human_delay(0.2, 0.4)
for ch in text:
box.send_keys(ch)
time.sleep(0.05)
human_delay(0.3, 0.6)
box.send_keys(Keys.RETURN)
log.debug("[%s] Enter sent", name)
# Wait for product cards
wait_long.until(
EC.presence_of_element_located((By.CSS_SELECTOR, _RESULTS_WAIT_CSS))
)
human_delay(1.0, 2.0)
ms = int((time.monotonic() - t0) * 1000)
log.info("[%s] Search OK — %d ms", name, ms)
return StepResult(name=name, success=True, data={"text": text}, duration_ms=ms)
except Exception as exc:
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Failed — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
def _build_item_selector(self, item_id: str) -> tuple[str, str]:
sel_type: str = self.cfg.get("item_selector_type", "onclick")
template: str = self.cfg.get("item_selector_template", "")
fragment = template.replace("{item_id}", item_id) if template else item_id
if sel_type == "onclick":
escaped = fragment.replace("'", "\\'")
return f"//a[contains(@onclick, '{escaped}')]", "xpath"
if sel_type == "text":
escaped = fragment.replace("'", "\\'")
return f"//a[.//*[contains(normalize-space(.), '{escaped}')]]", "xpath"
return super()._build_item_selector(item_id)
crawler/tasks.py
+9 -1
View File
@@ -308,9 +308,17 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
try:
target_url: str = cfg.get("target_url") or ""
scenario: str = cfg.get("scenario") or "dynamic"
with driver_session(headless=headless) as driver:
FreshSessionScenario(driver, target_url).run()
result = DynamicFlow(driver, cfg).run() if cfg else None
if cfg:
if scenario == "digipay":
from crawler.scenarios.digipay import DigiPayFlow
result = DigiPayFlow(driver, cfg).run()
else:
result = DynamicFlow(driver, cfg).run()
else:
result = None
total_ms = int((time.monotonic() - t0) * 1000)
success = result.success if result else True
db/connection.py
+1
View File
@@ -97,6 +97,7 @@ def _migrate(conn: sqlite3.Connection) -> None:
"max_pages": "INTEGER NOT NULL DEFAULT 10",
"pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
"target_url": "TEXT NOT NULL DEFAULT ''",
"scenario": "TEXT NOT NULL DEFAULT 'dynamic'",
}
for col, definition in additions.items():
if col not in existing:
db/repositories/flow_configs.py
+1 -1
View File
@@ -17,7 +17,7 @@ _FIELDS = [
"max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
"pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms",
"item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json",
"target_url",
"target_url", "scenario",
]
pyproject.toml
+1
View File
@@ -14,6 +14,7 @@ dependencies = [
"pydantic>=2.7.0",
"python-dotenv>=1.0.1",
"setuptools>=70.0.0",
"speedtest-cli>=2.1.3",
"rq>=1.16.0",
"redis>=5.0.0",
]
uv.lock
Generated
+11
View File
@@ -633,6 +633,7 @@ dependencies = [
{ name = "selenium" },
{ name = "selenium-stealth" },
{ name = "setuptools" },
{ name = "speedtest-cli" },
{ name = "undetected-chromedriver" },
{ name = "uvicorn", extra = ["standard"] },
]
@@ -655,6 +656,7 @@ requires-dist = [
{ name = "selenium", specifier = ">=4.18.0" },
{ name = "selenium-stealth", specifier = ">=1.0.6" },
{ name = "setuptools", specifier = ">=70.0.0" },
{ name = "speedtest-cli", specifier = ">=2.1.3" },
{ name = "undetected-chromedriver", specifier = ">=3.5.5" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
]
@@ -726,6 +728,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
]
[[package]]
name = "speedtest-cli"
version = "2.1.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/85/d2/32c8a30768b788d319f94cde3a77e0ccc1812dca464ad8062d3c4d703e06/speedtest-cli-2.1.3.tar.gz", hash = "sha256:5e2773233cedb5fa3d8120eb7f97bcc4974b5221b254d33ff16e2f1d413d90f0", size = 24721, upload-time = "2021-04-08T13:51:33.627Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9f/39/65259b7054368b370d3183762484fa2c779ddc41633894d895f9d1720f45/speedtest_cli-2.1.3-py2.py3-none-any.whl", hash = "sha256:75ff32c91af9ac1ce2b905476d6e92bd9eb2c0783f9e7d1939d74605c7d0b9ea", size = 23973, upload-time = "2021-04-08T13:51:32.028Z" },
]
[[package]]
name = "starlette"
version = "1.3.1"