nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Digipay and speedtest added

Browse Source
This commit is contained in:
Navid Filsaraee
2026-06-27 16:18:33 +03:30
parent 64886b2b10
commit 744550deaa
8 changed files with 421 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/routes.py
+66 -6
View File
@@ -1,9 +1,11 @@
"""FastAPI routes — dashboard API + auth.""" """FastAPI routes — dashboard API + auth."""
from __future__ import annotations from __future__ import annotations
import secrets import secrets
from typing import Annotated, Any from typing import Annotated, Any
import speedtest # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, HTTPException, status from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel from pydantic import BaseModel
@@ -24,9 +26,15 @@ router = APIRouter()
security = HTTPBasic() security = HTTPBasic()
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str: def require_auth(
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode()) credentials: Annotated[HTTPBasicCredentials, Depends(security)],
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode()) ) -> str:
ok_user = secrets.compare_digest(
credentials.username.encode(), config.admin_username.encode()
)
ok_pass = secrets.compare_digest(
credentials.password.encode(), config.admin_password.encode()
)
if not (ok_user and ok_pass): if not (ok_user and ok_pass):
raise HTTPException( raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED, status_code=status.HTTP_401_UNAUTHORIZED,
@@ -43,6 +51,7 @@ Auth = Annotated[str, Depends(require_auth)]
# Stats # Stats
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@router.get("/api/stats") @router.get("/api/stats")
async def stats(_: Auth) -> dict[str, object]: async def stats(_: Auth) -> dict[str, object]:
return await get_stats() return await get_stats()
@@ -52,6 +61,7 @@ async def stats(_: Auth) -> dict[str, object]:
# Flow configs # Flow configs
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
class FlowConfigIn(BaseModel): class FlowConfigIn(BaseModel):
name: str name: str
search_texts: list[str] = [] search_texts: list[str] = []
@@ -75,6 +85,7 @@ class FlowConfigIn(BaseModel):
item_url_template: str = "" item_url_template: str = ""
target_item_ids: list[str] = [] target_item_ids: list[str] = []
target_url: str = "" target_url: str = ""
scenario: str = "dynamic"
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -101,6 +112,7 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
"item_url_template": body.item_url_template, "item_url_template": body.item_url_template,
"target_item_ids_json": body.target_item_ids, "target_item_ids_json": body.target_item_ids,
"target_url": body.target_url, "target_url": body.target_url,
"scenario": body.scenario,
} }
@@ -146,6 +158,7 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
# Batch runs (managed via Huey task queue) # Batch runs (managed via Huey task queue)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
class BatchTaskIn(BaseModel): class BatchTaskIn(BaseModel):
config_id: int config_id: int
workers: int = 3 workers: int = 3
@@ -160,17 +173,25 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
cfg = await get_flow_config(body.config_id) cfg = await get_flow_config(body.config_id)
if not cfg: if not cfg:
raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found") raise HTTPException(
status_code=404, detail=f"Flow config {body.config_id} not found"
)
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless) result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
log.info("Task enqueued — config=%d runs=%d workers=%d status=%s", log.info(
body.config_id, body.total_runs, body.workers, result["status"]) "Task enqueued — config=%d runs=%d workers=%d status=%s",
body.config_id,
body.total_runs,
body.workers,
result["status"],
)
return result return result
@router.delete("/api/batch-queue/{queue_id}", status_code=200) @router.delete("/api/batch-queue/{queue_id}", status_code=200)
async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]: async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
from crawler.tasks import dequeue from crawler.tasks import dequeue
removed = dequeue(queue_id) removed = dequeue(queue_id)
if not removed: if not removed:
raise HTTPException(status_code=404, detail="Queue item not found") raise HTTPException(status_code=404, detail="Queue item not found")
@@ -180,6 +201,7 @@ async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
@router.post("/api/batch-queue/clear", status_code=200) @router.post("/api/batch-queue/clear", status_code=200)
async def clear_batch_queue(_: Auth) -> dict[str, object]: async def clear_batch_queue(_: Auth) -> dict[str, object]:
from crawler.tasks import clear_queue from crawler.tasks import clear_queue
count = clear_queue() count = clear_queue()
return {"cleared": count} return {"cleared": count}
@@ -187,6 +209,7 @@ async def clear_batch_queue(_: Auth) -> dict[str, object]:
@router.post("/api/stop-batch", status_code=200) @router.post("/api/stop-batch", status_code=200)
async def stop_batch(_: Auth) -> dict[str, object]: async def stop_batch(_: Auth) -> dict[str, object]:
from crawler.tasks import stop_current from crawler.tasks import stop_current
stopped = stop_current() stopped = stop_current()
if not stopped: if not stopped:
raise HTTPException(status_code=404, detail="No active batch to stop") raise HTTPException(status_code=404, detail="No active batch to stop")
@@ -196,12 +219,14 @@ async def stop_batch(_: Auth) -> dict[str, object]:
@router.get("/api/batch-status") @router.get("/api/batch-status")
async def batch_status(_: Auth) -> dict[str, object]: async def batch_status(_: Auth) -> dict[str, object]:
from crawler.tasks import batch_status as _status from crawler.tasks import batch_status as _status
return _status() # type: ignore[return-value] return _status() # type: ignore[return-value]
@router.get("/api/batch-runs") @router.get("/api/batch-runs")
async def list_batch_runs(_: Auth) -> list[dict[str, object]]: async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
from db.repositories.batch_runs import batch_runs_repo from db.repositories.batch_runs import batch_runs_repo
return await batch_runs_repo.list() # type: ignore[return-value] return await batch_runs_repo.list() # type: ignore[return-value]
@@ -209,6 +234,7 @@ async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
# Log viewer # Log viewer
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
@router.get("/api/logs") @router.get("/api/logs")
def get_logs( def get_logs(
_: Auth, _: Auth,
@@ -217,6 +243,7 @@ def get_logs(
q: str = "", q: str = "",
) -> dict[str, object]: ) -> dict[str, object]:
import os import os
from logger import _LOG_FILE from logger import _LOG_FILE
path = _LOG_FILE path = _LOG_FILE
@@ -245,3 +272,36 @@ def get_logs(
break break
return {"lines": results, "file": path} return {"lines": results, "file": path}
# ---------------------------------------------------------------------------
# Speed test
# ---------------------------------------------------------------------------
@router.post("/api/speedtest")
async def run_speedtest(_: Auth) -> dict[str, object]:
import asyncio
def _run() -> dict[str, object]:
try:
log.info("running speedtest")
st = speedtest.Speedtest(secure=True)
st.get_best_server()
download_bps: float = st.download()
upload_bps: float = st.upload()
r = st.results.dict()
server: dict[str, object] = r.get("server") or {}
client: dict[str, object] = r.get("client") or {}
return {
"download_mbps": round(download_bps / 1_000_000, 2),
"upload_mbps": round(upload_bps / 1_000_000, 2),
"ping_ms": round(float(r.get("ping") or 0), 1),
"server": f"{server.get('name', '')} ({server.get('country', '')})",
"isp": str(client.get("isp", "")),
}
except Exception as e:
log.error(e)
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, _run)
admin/static/dashboard.html
+199 -8
View File
@@ -4,6 +4,9 @@
<meta charset="UTF-8" /> <meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" /> <meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Seed — Admin</title> <title>Seed — Admin</title>
<link rel="preconnect" href="https://fonts.googleapis.com" />
<link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
<link href="https://fonts.googleapis.com/css2?family=Vazirmatn:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
<style> <style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; } *, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root { :root {
@@ -11,7 +14,7 @@
--border: #2a2d3e; --text: #e2e8f0; --muted: #8892a4; --border: #2a2d3e; --text: #e2e8f0; --muted: #8892a4;
--accent: #6366f1; --green: #22c55e; --red: #ef4444; --yellow: #eab308; --accent: #6366f1; --green: #22c55e; --red: #ef4444; --yellow: #eab308;
} }
body { background: var(--bg); color: var(--text); font-family: 'Inter', system-ui, sans-serif; font-size: 14px; } body { background: var(--bg); color: var(--text); font-family: 'Vazirmatn', 'Inter', system-ui, sans-serif; font-size: 14px; }
/* ── Header ─────────────────────────────────── */ /* ── Header ─────────────────────────────────── */
header { header {
@@ -120,7 +123,7 @@
width: 100%; background: var(--bg); border: 1px solid var(--border); border-radius: 7px; width: 100%; background: var(--bg); border: 1px solid var(--border); border-radius: 7px;
padding: 9px 11px; color: var(--text); font-size: 13px; font-family: inherit; padding: 9px 11px; color: var(--text); font-size: 13px; font-family: inherit;
} }
.fg textarea { resize: vertical; min-height: 64px; font-family: monospace; } .fg textarea { resize: vertical; min-height: 64px; font-family: 'Vazirmatn', 'Inter', system-ui, sans-serif; }
.fg input:focus, .fg select:focus, .fg textarea:focus { outline: none; border-color: var(--accent); } .fg input:focus, .fg select:focus, .fg textarea:focus { outline: none; border-color: var(--accent); }
.fg .hint { color: var(--muted); font-size: 11px; margin-top: 4px; } .fg .hint { color: var(--muted); font-size: 11px; margin-top: 4px; }
.form-footer { display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; } .form-footer { display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; }
@@ -168,6 +171,16 @@
<input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" /> <input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
</div> </div>
<!-- Scenario selector -->
<div class="fg">
<label>Scenario</label>
<select id="cfg-scenario" style="width:100%">
<option value="dynamic">Generic / DynamicFlow</option>
<option value="digipay">DigiPay (mydigipay.com)</option>
<option value="snappmarket">Snapp Market (snappshop.ir)</option>
</select>
</div>
<div class="fg"> <div class="fg">
<label>Target URL</label> <label>Target URL</label>
<input id="cfg-target-url" type="url" placeholder="https://example.com" /> <input id="cfg-target-url" type="url" placeholder="https://example.com" />
@@ -179,11 +192,34 @@
</div> </div>
<div class="fg"> <div class="fg">
<label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label> <label id="cfg-item-ids-label">Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
<textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea> <textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
</div> </div>
<!-- Advanced --> <!-- DigiPay-specific fields -->
<div id="digipay-fields" style="display:none">
<div class="fg">
<label>Item Match Mode</label>
<select id="cfg-digipay-match" style="width:100%">
<option value="onclick">URL match — onclick goToProduct URL fragment</option>
<option value="text">Text match — product title text</option>
</select>
<p class="hint" id="digipay-onclick-hint">Paste URL fragment from <code>goToProduct('https://…')</code> — e.g. <code>roshdbook.ir/product/شیمی-دهم/</code></p>
<p class="hint" id="digipay-text-hint" style="display:none">Paste the visible Persian product title — e.g. <code>شیمی دهم تک جلدی انتشارات مبتکران</code></p>
</div>
<div class="form-row">
<div class="fg">
<label>Max Scrolls</label>
<input id="cfg-dp-max-scrolls" type="number" min="1" max="500" value="20" />
</div>
<div class="fg">
<label>Scroll Pause (ms)</label>
<input id="cfg-dp-scroll-pause" type="number" min="200" max="10000" value="1500" />
</div>
</div>
</div>
<!-- Advanced (generic scenario only) -->
<details class="adv" id="adv-details"> <details class="adv" id="adv-details">
<summary>Advanced Settings</summary> <summary>Advanced Settings</summary>
<div class="adv-body"> <div class="adv-body">
@@ -321,6 +357,45 @@
<div class="card"><div class="label">Success Rate</div><div class="value" id="c-rate">–</div></div> <div class="card"><div class="label">Success Rate</div><div class="value" id="c-rate">–</div></div>
</div> </div>
<!-- Speed Test -->
<div class="panel" style="margin-bottom:20px">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:16px">
<div class="section-title" style="margin:0">Network Speed</div>
<button class="btn btn-primary" id="speedtest-btn" style="padding:5px 16px;font-size:12px" onclick="runSpeedTest()">▶ Run Test</button>
</div>
<div id="speedtest-status" style="color:var(--muted);font-size:12px;margin-bottom:12px;display:none">
<span id="speedtest-msg">Testing… this takes ~20 seconds</span>
</div>
<div id="speedtest-results" style="display:none">
<div style="display:grid;grid-template-columns:repeat(auto-fit,minmax(130px,1fr));gap:12px">
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Download</div>
<div id="st-download" style="font-size:24px;font-weight:700;color:var(--green)">–</div>
<div style="color:var(--muted);font-size:11px">Mbps</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Upload</div>
<div id="st-upload" style="font-size:24px;font-weight:700;color:var(--accent)">–</div>
<div style="color:var(--muted);font-size:11px">Mbps</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Ping</div>
<div id="st-ping" style="font-size:24px;font-weight:700;color:var(--yellow)">–</div>
<div style="color:var(--muted);font-size:11px">ms</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px;grid-column:span 2">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Server / ISP</div>
<div id="st-server" style="font-size:13px;font-weight:600">–</div>
<div id="st-isp" style="color:var(--muted);font-size:12px;margin-top:2px">–</div>
</div>
<div style="background:var(--surface2);border-radius:8px;padding:14px 16px">
<div style="color:var(--muted);font-size:11px;text-transform:uppercase;letter-spacing:.5px;margin-bottom:6px">Tested At</div>
<div id="st-time" style="font-size:13px;font-weight:600">–</div>
</div>
</div>
</div>
</div>
<div class="grid-2"> <div class="grid-2">
<div class="panel"> <div class="panel">
<div class="section-title">Scenario Breakdown</div> <div class="section-title">Scenario Breakdown</div>
@@ -637,6 +712,34 @@ document.querySelectorAll('.tab').forEach(tab => {
}); });
}); });
/* ─────────────────────────── Speed Test ─────────────────────── */
async function runSpeedTest() {
const btn = document.getElementById('speedtest-btn');
btn.disabled = true;
btn.textContent = '⏳ Testing…';
document.getElementById('speedtest-status').style.display = 'block';
document.getElementById('speedtest-results').style.display = 'none';
const res = await apiFetch('/api/speedtest', { method: 'POST' });
btn.disabled = false;
btn.textContent = '▶ Run Test';
document.getElementById('speedtest-status').style.display = 'none';
if (!res || !res.ok) {
document.getElementById('speedtest-msg').textContent = 'Speed test failed.';
document.getElementById('speedtest-status').style.display = 'block';
return;
}
const d = await res.json();
document.getElementById('st-download').textContent = d.download_mbps;
document.getElementById('st-upload').textContent = d.upload_mbps;
document.getElementById('st-ping').textContent = d.ping_ms;
document.getElementById('st-server').textContent = d.server || '–';
document.getElementById('st-isp').textContent = d.isp || '–';
document.getElementById('st-time').textContent = new Date().toLocaleTimeString();
document.getElementById('speedtest-results').style.display = 'block';
}
/* ─────────────────────────── Dashboard ──────────────────────── */ /* ─────────────────────────── Dashboard ──────────────────────── */
async function loadStats() { async function loadStats() {
const res = await apiFetch('/api/stats'); const res = await apiFetch('/api/stats');
@@ -779,6 +882,7 @@ function exportConfig(id) {
item_url_template: c.item_url_template, item_url_template: c.item_url_template,
target_item_ids: c.target_item_ids_json || [], target_item_ids: c.target_item_ids_json || [],
target_url: c.target_url || '', target_url: c.target_url || '',
scenario: c.scenario || 'dynamic',
}; };
const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' }); const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
const url = URL.createObjectURL(blob); const url = URL.createObjectURL(blob);
@@ -813,6 +917,7 @@ async function duplicateConfig(id) {
item_url_template: c.item_url_template || '', item_url_template: c.item_url_template || '',
target_item_ids: c.target_item_ids_json || [], target_item_ids: c.target_item_ids_json || [],
target_url: c.target_url || '', target_url: c.target_url || '',
scenario: c.scenario || 'dynamic',
}; };
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
if (res && res.ok) await loadConfigs(); if (res && res.ok) await loadConfigs();
@@ -850,12 +955,42 @@ document.getElementById('import-file-input').addEventListener('change', async (e
item_url_template: data.item_url_template || '', item_url_template: data.item_url_template || '',
target_item_ids: data.target_item_ids || [], target_item_ids: data.target_item_ids || [],
target_url: data.target_url || '', target_url: data.target_url || '',
scenario: data.scenario || 'dynamic',
}; };
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) }); const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
if (res && res.ok) await loadConfigs(); if (res && res.ok) await loadConfigs();
}); });
/* ─── Form ─── */ /* ─── Form ─── */
function _applyScenario(scenario) {
const isDigi = scenario === 'digipay';
const isSnapp = scenario === 'snappmarket';
document.getElementById('digipay-fields').style.display = isDigi ? '' : 'none';
document.getElementById('adv-details').style.display = isDigi ? 'none' : '';
if (isDigi) {
if (!document.getElementById('cfg-target-url').value)
document.getElementById('cfg-target-url').value = 'https://www.mydigipay.com/stores/all-stores/';
document.getElementById('cfg-target-url').placeholder = 'https://www.mydigipay.com/stores/all-stores/';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line — URL fragments or product titles)</span>';
} else if (isSnapp) {
if (!document.getElementById('cfg-target-url').value)
document.getElementById('cfg-target-url').value = 'https://snappshop.ir/';
document.getElementById('cfg-target-url').placeholder = 'https://snappshop.ir/';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line — href URL fragments)</span>';
// Pre-set item selector to a-link for href-based matching
document.getElementById('cfg-item-sel-type').value = 'a-link';
document.getElementById('alink-hint').style.display = '';
} else {
document.getElementById('cfg-target-url').placeholder = 'https://example.com';
document.getElementById('cfg-item-ids-label').innerHTML =
'Target Item IDs <span style="color:var(--muted)">(one per line)</span>';
}
}
function _resetAdvanced() { function _resetAdvanced() {
document.getElementById('cfg-search-sel').value = ''; document.getElementById('cfg-search-sel').value = '';
document.getElementById('cfg-search-sel-type').value = 'css'; document.getElementById('cfg-search-sel-type').value = 'css';
@@ -876,29 +1011,40 @@ function _resetAdvanced() {
document.getElementById('cfg-item-sel-type').value = 'css'; document.getElementById('cfg-item-sel-type').value = 'css';
document.getElementById('alink-hint').style.display = 'none'; document.getElementById('alink-hint').style.display = 'none';
document.getElementById('cfg-item-url-tpl').value = ''; document.getElementById('cfg-item-url-tpl').value = '';
// DigiPay defaults
document.getElementById('cfg-digipay-match').value = 'onclick';
document.getElementById('cfg-dp-max-scrolls').value = '20';
document.getElementById('cfg-dp-scroll-pause').value = '1500';
document.getElementById('digipay-onclick-hint').style.display = '';
document.getElementById('digipay-text-hint').style.display = 'none';
} }
function openNewForm() { function openNewForm() {
document.getElementById('form-title').textContent = 'New Flow Config'; document.getElementById('form-title').textContent = 'New Flow Config';
document.getElementById('cfg-id').value = ''; document.getElementById('cfg-id').value = '';
document.getElementById('cfg-name').value = ''; document.getElementById('cfg-name').value = '';
document.getElementById('cfg-scenario').value = 'dynamic';
document.getElementById('cfg-target-url').value = ''; document.getElementById('cfg-target-url').value = '';
document.getElementById('cfg-search-texts').value = ''; document.getElementById('cfg-search-texts').value = '';
document.getElementById('cfg-item-ids').value = ''; document.getElementById('cfg-item-ids').value = '';
_resetAdvanced(); _resetAdvanced();
document.getElementById('adv-details').removeAttribute('open'); document.getElementById('adv-details').removeAttribute('open');
_applyScenario('dynamic');
document.getElementById('config-overlay').classList.remove('hidden'); document.getElementById('config-overlay').classList.remove('hidden');
} }
function openEditForm(id) { function openEditForm(id) {
const c = _configs.find(x => x.id === id); const c = _configs.find(x => x.id === id);
if (!c) return; if (!c) return;
const scenario = c.scenario || 'dynamic';
document.getElementById('form-title').textContent = 'Edit Flow Config'; document.getElementById('form-title').textContent = 'Edit Flow Config';
document.getElementById('cfg-id').value = id; document.getElementById('cfg-id').value = id;
document.getElementById('cfg-name').value = c.name || ''; document.getElementById('cfg-name').value = c.name || '';
document.getElementById('cfg-scenario').value = scenario;
document.getElementById('cfg-target-url').value = c.target_url || ''; document.getElementById('cfg-target-url').value = c.target_url || '';
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n'); document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n'); document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
// Generic fields
document.getElementById('cfg-search-sel').value = c.search_box_selector || ''; document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css'; document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || ''; document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
@@ -919,7 +1065,15 @@ function openEditForm(id) {
document.getElementById('cfg-item-sel-type').value = _ist; document.getElementById('cfg-item-sel-type').value = _ist;
document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none'; document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
document.getElementById('cfg-item-url-tpl').value = c.item_url_template || ''; document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
// DigiPay fields
const dpMatch = (scenario === 'digipay') ? (_ist || 'onclick') : 'onclick';
document.getElementById('cfg-digipay-match').value = dpMatch;
document.getElementById('cfg-dp-max-scrolls').value = c.max_scrolls ?? 20;
document.getElementById('cfg-dp-scroll-pause').value = c.scroll_pause_ms ?? 1500;
document.getElementById('digipay-onclick-hint').style.display = dpMatch === 'onclick' ? '' : 'none';
document.getElementById('digipay-text-hint').style.display = dpMatch === 'text' ? '' : 'none';
document.getElementById('adv-details').setAttribute('open', ''); document.getElementById('adv-details').setAttribute('open', '');
_applyScenario(scenario);
document.getElementById('config-overlay').classList.remove('hidden'); document.getElementById('config-overlay').classList.remove('hidden');
} }
@@ -928,9 +1082,41 @@ function closeForm() {
} }
function collectForm() { function collectForm() {
return { const scenario = document.getElementById('cfg-scenario').value;
name: document.getElementById('cfg-name').value.trim(), const base = {
name: document.getElementById('cfg-name').value.trim(),
scenario,
target_url: document.getElementById('cfg-target-url').value.trim(),
search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean), search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean),
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
};
if (scenario === 'digipay') {
const matchMode = document.getElementById('cfg-digipay-match').value;
return {
...base,
// Fixed DigiPay selectors (hardcoded in DigiPayFlow, stored for reference)
search_box_selector: '/html/body/header/header/div[2]',
search_box_selector_type: 'xpath',
search_results_selector: "a[onclick*='goToProduct']",
search_results_selector_type: 'css',
search_submit_selector: '',
search_submit_selector_type: 'css',
scroll_container_selector: '',
scroll_container_selector_type: 'css',
item_selector_type: matchMode,
item_selector_template: '',
item_url_template: '',
max_scrolls: parseInt(document.getElementById('cfg-dp-max-scrolls').value) || 20,
scroll_pause_ms: parseInt(document.getElementById('cfg-dp-scroll-pause').value) || 1500,
no_new_content_timeout_ms: 3000,
pagination_selector: '', pagination_selector_type: 'css',
max_pages: 10, pagination_wait_ms: 1500,
};
}
return {
...base,
search_box_selector: document.getElementById('cfg-search-sel').value.trim(), search_box_selector: document.getElementById('cfg-search-sel').value.trim(),
search_box_selector_type: document.getElementById('cfg-search-sel-type').value, search_box_selector_type: document.getElementById('cfg-search-sel-type').value,
search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(), search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
@@ -949,8 +1135,6 @@ function collectForm() {
item_selector_template: document.getElementById('cfg-item-sel').value.trim(), item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
item_selector_type: document.getElementById('cfg-item-sel-type').value, item_selector_type: document.getElementById('cfg-item-sel-type').value,
item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(), item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
target_url: document.getElementById('cfg-target-url').value.trim(),
}; };
} }
@@ -960,6 +1144,13 @@ document.getElementById('config-overlay').addEventListener('click', e => { if (e
document.getElementById('cfg-item-sel-type').addEventListener('change', e => { document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none'; document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
}); });
document.getElementById('cfg-scenario').addEventListener('change', e => {
_applyScenario(e.target.value);
});
document.getElementById('cfg-digipay-match').addEventListener('change', e => {
document.getElementById('digipay-onclick-hint').style.display = e.target.value === 'onclick' ? '' : 'none';
document.getElementById('digipay-text-hint').style.display = e.target.value === 'text' ? '' : 'none';
});
document.getElementById('form-save').addEventListener('click', async () => { document.getElementById('form-save').addEventListener('click', async () => {
const body = collectForm(); const body = collectForm();
crawler/scenarios/digipay.py
+133
View File
@@ -0,0 +1,133 @@
"""DigiPay-specific flow — search + find via onclick goToProduct URL."""
from __future__ import annotations
import time
from typing import Any
import undetected_chromedriver as uc
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
from crawler.dynamic_flow import DynamicFlow, StepResult, _wait_for
from logger import get_logger
log = get_logger(__name__)
_RESULTS_WAIT_CSS = "a[onclick*='goToProduct']"
# Ordered list of selectors tried to locate the search input
_SEARCH_INPUT_CANDIDATES = [
(By.XPATH, "/html/body/header/header/div[2]//input"),
(By.XPATH, "//header//input[@type='search']"),
(By.XPATH, "//header//input[contains(@class,'search')]"),
(By.XPATH, "//input[@type='search']"),
(By.CSS_SELECTOR, "input[type='search']"),
(By.XPATH, "//input[contains(@placeholder,'جست')]"), # 'جست' = search (Farsi prefix)
]
# Selectors tried to trigger / open the search box before locating the input
_SEARCH_TRIGGER_CANDIDATES = [
(By.XPATH, "/html/body/header/header/div[2]"),
(By.XPATH, "//header//*[contains(@class,'search')]"),
(By.CSS_SELECTOR, "header [class*='search']"),
]
def _wait_spa_ready(driver: uc.Chrome, timeout: float = 20.0) -> None:
"""Wait for DOM ready + Vue/React hydration (no pending network requests)."""
WebDriverWait(driver, timeout).until(
lambda d: d.execute_script("return document.readyState") == "complete"
)
# Extra pause for SPA JS to hydrate
time.sleep(1.5)
def _find_search_input(driver: uc.Chrome, wait: WebDriverWait) -> Any:
"""Try each candidate selector until one is clickable."""
for by, sel in _SEARCH_INPUT_CANDIDATES:
try:
el = wait.until(EC.element_to_be_clickable((by, sel)))
log.debug("Search input found via [%s:%s]", by, sel)
return el
except Exception:
continue
raise RuntimeError("Search input not found — tried all candidate selectors")
class DigiPayFlow(DynamicFlow):
"""
DigiPay-specific flow:
- Waits for SPA hydration before interacting.
- Tries multiple selectors to find/open search input.
- Waits for `a[onclick*='goToProduct']` cards after submit.
- Item matching via onclick URL fragment or product title text.
"""
def _run_search(self, text: str) -> StepResult:
t0 = time.monotonic()
name = f"search:{text}"
log.info("[%s] DigiPay search for '%s'", name, text)
try:
wait_long = WebDriverWait(self.driver, 25)
wait_short = WebDriverWait(self.driver, 5)
# Wait for SPA to fully hydrate
_wait_spa_ready(self.driver)
log.debug("[%s] SPA ready", name)
# Try to click a trigger element to open/reveal the search input
for by, sel in _SEARCH_TRIGGER_CANDIDATES:
try:
trigger = wait_short.until(EC.element_to_be_clickable((by, sel)))
trigger.click()
log.debug("[%s] Clicked trigger [%s:%s]", name, by, sel)
human_delay(0.4, 0.7)
break
except Exception:
continue
# Locate the actual input (may now be in an overlay)
box = _find_search_input(self.driver, wait_long)
box.clear()
human_delay(0.2, 0.4)
for ch in text:
box.send_keys(ch)
time.sleep(0.05)
human_delay(0.3, 0.6)
box.send_keys(Keys.RETURN)
log.debug("[%s] Enter sent", name)
# Wait for product cards
wait_long.until(
EC.presence_of_element_located((By.CSS_SELECTOR, _RESULTS_WAIT_CSS))
)
human_delay(1.0, 2.0)
ms = int((time.monotonic() - t0) * 1000)
log.info("[%s] Search OK — %d ms", name, ms)
return StepResult(name=name, success=True, data={"text": text}, duration_ms=ms)
except Exception as exc:
ms = int((time.monotonic() - t0) * 1000)
log.error("[%s] Failed — %s", name, exc)
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
def _build_item_selector(self, item_id: str) -> tuple[str, str]:
sel_type: str = self.cfg.get("item_selector_type", "onclick")
template: str = self.cfg.get("item_selector_template", "")
fragment = template.replace("{item_id}", item_id) if template else item_id
if sel_type == "onclick":
escaped = fragment.replace("'", "\\'")
return f"//a[contains(@onclick, '{escaped}')]", "xpath"
if sel_type == "text":
escaped = fragment.replace("'", "\\'")
return f"//a[.//*[contains(normalize-space(.), '{escaped}')]]", "xpath"
return super()._build_item_selector(item_id)
crawler/tasks.py
+9 -1
View File
@@ -308,9 +308,17 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
try: try:
target_url: str = cfg.get("target_url") or "" target_url: str = cfg.get("target_url") or ""
scenario: str = cfg.get("scenario") or "dynamic"
with driver_session(headless=headless) as driver: with driver_session(headless=headless) as driver:
FreshSessionScenario(driver, target_url).run() FreshSessionScenario(driver, target_url).run()
result = DynamicFlow(driver, cfg).run() if cfg else None if cfg:
if scenario == "digipay":
from crawler.scenarios.digipay import DigiPayFlow
result = DigiPayFlow(driver, cfg).run()
else:
result = DynamicFlow(driver, cfg).run()
else:
result = None
total_ms = int((time.monotonic() - t0) * 1000) total_ms = int((time.monotonic() - t0) * 1000)
success = result.success if result else True success = result.success if result else True
db/connection.py
+1
View File
@@ -97,6 +97,7 @@ def _migrate(conn: sqlite3.Connection) -> None:
"max_pages": "INTEGER NOT NULL DEFAULT 10", "max_pages": "INTEGER NOT NULL DEFAULT 10",
"pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500", "pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
"target_url": "TEXT NOT NULL DEFAULT ''", "target_url": "TEXT NOT NULL DEFAULT ''",
"scenario": "TEXT NOT NULL DEFAULT 'dynamic'",
} }
for col, definition in additions.items(): for col, definition in additions.items():
if col not in existing: if col not in existing:
db/repositories/flow_configs.py
+1 -1
View File
@@ -17,7 +17,7 @@ _FIELDS = [
"max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
"pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms", "pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms",
"item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json", "item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json",
"target_url", "target_url", "scenario",
] ]
pyproject.toml
+1
View File
@@ -14,6 +14,7 @@ dependencies = [
"pydantic>=2.7.0", "pydantic>=2.7.0",
"python-dotenv>=1.0.1", "python-dotenv>=1.0.1",
"setuptools>=70.0.0", "setuptools>=70.0.0",
"speedtest-cli>=2.1.3",
"rq>=1.16.0", "rq>=1.16.0",
"redis>=5.0.0", "redis>=5.0.0",
] ]
uv.lock
Generated
+11
View File
@@ -633,6 +633,7 @@ dependencies = [
{ name = "selenium" }, { name = "selenium" },
{ name = "selenium-stealth" }, { name = "selenium-stealth" },
{ name = "setuptools" }, { name = "setuptools" },
{ name = "speedtest-cli" },
{ name = "undetected-chromedriver" }, { name = "undetected-chromedriver" },
{ name = "uvicorn", extra = ["standard"] }, { name = "uvicorn", extra = ["standard"] },
] ]
@@ -655,6 +656,7 @@ requires-dist = [
{ name = "selenium", specifier = ">=4.18.0" }, { name = "selenium", specifier = ">=4.18.0" },
{ name = "selenium-stealth", specifier = ">=1.0.6" }, { name = "selenium-stealth", specifier = ">=1.0.6" },
{ name = "setuptools", specifier = ">=70.0.0" }, { name = "setuptools", specifier = ">=70.0.0" },
{ name = "speedtest-cli", specifier = ">=2.1.3" },
{ name = "undetected-chromedriver", specifier = ">=3.5.5" }, { name = "undetected-chromedriver", specifier = ">=3.5.5" },
{ name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" }, { name = "uvicorn", extras = ["standard"], specifier = ">=0.30.0" },
] ]
@@ -726,6 +728,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" }, { url = "https://files.pythonhosted.org/packages/32/46/9cb0e58b2deb7f82b84065f37f3bffeb12413f947f9388e4cac22c4621ce/sortedcontainers-2.4.0-py2.py3-none-any.whl", hash = "sha256:a163dcaede0f1c021485e957a39245190e74249897e2ae4b2aa38595db237ee0", size = 29575, upload-time = "2021-05-16T22:03:41.177Z" },
] ]
[[package]]
name = "speedtest-cli"
version = "2.1.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/85/d2/32c8a30768b788d319f94cde3a77e0ccc1812dca464ad8062d3c4d703e06/speedtest-cli-2.1.3.tar.gz", hash = "sha256:5e2773233cedb5fa3d8120eb7f97bcc4974b5221b254d33ff16e2f1d413d90f0", size = 24721, upload-time = "2021-04-08T13:51:33.627Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/9f/39/65259b7054368b370d3183762484fa2c779ddc41633894d895f9d1720f45/speedtest_cli-2.1.3-py2.py3-none-any.whl", hash = "sha256:75ff32c91af9ac1ce2b905476d6e92bd9eb2c0783f9e7d1939d74605c7d0b9ea", size = 23973, upload-time = "2021-04-08T13:51:32.028Z" },
]
[[package]] [[package]]
name = "starlette" name = "starlette"
version = "1.3.1" version = "1.3.1"