wip
This commit is contained in:
+5
-2
@@ -1,6 +1,8 @@
|
||||
# Copy to .env and fill in
|
||||
|
||||
TARGET_URL=https://example.com
|
||||
# Logging
|
||||
LOG_LEVEL=INFO # DEBUG | INFO | WARNING | ERROR | CRITICAL
|
||||
# LOG_FILE=logs/seed.log # optional rotating file log
|
||||
|
||||
# Admin panel credentials
|
||||
ADMIN_USERNAME=admin
|
||||
@@ -13,7 +15,8 @@ DB_PATH=data/tracker.db
|
||||
|
||||
# Chrome: set to false to watch the browser window
|
||||
HEADLESS=true
|
||||
# CHROME_BINARY=/usr/bin/chromium # optional: explicit Chrome path
|
||||
# CHROME_BINARY=/usr/bin/google-chrome-stable # optional: explicit Chrome binary path
|
||||
# CHROMEDRIVER_PATH=drivers/chromedriver # pre-patched driver (run: make patch-driver)
|
||||
|
||||
# Scenario 1 — comma-separated phone numbers
|
||||
PHONE_NUMBERS=+989100000001,+989100000002,+989100000003
|
||||
|
||||
@@ -5,3 +5,4 @@ __pycache__/
|
||||
data/
|
||||
*.db
|
||||
*.swp
|
||||
drivers/
|
||||
|
||||
@@ -11,7 +11,7 @@ GREEN := $(shell tput setaf 2 2>/dev/null)
|
||||
CYAN := $(shell tput setaf 6 2>/dev/null)
|
||||
|
||||
.PHONY: help install sync lock lint typecheck \
|
||||
admin otp fresh \
|
||||
admin otp fresh batch \
|
||||
build up down logs shell \
|
||||
clean
|
||||
|
||||
@@ -46,6 +46,10 @@ typecheck: ## Run Pyright type-checker
|
||||
|
||||
# ── Crawler ───────────────────────────────────────────────────────────────────
|
||||
|
||||
patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chrome updates)
|
||||
@mkdir -p drivers
|
||||
PYTHONPATH=. $(PYTHON) scripts/patch_driver.py
|
||||
|
||||
admin: ## Start the admin panel (http://localhost:8000)
|
||||
@mkdir -p $(DATA_DIR)
|
||||
$(PYTHON) main.py admin
|
||||
@@ -62,6 +66,14 @@ fresh: ## Run Scenario 2 — fresh cookie session
|
||||
@mkdir -p $(DATA_DIR)
|
||||
$(PYTHON) main.py fresh
|
||||
|
||||
WORKERS ?= 3
|
||||
RUNS ?= 10
|
||||
STAGGER ?= 500
|
||||
|
||||
batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500)
|
||||
@mkdir -p $(DATA_DIR)
|
||||
$(PYTHON) main.py batch --workers $(WORKERS) --runs $(RUNS) --stagger $(STAGGER)
|
||||
|
||||
# ── Docker ────────────────────────────────────────────────────────────────────
|
||||
|
||||
build: ## Build the Docker image
|
||||
|
||||
@@ -1,6 +1,7 @@
|
||||
"""FastAPI admin application."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
@@ -21,6 +22,20 @@ app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
|
||||
@app.on_event("startup")
|
||||
def on_startup() -> None:
|
||||
init_db()
|
||||
_start_huey_consumer()
|
||||
|
||||
|
||||
def _start_huey_consumer() -> None:
|
||||
from huey.consumer import Consumer
|
||||
from crawler.tasks import huey
|
||||
|
||||
class _ThreadConsumer(Consumer):
|
||||
def _set_signal_handlers(self) -> None:
|
||||
pass # signal.signal() only works on the main thread
|
||||
|
||||
consumer = _ThreadConsumer(huey, workers=1, periodic=False)
|
||||
t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
|
||||
t.start()
|
||||
|
||||
|
||||
@app.get("/")
|
||||
|
||||
@@ -2,6 +2,7 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
import uuid
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
@@ -17,6 +18,9 @@ from admin.models import (
|
||||
upsert_flow_config,
|
||||
)
|
||||
from config import config
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
router = APIRouter()
|
||||
security = HTTPBasic()
|
||||
@@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel):
|
||||
search_submit_selector: str = ""
|
||||
search_submit_selector_type: str = "css"
|
||||
search_results_selector: str = ""
|
||||
search_results_selector_type: str = "css"
|
||||
scroll_container_selector: str = ""
|
||||
scroll_container_selector_type: str = "css"
|
||||
max_scrolls: int = 30
|
||||
scroll_pause_ms: int = 1200
|
||||
no_new_content_timeout_ms: int = 3000
|
||||
pagination_selector: str = ""
|
||||
pagination_selector_type: str = "css"
|
||||
max_pages: int = 10
|
||||
pagination_wait_ms: int = 1500
|
||||
item_selector_template: str = ""
|
||||
item_selector_type: str = "css"
|
||||
item_url_template: str = ""
|
||||
target_item_ids: list[str] = []
|
||||
target_url: str = ""
|
||||
|
||||
|
||||
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
||||
@@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
||||
"search_submit_selector": body.search_submit_selector,
|
||||
"search_submit_selector_type": body.search_submit_selector_type,
|
||||
"search_results_selector": body.search_results_selector,
|
||||
"search_results_selector_type": body.search_results_selector_type,
|
||||
"scroll_container_selector": body.scroll_container_selector,
|
||||
"scroll_container_selector_type": body.scroll_container_selector_type,
|
||||
"max_scrolls": body.max_scrolls,
|
||||
"scroll_pause_ms": body.scroll_pause_ms,
|
||||
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
|
||||
"pagination_selector": body.pagination_selector,
|
||||
"pagination_selector_type": body.pagination_selector_type,
|
||||
"max_pages": body.max_pages,
|
||||
"pagination_wait_ms": body.pagination_wait_ms,
|
||||
"item_selector_template": body.item_selector_template,
|
||||
"item_selector_type": body.item_selector_type,
|
||||
"item_url_template": body.item_url_template,
|
||||
"target_item_ids_json": body.target_item_ids,
|
||||
"target_url": body.target_url,
|
||||
}
|
||||
|
||||
|
||||
@@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
|
||||
if not existing:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
await delete_flow_config(cfg_id)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Batch runs (managed via Huey task queue)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class BatchRunIn(BaseModel):
|
||||
workers: int = 3
|
||||
total_runs: int = 10
|
||||
stagger_ms: int = 500
|
||||
headless: bool = True
|
||||
|
||||
|
||||
@router.post("/api/run-batch", status_code=202)
|
||||
async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
|
||||
from crawler.tasks import batch_status as _status, run_batch_task, set_current
|
||||
|
||||
current = _status()
|
||||
if current.get("running"):
|
||||
raise HTTPException(status_code=409, detail="A batch is already running")
|
||||
|
||||
from admin.models import get_active_flow_config
|
||||
active_cfg = await get_active_flow_config()
|
||||
if not active_cfg:
|
||||
raise HTTPException(status_code=422, detail="No active flow config — activate one first")
|
||||
|
||||
batch_id = uuid.uuid4().hex
|
||||
result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
|
||||
set_current(batch_id, result)
|
||||
|
||||
log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
|
||||
return {"batch_id": batch_id, "status": "queued"}
|
||||
|
||||
|
||||
@router.post("/api/stop-batch", status_code=200)
|
||||
async def stop_batch(_: Auth) -> dict[str, object]:
|
||||
from crawler.tasks import stop_current
|
||||
stopped = stop_current()
|
||||
if not stopped:
|
||||
raise HTTPException(status_code=404, detail="No active batch to stop")
|
||||
return {"status": "stopping"}
|
||||
|
||||
|
||||
@router.get("/api/batch-status")
|
||||
async def batch_status(_: Auth) -> dict[str, object]:
|
||||
from crawler.tasks import batch_status as _status
|
||||
return _status() # type: ignore[return-value]
|
||||
|
||||
+592
-78
@@ -102,6 +102,17 @@
|
||||
}
|
||||
.form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
|
||||
.form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
|
||||
details.adv { margin-top: 20px; border: 1px solid var(--border); border-radius: 9px; }
|
||||
details.adv > summary {
|
||||
list-style: none; cursor: pointer; padding: 11px 14px;
|
||||
font-size: 12px; font-weight: 600; color: var(--muted);
|
||||
display: flex; align-items: center; gap: 7px; user-select: none;
|
||||
}
|
||||
details.adv > summary::-webkit-details-marker { display: none; }
|
||||
details.adv > summary::before { content: '▶'; font-size: 9px; transition: transform .15s; }
|
||||
details.adv[open] > summary::before { transform: rotate(90deg); }
|
||||
details.adv > summary:hover { color: var(--text); }
|
||||
.adv-body { padding: 4px 14px 14px; }
|
||||
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
|
||||
.fg { margin-bottom: 14px; }
|
||||
.fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
|
||||
@@ -124,6 +135,9 @@
|
||||
.modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
|
||||
.error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
|
||||
.error-msg.hidden { display: none; }
|
||||
.remember-row { display: flex; align-items: center; gap: 8px; margin-top: 12px; }
|
||||
.remember-row input[type=checkbox] { width: 15px; height: 15px; accent-color: var(--accent); cursor: pointer; }
|
||||
.remember-row label { color: var(--muted); font-size: 13px; cursor: pointer; user-select: none; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
@@ -134,7 +148,11 @@
|
||||
<h2>Seed Admin</h2>
|
||||
<div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
|
||||
<div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
|
||||
<button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
|
||||
<div class="remember-row">
|
||||
<input type="checkbox" id="inp-remember" />
|
||||
<label for="inp-remember">Remember me</label>
|
||||
</div>
|
||||
<button class="btn btn-primary" style="width:100%;margin-top:14px" id="login-btn">Sign in</button>
|
||||
<p class="error-msg hidden" id="auth-error">Invalid credentials</p>
|
||||
</div>
|
||||
</div>
|
||||
@@ -149,80 +167,130 @@
|
||||
<label>Config Name</label>
|
||||
<input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
|
||||
</div>
|
||||
|
||||
<div class="fg">
|
||||
<label style="display:flex;align-items:center;gap:8px;cursor:pointer">
|
||||
<input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
|
||||
</label>
|
||||
<label>Target URL</label>
|
||||
<input id="cfg-target-url" type="url" placeholder="https://example.com" />
|
||||
</div>
|
||||
|
||||
<!-- Search -->
|
||||
<div class="form-section">Search Step</div>
|
||||
<div class="fg">
|
||||
<div class="fg" style="margin-top:6px">
|
||||
<label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
|
||||
<textarea id="cfg-search-texts" placeholder="shoes blue sneakers running shoes"></textarea>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Search Box Selector</label>
|
||||
<input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
|
||||
<input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
|
||||
<input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
|
||||
</div>
|
||||
|
||||
<!-- Scroll -->
|
||||
<div class="form-section">Scroll Step (Infinite Scroll)</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
|
||||
<input id="cfg-scroll-container" type="text" placeholder=".product-list" />
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Max Scrolls</label>
|
||||
<input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Pause (ms)</label>
|
||||
<input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
|
||||
<input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
|
||||
</div>
|
||||
|
||||
<!-- Item -->
|
||||
<div class="form-section">Item Selection</div>
|
||||
<div class="fg">
|
||||
<label>Item Selector Template</label>
|
||||
<input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
|
||||
<p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Selector Type</label>
|
||||
<select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
|
||||
<textarea id="cfg-item-ids" placeholder="prod-1234 prod-5678"></textarea>
|
||||
</div>
|
||||
|
||||
<!-- Advanced -->
|
||||
<details class="adv" id="adv-details">
|
||||
<summary>Advanced Settings</summary>
|
||||
<div class="adv-body">
|
||||
|
||||
<div class="fg">
|
||||
<label style="display:flex;align-items:center;gap:8px;cursor:pointer">
|
||||
<input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<div class="form-section">Search Step</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Search Box Selector</label>
|
||||
<input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Submit Button Selector <span style="color:var(--muted)">(blank → Enter key)</span></label>
|
||||
<input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
|
||||
<input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-results-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section">Scroll</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
|
||||
<div style="display:flex;gap:8px;align-items:center">
|
||||
<input id="cfg-scroll-container" type="text" placeholder=".product-list" style="flex:1" />
|
||||
<select id="cfg-scroll-container-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Max Scrolls</label>
|
||||
<input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Pause (ms)</label>
|
||||
<input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>No New Content Timeout (ms)</label>
|
||||
<input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
|
||||
</div>
|
||||
|
||||
<div class="form-section">Pagination <span style="font-weight:400;color:var(--muted)">(optional)</span></div>
|
||||
<div class="fg">
|
||||
<label>Next Page Button Selector <span style="color:var(--muted)">(blank = disabled)</span></label>
|
||||
<div style="display:flex;gap:8px;align-items:center">
|
||||
<input id="cfg-pagination-sel" type="text" placeholder=".pagination .next, a[rel='next']" style="flex:1" />
|
||||
<select id="cfg-pagination-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Max Pages</label>
|
||||
<input id="cfg-max-pages" type="number" min="1" max="200" value="10" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Wait After Page Turn (ms)</label>
|
||||
<input id="cfg-pagination-wait" type="number" min="200" max="15000" value="1500" />
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="form-section">Item Selection</div>
|
||||
<div class="fg">
|
||||
<label>Item URL Template <span style="color:var(--muted)">(href pattern, highest priority)</span></label>
|
||||
<input id="cfg-item-url-tpl" type="text" placeholder="/products/{item_id}" />
|
||||
<p class="hint">Finds <code><a href></code> whose href contains this pattern. Use <code>{item_id}</code> as placeholder.</p>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Item Selector Template <span style="color:var(--muted)">(fallback)</span></label>
|
||||
<input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or /products/{item_id}" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Selector Type</label>
|
||||
<select id="cfg-item-sel-type" style="width:auto">
|
||||
<option value="css">CSS</option>
|
||||
<option value="xpath">XPath</option>
|
||||
<option value="a-link">A-Link (href contains)</option>
|
||||
</select>
|
||||
<p class="hint" id="alink-hint" style="display:none">Template matched against <code><a href></code> — e.g. <code>/products/{item_id}</code>.</p>
|
||||
</div>
|
||||
|
||||
</div>
|
||||
</details>
|
||||
|
||||
<div class="form-footer">
|
||||
<button class="btn btn-ghost" id="form-cancel">Cancel</button>
|
||||
<button class="btn btn-primary" id="form-save">Save Config</button>
|
||||
@@ -244,6 +312,7 @@
|
||||
<div class="tabs">
|
||||
<div class="tab active" data-tab="dashboard">Dashboard</div>
|
||||
<div class="tab" data-tab="flows">Flow Configs</div>
|
||||
<div class="tab" data-tab="batch">Batch Run</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Dashboard Tab ──────────────────────────────────────────── -->
|
||||
@@ -288,8 +357,12 @@
|
||||
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
|
||||
<div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
|
||||
</div>
|
||||
<button class="btn btn-primary" id="btn-new-config">+ New Config</button>
|
||||
<div style="display:flex;gap:8px">
|
||||
<button class="btn btn-ghost" id="btn-import-config">↑ Import</button>
|
||||
<button class="btn btn-primary" id="btn-new-config">+ New Config</button>
|
||||
</div>
|
||||
</div>
|
||||
<input type="file" id="import-file-input" accept=".json" style="display:none">
|
||||
|
||||
<div class="config-list" id="config-list">
|
||||
<p class="empty">No configs yet. Create one to get started.</p>
|
||||
@@ -297,6 +370,96 @@
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<!-- ── Batch Run Tab ─────────────────────────────────────────── -->
|
||||
<div class="tab-panel" id="tab-batch">
|
||||
<main>
|
||||
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div>
|
||||
<div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using the active flow config.</div>
|
||||
|
||||
<div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px">
|
||||
|
||||
<!-- Controls -->
|
||||
<div class="panel" style="align-self:start">
|
||||
<div class="section-title">Configuration</div>
|
||||
<div class="fg" style="margin-bottom:14px">
|
||||
<label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label>
|
||||
<input id="batch-workers" type="number" min="1" max="50" value="3" />
|
||||
</div>
|
||||
<div class="fg" style="margin-bottom:14px">
|
||||
<label>Total Runs</label>
|
||||
<input id="batch-runs" type="number" min="1" max="10000" value="10" />
|
||||
</div>
|
||||
<div class="fg" style="margin-bottom:14px">
|
||||
<label>Stagger Between Starts (ms)</label>
|
||||
<input id="batch-stagger" type="number" min="0" max="30000" value="500" />
|
||||
</div>
|
||||
<div style="display:flex;align-items:center;gap:8px;margin-bottom:20px">
|
||||
<input id="batch-headless" type="checkbox" checked style="width:16px;height:16px;cursor:pointer" />
|
||||
<label for="batch-headless" style="cursor:pointer;margin:0">Headless</label>
|
||||
</div>
|
||||
<div style="display:flex;gap:10px">
|
||||
<button class="btn btn-primary" id="batch-start-btn" style="flex:1">▶ Start</button>
|
||||
<button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button>
|
||||
</div>
|
||||
<div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div>
|
||||
</div>
|
||||
|
||||
<!-- Status -->
|
||||
<div class="panel">
|
||||
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
|
||||
<div class="section-title" style="margin:0">Live Status</div>
|
||||
<span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span>
|
||||
</div>
|
||||
<div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div>
|
||||
<div id="batch-status-body" style="display:none">
|
||||
<div style="margin-bottom:16px">
|
||||
<div style="display:flex;justify-content:space-between;margin-bottom:6px">
|
||||
<span style="color:var(--muted);font-size:12px">Progress</span>
|
||||
<span id="batch-progress-label" style="font-size:12px">0 / 0</span>
|
||||
</div>
|
||||
<div style="background:var(--surface2);border-radius:4px;height:8px;overflow:hidden">
|
||||
<div id="batch-progress-bar" style="height:100%;background:var(--accent);width:0%;transition:width .3s"></div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:10px;margin-bottom:16px">
|
||||
<div class="card" style="padding:12px">
|
||||
<div class="label">Completed</div>
|
||||
<div class="value accent" id="bs-completed">0</div>
|
||||
</div>
|
||||
<div class="card" style="padding:12px">
|
||||
<div class="label">Succeeded</div>
|
||||
<div class="value green" id="bs-succeeded">0</div>
|
||||
</div>
|
||||
<div class="card" style="padding:12px">
|
||||
<div class="label">Failed</div>
|
||||
<div class="value red" id="bs-failed">0</div>
|
||||
</div>
|
||||
</div>
|
||||
<div style="font-size:12px;color:var(--muted);margin-bottom:14px">
|
||||
Workers: <strong id="bs-workers">–</strong> ·
|
||||
Started: <strong id="bs-started">–</strong>
|
||||
<span id="bs-finished-row" style="display:none"> · Finished: <strong id="bs-finished">–</strong></span>
|
||||
</div>
|
||||
|
||||
<!-- Per-worker table -->
|
||||
<div class="section-title">Workers</div>
|
||||
<table id="bs-worker-table" style="font-size:12px">
|
||||
<thead>
|
||||
<tr>
|
||||
<th>#</th>
|
||||
<th>Status</th>
|
||||
<th>Duration</th>
|
||||
<th>Error</th>
|
||||
</tr>
|
||||
</thead>
|
||||
<tbody id="bs-worker-tbody"></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
/* ─────────────────────────── Auth ─────────────────────────── */
|
||||
let _creds = null;
|
||||
@@ -309,24 +472,73 @@ async function apiFetch(path, opts = {}) {
|
||||
return res;
|
||||
}
|
||||
|
||||
function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
|
||||
function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
|
||||
const _STORAGE_KEY = 'seed_admin_creds';
|
||||
|
||||
function showAuth() {
|
||||
document.getElementById('auth-modal').classList.remove('hidden');
|
||||
}
|
||||
function hideAuth() {
|
||||
document.getElementById('auth-modal').classList.add('hidden');
|
||||
}
|
||||
|
||||
function _loadSaved() {
|
||||
try {
|
||||
const raw = localStorage.getItem(_STORAGE_KEY);
|
||||
if (!raw) return null;
|
||||
return JSON.parse(raw);
|
||||
} catch { return null; }
|
||||
}
|
||||
|
||||
function _saveCreds(u, p) {
|
||||
localStorage.setItem(_STORAGE_KEY, JSON.stringify({ u, p }));
|
||||
}
|
||||
|
||||
function _clearCreds() {
|
||||
localStorage.removeItem(_STORAGE_KEY);
|
||||
}
|
||||
|
||||
async function _attemptLogin(u, p) {
|
||||
const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
|
||||
return res;
|
||||
}
|
||||
|
||||
document.getElementById('login-btn').addEventListener('click', async () => {
|
||||
const u = document.getElementById('inp-user').value.trim();
|
||||
const p = document.getElementById('inp-pass').value;
|
||||
const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
|
||||
const remember = document.getElementById('inp-remember').checked;
|
||||
const res = await _attemptLogin(u, p);
|
||||
if (res.ok) {
|
||||
_creds = { u, p };
|
||||
remember ? _saveCreds(u, p) : _clearCreds();
|
||||
document.getElementById('auth-error').classList.add('hidden');
|
||||
hideAuth();
|
||||
await Promise.all([loadStats(), loadConfigs()]);
|
||||
startAutoRefresh();
|
||||
} else {
|
||||
_clearCreds();
|
||||
document.getElementById('auth-error').classList.remove('hidden');
|
||||
}
|
||||
});
|
||||
document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
|
||||
document.getElementById('inp-pass').addEventListener('keydown', e => {
|
||||
if (e.key === 'Enter') document.getElementById('login-btn').click();
|
||||
});
|
||||
|
||||
// Auto-login from saved credentials on page load
|
||||
(async () => {
|
||||
const saved = _loadSaved();
|
||||
if (!saved) return;
|
||||
const res = await _attemptLogin(saved.u, saved.p);
|
||||
if (res.ok) {
|
||||
_creds = saved;
|
||||
document.getElementById('inp-user').value = saved.u;
|
||||
document.getElementById('inp-remember').checked = true;
|
||||
hideAuth();
|
||||
await Promise.all([loadStats(), loadConfigs()]);
|
||||
startAutoRefresh();
|
||||
} else {
|
||||
_clearCreds(); // saved creds are stale (password changed)
|
||||
}
|
||||
})();
|
||||
|
||||
/* ─────────────────────────── Tabs ─────────────────────────── */
|
||||
document.querySelectorAll('.tab').forEach(tab => {
|
||||
@@ -336,6 +548,7 @@ document.querySelectorAll('.tab').forEach(tab => {
|
||||
tab.classList.add('active');
|
||||
document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
|
||||
if (tab.dataset.tab === 'flows') loadConfigs();
|
||||
if (tab.dataset.tab === 'batch') pollBatchStatus();
|
||||
});
|
||||
});
|
||||
|
||||
@@ -426,11 +639,15 @@ function renderConfigs() {
|
||||
</div>
|
||||
<div class="config-meta" style="margin-top:4px">
|
||||
Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
|
||||
${c.item_url_template ? ` · href pattern: <code style="font-size:11px">${esc(c.item_url_template)}</code>` : ''}
|
||||
${c.pagination_selector ? ` · Pagination: <code style="font-size:11px">${esc(c.pagination_selector)}</code> (max ${c.max_pages} pages)` : ''}
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-actions">
|
||||
${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
|
||||
<button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button>
|
||||
<button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button>
|
||||
<button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
|
||||
</div>
|
||||
</div>
|
||||
@@ -449,25 +666,143 @@ async function deleteConfig(id) {
|
||||
if (res && (res.ok || res.status === 204)) await loadConfigs();
|
||||
}
|
||||
|
||||
function exportConfig(id) {
|
||||
const c = _configs.find(x => x.id === id);
|
||||
if (!c) return;
|
||||
const out = {
|
||||
name: c.name,
|
||||
search_texts: c.search_texts_json || [],
|
||||
search_box_selector: c.search_box_selector,
|
||||
search_box_selector_type: c.search_box_selector_type,
|
||||
search_submit_selector: c.search_submit_selector,
|
||||
search_submit_selector_type: c.search_submit_selector_type,
|
||||
search_results_selector: c.search_results_selector,
|
||||
search_results_selector_type: c.search_results_selector_type,
|
||||
scroll_container_selector: c.scroll_container_selector,
|
||||
scroll_container_selector_type: c.scroll_container_selector_type,
|
||||
max_scrolls: c.max_scrolls,
|
||||
scroll_pause_ms: c.scroll_pause_ms,
|
||||
no_new_content_timeout_ms: c.no_new_content_timeout_ms,
|
||||
pagination_selector: c.pagination_selector,
|
||||
pagination_selector_type: c.pagination_selector_type,
|
||||
max_pages: c.max_pages,
|
||||
pagination_wait_ms: c.pagination_wait_ms,
|
||||
item_selector_template: c.item_selector_template,
|
||||
item_selector_type: c.item_selector_type,
|
||||
item_url_template: c.item_url_template,
|
||||
target_item_ids: c.target_item_ids_json || [],
|
||||
target_url: c.target_url || '',
|
||||
};
|
||||
const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
|
||||
const url = URL.createObjectURL(blob);
|
||||
const a = document.createElement('a');
|
||||
a.href = url; a.download = `${c.name.replace(/[^a-z0-9_-]/gi, '_')}.json`;
|
||||
a.click(); URL.revokeObjectURL(url);
|
||||
}
|
||||
|
||||
async function duplicateConfig(id) {
|
||||
const c = _configs.find(x => x.id === id);
|
||||
if (!c) return;
|
||||
const body = {
|
||||
name: c.name + ' (copy)',
|
||||
is_active: false,
|
||||
search_texts: c.search_texts_json || [],
|
||||
search_box_selector: c.search_box_selector || '',
|
||||
search_box_selector_type: c.search_box_selector_type || 'css',
|
||||
search_submit_selector: c.search_submit_selector || '',
|
||||
search_submit_selector_type: c.search_submit_selector_type || 'css',
|
||||
search_results_selector: c.search_results_selector || '',
|
||||
search_results_selector_type: c.search_results_selector_type || 'css',
|
||||
scroll_container_selector: c.scroll_container_selector || '',
|
||||
scroll_container_selector_type: c.scroll_container_selector_type || 'css',
|
||||
max_scrolls: c.max_scrolls ?? 30,
|
||||
scroll_pause_ms: c.scroll_pause_ms ?? 1200,
|
||||
no_new_content_timeout_ms: c.no_new_content_timeout_ms ?? 3000,
|
||||
pagination_selector: c.pagination_selector || '',
|
||||
pagination_selector_type: c.pagination_selector_type || 'css',
|
||||
max_pages: c.max_pages ?? 10,
|
||||
pagination_wait_ms: c.pagination_wait_ms ?? 1500,
|
||||
item_selector_template: c.item_selector_template || '',
|
||||
item_selector_type: c.item_selector_type || 'css',
|
||||
item_url_template: c.item_url_template || '',
|
||||
target_item_ids: c.target_item_ids_json || [],
|
||||
target_url: c.target_url || '',
|
||||
};
|
||||
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
if (res && res.ok) await loadConfigs();
|
||||
}
|
||||
|
||||
document.getElementById('btn-import-config').addEventListener('click', () => {
|
||||
document.getElementById('import-file-input').click();
|
||||
});
|
||||
document.getElementById('import-file-input').addEventListener('change', async (e) => {
|
||||
const file = e.target.files[0];
|
||||
if (!file) return;
|
||||
e.target.value = '';
|
||||
let data;
|
||||
try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; }
|
||||
const body = {
|
||||
name: data.name || file.name.replace(/\.json$/, ''),
|
||||
is_active: false,
|
||||
search_texts: data.search_texts || [],
|
||||
search_box_selector: data.search_box_selector || '',
|
||||
search_box_selector_type: data.search_box_selector_type || 'css',
|
||||
search_submit_selector: data.search_submit_selector || '',
|
||||
search_submit_selector_type: data.search_submit_selector_type || 'css',
|
||||
search_results_selector: data.search_results_selector || '',
|
||||
search_results_selector_type: data.search_results_selector_type || 'css',
|
||||
scroll_container_selector: data.scroll_container_selector || '',
|
||||
scroll_container_selector_type: data.scroll_container_selector_type || 'css',
|
||||
max_scrolls: data.max_scrolls ?? 30,
|
||||
scroll_pause_ms: data.scroll_pause_ms ?? 1200,
|
||||
no_new_content_timeout_ms: data.no_new_content_timeout_ms ?? 3000,
|
||||
pagination_selector: data.pagination_selector || '',
|
||||
pagination_selector_type: data.pagination_selector_type || 'css',
|
||||
max_pages: data.max_pages ?? 10,
|
||||
pagination_wait_ms: data.pagination_wait_ms ?? 1500,
|
||||
item_selector_template: data.item_selector_template || '',
|
||||
item_selector_type: data.item_selector_type || 'css',
|
||||
item_url_template: data.item_url_template || '',
|
||||
target_item_ids: data.target_item_ids || [],
|
||||
target_url: data.target_url || '',
|
||||
};
|
||||
const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
|
||||
if (res && res.ok) await loadConfigs();
|
||||
});
|
||||
|
||||
/* ─── Form ─── */
|
||||
function openNewForm() {
|
||||
document.getElementById('form-title').textContent = 'New Flow Config';
|
||||
document.getElementById('cfg-id').value = '';
|
||||
document.getElementById('cfg-name').value = '';
|
||||
function _resetAdvanced() {
|
||||
document.getElementById('cfg-active').checked = false;
|
||||
document.getElementById('cfg-search-texts').value = '';
|
||||
document.getElementById('cfg-search-sel').value = '';
|
||||
document.getElementById('cfg-search-sel-type').value = 'css';
|
||||
document.getElementById('cfg-submit-sel').value = '';
|
||||
document.getElementById('cfg-submit-sel-type').value = 'css';
|
||||
document.getElementById('cfg-results-sel').value = '';
|
||||
document.getElementById('cfg-results-sel-type').value = 'css';
|
||||
document.getElementById('cfg-scroll-container').value = '';
|
||||
document.getElementById('cfg-scroll-container-type').value = 'css';
|
||||
document.getElementById('cfg-max-scrolls').value = '30';
|
||||
document.getElementById('cfg-scroll-pause').value = '1200';
|
||||
document.getElementById('cfg-no-content-timeout').value = '3000';
|
||||
document.getElementById('cfg-pagination-sel').value = '';
|
||||
document.getElementById('cfg-pagination-sel-type').value = 'css';
|
||||
document.getElementById('cfg-max-pages').value = '10';
|
||||
document.getElementById('cfg-pagination-wait').value = '1500';
|
||||
document.getElementById('cfg-item-sel').value = '';
|
||||
document.getElementById('cfg-item-sel-type').value = 'css';
|
||||
document.getElementById('alink-hint').style.display = 'none';
|
||||
document.getElementById('cfg-item-url-tpl').value = '';
|
||||
}
|
||||
|
||||
function openNewForm() {
|
||||
document.getElementById('form-title').textContent = 'New Flow Config';
|
||||
document.getElementById('cfg-id').value = '';
|
||||
document.getElementById('cfg-name').value = '';
|
||||
document.getElementById('cfg-target-url').value = '';
|
||||
document.getElementById('cfg-search-texts').value = '';
|
||||
document.getElementById('cfg-item-ids').value = '';
|
||||
_resetAdvanced();
|
||||
document.getElementById('adv-details').removeAttribute('open');
|
||||
document.getElementById('config-overlay').classList.remove('hidden');
|
||||
}
|
||||
|
||||
@@ -477,20 +812,31 @@ function openEditForm(id) {
|
||||
document.getElementById('form-title').textContent = 'Edit Flow Config';
|
||||
document.getElementById('cfg-id').value = id;
|
||||
document.getElementById('cfg-name').value = c.name || '';
|
||||
document.getElementById('cfg-active').checked = !!c.is_active;
|
||||
document.getElementById('cfg-target-url').value = c.target_url || '';
|
||||
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
|
||||
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
|
||||
document.getElementById('cfg-active').checked = !!c.is_active;
|
||||
document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
|
||||
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
|
||||
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
|
||||
document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
|
||||
document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
|
||||
document.getElementById('cfg-results-sel-type').value = c.search_results_selector_type || 'css';
|
||||
document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
|
||||
document.getElementById('cfg-scroll-container-type').value = c.scroll_container_selector_type || 'css';
|
||||
document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
|
||||
document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
|
||||
document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
|
||||
document.getElementById('cfg-pagination-sel').value = c.pagination_selector || '';
|
||||
document.getElementById('cfg-pagination-sel-type').value = c.pagination_selector_type || 'css';
|
||||
document.getElementById('cfg-max-pages').value = c.max_pages ?? 10;
|
||||
document.getElementById('cfg-pagination-wait').value = c.pagination_wait_ms ?? 1500;
|
||||
document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
|
||||
document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
|
||||
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
|
||||
const _ist = c.item_selector_type || 'css';
|
||||
document.getElementById('cfg-item-sel-type').value = _ist;
|
||||
document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
|
||||
document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
|
||||
document.getElementById('adv-details').setAttribute('open', '');
|
||||
document.getElementById('config-overlay').classList.remove('hidden');
|
||||
}
|
||||
|
||||
@@ -508,19 +854,30 @@ function collectForm() {
|
||||
search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
|
||||
search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
|
||||
search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
|
||||
search_results_selector_type: document.getElementById('cfg-results-sel-type').value,
|
||||
scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
|
||||
scroll_container_selector_type: document.getElementById('cfg-scroll-container-type').value,
|
||||
max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
|
||||
scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
|
||||
no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
|
||||
pagination_selector: document.getElementById('cfg-pagination-sel').value.trim(),
|
||||
pagination_selector_type: document.getElementById('cfg-pagination-sel-type').value,
|
||||
max_pages: parseInt(document.getElementById('cfg-max-pages').value) || 10,
|
||||
pagination_wait_ms: parseInt(document.getElementById('cfg-pagination-wait').value) || 1500,
|
||||
item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
|
||||
item_selector_type: document.getElementById('cfg-item-sel-type').value,
|
||||
item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
|
||||
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
|
||||
target_url: document.getElementById('cfg-target-url').value.trim(),
|
||||
};
|
||||
}
|
||||
|
||||
document.getElementById('btn-new-config').addEventListener('click', openNewForm);
|
||||
document.getElementById('form-cancel').addEventListener('click', closeForm);
|
||||
document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
|
||||
document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
|
||||
document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
|
||||
});
|
||||
|
||||
document.getElementById('form-save').addEventListener('click', async () => {
|
||||
const body = collectForm();
|
||||
@@ -540,8 +897,9 @@ document.getElementById('form-save').addEventListener('click', async () => {
|
||||
closeForm();
|
||||
await loadConfigs();
|
||||
} else {
|
||||
alert('Failed to save config. Check console.');
|
||||
console.error(await res?.text());
|
||||
const detail = res ? await res.text().catch(() => `HTTP ${res.status}`) : 'No response (network error)';
|
||||
console.error('Save failed:', detail);
|
||||
alert(`Save failed (${res?.status ?? '?'}):\n${detail}\n\nIf columns are missing, restart the admin panel to apply DB migrations.`);
|
||||
}
|
||||
});
|
||||
|
||||
@@ -553,6 +911,162 @@ function fmtTime(iso) {
|
||||
if (!iso) return '–';
|
||||
return new Date(iso).toLocaleString();
|
||||
}
|
||||
|
||||
/* ─────────────────────────── Batch Run ─────────────────────── */
|
||||
let _batchPollTimer = null;
|
||||
|
||||
const _STATUS_COLOR = {
|
||||
running: 'var(--accent)',
|
||||
ok: 'var(--green)',
|
||||
failed: 'var(--red)',
|
||||
error: 'var(--red)',
|
||||
skipped: 'var(--muted)',
|
||||
pending: 'var(--muted)',
|
||||
};
|
||||
|
||||
function fmtDuration(ms) {
|
||||
if (ms == null) return '–';
|
||||
if (ms < 1000) return ms + ' ms';
|
||||
return (ms / 1000).toFixed(1) + ' s';
|
||||
}
|
||||
|
||||
function fmtTs(ts) {
|
||||
if (!ts) return '–';
|
||||
return new Date(ts * 1000).toLocaleTimeString();
|
||||
}
|
||||
|
||||
function renderBatchStatus(s) {
|
||||
const idle = document.getElementById('batch-idle-msg');
|
||||
const body = document.getElementById('batch-status-body');
|
||||
const errEl = document.getElementById('batch-error');
|
||||
const startBtn = document.getElementById('batch-start-btn');
|
||||
const stopBtn = document.getElementById('batch-stop-btn');
|
||||
const badge = document.getElementById('bs-active-badge');
|
||||
|
||||
if (!s.batch_id) {
|
||||
idle.style.display = '';
|
||||
body.style.display = 'none';
|
||||
startBtn.style.display = '';
|
||||
stopBtn.style.display = 'none';
|
||||
badge.style.display = 'none';
|
||||
return;
|
||||
}
|
||||
|
||||
idle.style.display = 'none';
|
||||
body.style.display = '';
|
||||
|
||||
const pct = s.total ? Math.round(s.completed / s.total * 100) : 0;
|
||||
document.getElementById('batch-progress-bar').style.width = pct + '%';
|
||||
document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`;
|
||||
document.getElementById('bs-completed').textContent = s.completed ?? 0;
|
||||
document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0;
|
||||
document.getElementById('bs-failed').textContent = s.failed ?? 0;
|
||||
document.getElementById('bs-workers').textContent = s.configured_workers ?? '–';
|
||||
document.getElementById('bs-started').textContent = fmtTs(s.started_at);
|
||||
|
||||
const finRow = document.getElementById('bs-finished-row');
|
||||
if (s.finished_at) {
|
||||
finRow.style.display = '';
|
||||
document.getElementById('bs-finished').textContent = fmtTs(s.finished_at);
|
||||
} else {
|
||||
finRow.style.display = 'none';
|
||||
}
|
||||
|
||||
// Active worker badge
|
||||
const active = s.active_workers ?? 0;
|
||||
if (s.running && active > 0) {
|
||||
badge.textContent = `${active} / ${s.configured_workers} active`;
|
||||
badge.style.display = '';
|
||||
} else {
|
||||
badge.style.display = 'none';
|
||||
}
|
||||
|
||||
// Per-worker table
|
||||
const tbody = document.getElementById('bs-worker-tbody');
|
||||
const workers = s.workers_detail || [];
|
||||
tbody.innerHTML = workers.map(w => {
|
||||
const color = _STATUS_COLOR[w.status] || 'var(--muted)';
|
||||
const label = w.status === 'running' ? '⟳ running' : w.status;
|
||||
const err = w.error && w.status !== 'skipped' ? `<span style="color:var(--red)">${esc(w.error)}</span>` : '–';
|
||||
return `<tr>
|
||||
<td style="font-family:monospace">${w.slot}</td>
|
||||
<td style="color:${color};font-weight:600">${label}</td>
|
||||
<td>${fmtDuration(w.duration_ms)}</td>
|
||||
<td style="max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td>
|
||||
</tr>`;
|
||||
}).join('');
|
||||
|
||||
if (s.error) {
|
||||
errEl.textContent = 'Error: ' + s.error;
|
||||
errEl.style.display = '';
|
||||
} else {
|
||||
errEl.style.display = 'none';
|
||||
}
|
||||
|
||||
if (s.running) {
|
||||
startBtn.style.display = 'none';
|
||||
stopBtn.style.display = '';
|
||||
stopBtn.disabled = s.stopping || false;
|
||||
stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop';
|
||||
} else {
|
||||
startBtn.style.display = '';
|
||||
stopBtn.style.display = 'none';
|
||||
}
|
||||
}
|
||||
|
||||
async function pollBatchStatus() {
|
||||
const res = await apiFetch('/api/batch-status');
|
||||
if (!res || !res.ok) return;
|
||||
const s = await res.json();
|
||||
renderBatchStatus(s);
|
||||
if (s.running && !_batchPollTimer) startBatchPoll();
|
||||
}
|
||||
|
||||
function startBatchPoll() {
|
||||
if (_batchPollTimer) return;
|
||||
_batchPollTimer = setInterval(async () => {
|
||||
const r = await apiFetch('/api/batch-status');
|
||||
if (!r || !r.ok) return;
|
||||
const st = await r.json();
|
||||
renderBatchStatus(st);
|
||||
if (!st.running) {
|
||||
clearInterval(_batchPollTimer);
|
||||
_batchPollTimer = null;
|
||||
}
|
||||
}, 1500);
|
||||
}
|
||||
|
||||
document.getElementById('batch-start-btn').addEventListener('click', async () => {
|
||||
const workers = parseInt(document.getElementById('batch-workers').value) || 3;
|
||||
const runs = parseInt(document.getElementById('batch-runs').value) || 10;
|
||||
const stagger = parseInt(document.getElementById('batch-stagger').value) || 0;
|
||||
const headless = document.getElementById('batch-headless').checked;
|
||||
|
||||
const res = await apiFetch('/api/run-batch', {
|
||||
method: 'POST',
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify({ workers, total_runs: runs, stagger_ms: stagger, headless }),
|
||||
});
|
||||
|
||||
if (!res) return;
|
||||
if (res.status === 409) { alert('A batch is already running.'); return; }
|
||||
if (res.status === 422) { const d = await res.json(); alert(d.detail || 'No active flow config.'); return; }
|
||||
if (!res.ok) { alert('Failed to start batch.'); return; }
|
||||
|
||||
await pollBatchStatus();
|
||||
startBatchPoll();
|
||||
});
|
||||
|
||||
document.getElementById('batch-stop-btn').addEventListener('click', async () => {
|
||||
const btn = document.getElementById('batch-stop-btn');
|
||||
btn.disabled = true;
|
||||
btn.textContent = '■ Stopping…';
|
||||
const res = await apiFetch('/api/stop-batch', { method: 'POST' });
|
||||
if (!res || !res.ok) {
|
||||
btn.disabled = false;
|
||||
btn.textContent = '■ Stop';
|
||||
}
|
||||
});
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@@ -9,7 +9,6 @@ load_dotenv()
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
target_url: str = os.getenv("TARGET_URL", "https://example.com")
|
||||
admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
|
||||
admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
|
||||
admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
|
||||
@@ -17,6 +16,7 @@ class Config:
|
||||
db_path: str = os.getenv("DB_PATH", "data/tracker.db")
|
||||
headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
|
||||
chrome_binary: str | None = os.getenv("CHROME_BINARY")
|
||||
chromedriver_path: str = os.getenv("CHROMEDRIVER_PATH", "drivers/chromedriver")
|
||||
|
||||
# Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
|
||||
phone_numbers: list[str] = field(default_factory=list)
|
||||
|
||||
@@ -0,0 +1,117 @@
|
||||
"""Batch runner — spawn N parallel Chrome workers to execute the active flow config."""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import threading
|
||||
import time
|
||||
import uuid
|
||||
from concurrent.futures import Future, ThreadPoolExecutor, as_completed
|
||||
from typing import Any, Callable
|
||||
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def _single_run(
|
||||
run_index: int,
|
||||
batch_id: str,
|
||||
cfg: dict[str, Any],
|
||||
headless: bool,
|
||||
) -> tuple[bool, str | None]:
|
||||
"""One isolated Chrome session. Returns (success, failure_reason)."""
|
||||
from crawler.driver import driver_session
|
||||
from crawler.dynamic_flow import DynamicFlow
|
||||
from crawler.scenarios.fresh_session import FreshSessionScenario
|
||||
from db.repositories.runs import runs_repo
|
||||
|
||||
identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
|
||||
t0 = time.monotonic()
|
||||
log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
|
||||
try:
|
||||
target_url: str = cfg.get("target_url") or ""
|
||||
with driver_session(headless=headless) as driver:
|
||||
FreshSessionScenario(driver, target_url).run()
|
||||
result = DynamicFlow(driver, cfg).run() if cfg else None
|
||||
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
if result:
|
||||
steps_data = [
|
||||
{"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
|
||||
for s in result.steps
|
||||
]
|
||||
asyncio.run(runs_repo.insert(
|
||||
"batch", identifier, result.success,
|
||||
result.failure_reason, steps_data, total_ms,
|
||||
))
|
||||
log.info("[batch:%s] Run %d done — success=%s (%d ms)", batch_id[:8], run_index, result.success, total_ms)
|
||||
return result.success, result.failure_reason
|
||||
else:
|
||||
asyncio.run(runs_repo.insert("batch", identifier, True, None, [], total_ms))
|
||||
log.info("[batch:%s] Run %d done (no flow config) — %d ms", batch_id[:8], run_index, total_ms)
|
||||
return True, None
|
||||
|
||||
except Exception as exc:
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
|
||||
log.error("[batch:%s] Run %d failed — %s (%d ms)", batch_id[:8], run_index, exc, total_ms)
|
||||
return False, str(exc)
|
||||
|
||||
|
||||
def run_batch(
|
||||
cfg: dict[str, Any],
|
||||
workers: int,
|
||||
total_runs: int,
|
||||
headless: bool = True,
|
||||
stagger_ms: int = 0,
|
||||
on_progress: Callable[[int, int, int], None] | None = None,
|
||||
) -> dict[str, Any]:
|
||||
"""
|
||||
Run `total_runs` crawler sessions with up to `workers` parallel Chrome processes.
|
||||
|
||||
`stagger_ms` delays each successive submission to the pool, spreading
|
||||
Chrome startup load and reducing bot-detection fingerprint clustering.
|
||||
|
||||
`on_progress(completed, succeeded, failed)` is called after every finished run.
|
||||
|
||||
Returns {"batch_id", "total", "succeeded", "failed"}.
|
||||
"""
|
||||
batch_id = uuid.uuid4().hex
|
||||
log.info(
|
||||
"Batch %s — %d run(s), %d worker(s), stagger=%d ms, headless=%s",
|
||||
batch_id[:8], total_runs, workers, stagger_ms, headless,
|
||||
)
|
||||
|
||||
succeeded = 0
|
||||
failed = 0
|
||||
lock = threading.Lock()
|
||||
|
||||
with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=f"batch-{batch_id[:8]}") as pool:
|
||||
futures: list[Future[tuple[bool, str | None]]] = []
|
||||
for i in range(total_runs):
|
||||
futures.append(pool.submit(_single_run, i, batch_id, cfg, headless))
|
||||
if stagger_ms > 0 and i < total_runs - 1:
|
||||
time.sleep(stagger_ms / 1000)
|
||||
|
||||
for fut in as_completed(futures):
|
||||
try:
|
||||
success, _ = fut.result()
|
||||
except Exception as exc:
|
||||
log.error("[batch:%s] Unhandled thread exception: %s", batch_id[:8], exc)
|
||||
success = False
|
||||
|
||||
with lock:
|
||||
if success:
|
||||
succeeded += 1
|
||||
else:
|
||||
failed += 1
|
||||
completed = succeeded + failed
|
||||
|
||||
if on_progress:
|
||||
on_progress(completed, succeeded, failed)
|
||||
|
||||
log.info(
|
||||
"Batch %s finished — %d/%d succeeded, %d failed",
|
||||
batch_id[:8], succeeded, total_runs, failed,
|
||||
)
|
||||
return {"batch_id": batch_id, "total": total_runs, "succeeded": succeeded, "failed": failed}
|
||||
+72
-10
@@ -1,9 +1,14 @@
|
||||
"""Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import random
|
||||
import shutil
|
||||
import subprocess
|
||||
import sys
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from pathlib import Path
|
||||
from typing import Generator
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
@@ -11,6 +16,16 @@ from selenium.webdriver.chrome.options import Options
|
||||
from selenium_stealth import stealth
|
||||
|
||||
from config import config
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
# Selenium talks to ChromeDriver over localhost — must not go through any system proxy.
|
||||
for _var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
|
||||
os.environ.pop(_var, None)
|
||||
_no_proxy = "localhost,127.0.0.1,::1"
|
||||
os.environ.setdefault("no_proxy", _no_proxy)
|
||||
os.environ.setdefault("NO_PROXY", _no_proxy)
|
||||
|
||||
_USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
@@ -18,31 +33,79 @@ _USER_AGENTS = [
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
_CHROME_CANDIDATES = [
|
||||
"google-chrome-stable",
|
||||
"google-chrome",
|
||||
"chromium",
|
||||
"chromium-browser",
|
||||
]
|
||||
|
||||
def _build_options(user_agent: str, headless: bool) -> Options:
|
||||
|
||||
def _detect_chrome() -> tuple[str, int]:
|
||||
"""Return (binary_path, major_version) for the first Chrome found."""
|
||||
candidates = list(_CHROME_CANDIDATES)
|
||||
if config.chrome_binary:
|
||||
candidates.insert(0, config.chrome_binary)
|
||||
|
||||
for binary in candidates:
|
||||
resolved = shutil.which(binary) or (binary if Path(binary).exists() else None)
|
||||
if not resolved:
|
||||
continue
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
[resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
|
||||
).strip()
|
||||
major = int(out.split()[-1].split(".")[0])
|
||||
log.debug("Detected Chrome %d via '%s'", major, resolved)
|
||||
return resolved, major
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
log.critical("No Chrome binary found. Install Chrome or set CHROME_BINARY in .env")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _local_driver_path() -> str | None:
|
||||
path = Path(config.chromedriver_path)
|
||||
if path.exists():
|
||||
return str(path)
|
||||
log.warning(
|
||||
"No patched ChromeDriver at '%s'. Run 'make patch-driver' once. Falling back to auto-download.",
|
||||
path,
|
||||
)
|
||||
return None
|
||||
|
||||
|
||||
def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
|
||||
opts = Options()
|
||||
opts.binary_location = chrome_binary
|
||||
if headless:
|
||||
opts.add_argument("--headless=new")
|
||||
opts.add_argument(f"--user-agent={user_agent}")
|
||||
opts.add_argument("--no-sandbox")
|
||||
opts.add_argument("--disable-dev-shm-usage")
|
||||
opts.add_argument("--disable-gpu")
|
||||
opts.add_argument("--disable-blink-features=AutomationControlled")
|
||||
opts.add_argument("--disable-infobars")
|
||||
opts.add_argument("--window-size=1920,1080")
|
||||
opts.add_experimental_option("excludeSwitches", ["enable-automation"])
|
||||
opts.add_experimental_option("useAutomationExtension", False)
|
||||
if config.chrome_binary:
|
||||
opts.binary_location = config.chrome_binary
|
||||
return opts
|
||||
|
||||
|
||||
def make_driver(headless: bool | None = None) -> uc.Chrome:
|
||||
"""Return a stealthed undetected Chrome instance."""
|
||||
use_headless = config.headless if headless is None else headless
|
||||
chrome_binary, major = _detect_chrome()
|
||||
ua = random.choice(_USER_AGENTS)
|
||||
opts = _build_options(ua, use_headless)
|
||||
opts = _build_options(ua, use_headless, chrome_binary)
|
||||
|
||||
driver = uc.Chrome(options=opts, use_subprocess=True)
|
||||
log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
|
||||
|
||||
driver = uc.Chrome(
|
||||
options=opts,
|
||||
driver_executable_path=_local_driver_path(),
|
||||
version_main=major,
|
||||
use_subprocess=True,
|
||||
)
|
||||
|
||||
stealth(
|
||||
driver,
|
||||
@@ -54,7 +117,6 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
|
||||
fix_hairline=True,
|
||||
)
|
||||
|
||||
# Mask navigator.webdriver via CDP
|
||||
driver.execute_cdp_cmd(
|
||||
"Page.addScriptToEvaluateOnNewDocument",
|
||||
{
|
||||
@@ -65,19 +127,19 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
|
||||
"""
|
||||
},
|
||||
)
|
||||
log.info("Driver ready (Chrome %d)", major)
|
||||
return driver
|
||||
|
||||
|
||||
@contextmanager
|
||||
def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
|
||||
"""Context manager that guarantees driver.quit() on exit."""
|
||||
driver = make_driver(headless)
|
||||
try:
|
||||
yield driver
|
||||
finally:
|
||||
driver.quit()
|
||||
log.debug("Driver session closed")
|
||||
|
||||
|
||||
def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
|
||||
"""Sleep for a random human-like interval."""
|
||||
time.sleep(random.uniform(lo, hi))
|
||||
|
||||
+492
-76
@@ -4,15 +4,17 @@ Dynamic flow executor — reads the active flow config from DB at runtime and ru
|
||||
2. scroll_step : infinite-scroll until target item is visible or max_scrolls reached
|
||||
3. click_step : click each target item id
|
||||
"""
|
||||
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import math
|
||||
import random
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.common.exceptions import NoSuchElementException, TimeoutException
|
||||
from selenium.common.exceptions import NoSuchElementException
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
@@ -21,6 +23,9 @@ from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from crawler.driver import human_delay
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
class DynamicFlowError(Exception):
|
||||
@@ -71,10 +76,30 @@ def _wait_for(
|
||||
selector: str,
|
||||
selector_type: str,
|
||||
timeout: float = 15.0,
|
||||
label: str = "",
|
||||
) -> WebElement:
|
||||
return WebDriverWait(driver, timeout).until(
|
||||
EC.presence_of_element_located((_by(selector_type), selector))
|
||||
log.debug(
|
||||
"Waiting for %s [%s:%s] (timeout=%ss)",
|
||||
label or "element",
|
||||
selector_type,
|
||||
selector,
|
||||
timeout,
|
||||
)
|
||||
try:
|
||||
el = WebDriverWait(driver, timeout).until(
|
||||
EC.presence_of_element_located((_by(selector_type), selector))
|
||||
)
|
||||
log.debug("Found %s [%s:%s]", label or "element", selector_type, selector)
|
||||
return el
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Selector not found — %s [%s:%s]: %s",
|
||||
label or "element",
|
||||
selector_type,
|
||||
selector,
|
||||
exc,
|
||||
)
|
||||
raise
|
||||
|
||||
|
||||
class DynamicFlow:
|
||||
@@ -87,6 +112,7 @@ class DynamicFlow:
|
||||
config_id=self.cfg["id"],
|
||||
config_name=self.cfg["name"],
|
||||
)
|
||||
log.info("Starting flow '%s' (id=%s)", self.cfg["name"], self.cfg["id"])
|
||||
|
||||
search_texts: list[str] = self.cfg.get("search_texts_json") or []
|
||||
item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
|
||||
@@ -95,6 +121,7 @@ class DynamicFlow:
|
||||
step = self._run_search(text)
|
||||
result.steps.append(step)
|
||||
if not step.success:
|
||||
log.error("Search step failed for '%s' — aborting flow", text)
|
||||
return result
|
||||
human_delay(0.8, 1.5)
|
||||
|
||||
@@ -102,12 +129,21 @@ class DynamicFlow:
|
||||
scroll_step = self._run_scroll_and_find(item_id)
|
||||
result.steps.append(scroll_step)
|
||||
if not scroll_step.success:
|
||||
continue # try next item_id, don't abort the whole flow
|
||||
log.warning(
|
||||
"Scroll/find failed for item '%s' — skipping click", item_id
|
||||
)
|
||||
continue
|
||||
|
||||
click_step = self._run_click(item_id, scroll_step.data.get("element"))
|
||||
result.steps.append(click_step)
|
||||
human_delay(1.0, 2.5)
|
||||
|
||||
log.info(
|
||||
"Flow '%s' finished — success=%s, steps=%d",
|
||||
self.cfg["name"],
|
||||
result.success,
|
||||
len(result.steps),
|
||||
)
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -116,44 +152,70 @@ class DynamicFlow:
|
||||
def _run_search(self, text: str) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"search:{text}"
|
||||
log.info("[%s] Searching for '%s'", name, text)
|
||||
try:
|
||||
sel = self.cfg["search_box_selector"]
|
||||
sel_type = self.cfg["search_box_selector_type"]
|
||||
if not sel:
|
||||
raise DynamicFlowError("search_box_selector is not configured")
|
||||
|
||||
box = _wait_for(self.driver, sel, sel_type)
|
||||
box = _wait_for(self.driver, sel, sel_type, label="search box")
|
||||
box.clear()
|
||||
human_delay(0.3, 0.6)
|
||||
|
||||
# Type character by character for a human feel
|
||||
for ch in text:
|
||||
box.send_keys(ch)
|
||||
time.sleep(0.04)
|
||||
log.debug("[%s] Typed search text", name)
|
||||
|
||||
human_delay(0.4, 0.8)
|
||||
|
||||
# Submit: explicit button or Enter
|
||||
submit_sel = self.cfg.get("search_submit_selector", "")
|
||||
if submit_sel:
|
||||
submit_type = self.cfg.get("search_submit_selector_type", "css")
|
||||
btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
|
||||
log.debug(
|
||||
"[%s] Clicking submit button [%s:%s]", name, submit_type, submit_sel
|
||||
)
|
||||
btn = _wait_for(
|
||||
self.driver,
|
||||
submit_sel,
|
||||
submit_type,
|
||||
timeout=5.0,
|
||||
label="submit button",
|
||||
)
|
||||
btn.click()
|
||||
else:
|
||||
log.debug("[%s] Submitting via Enter key", name)
|
||||
box.send_keys(Keys.RETURN)
|
||||
|
||||
# Wait for results container if configured
|
||||
results_sel = self.cfg.get("search_results_selector", "")
|
||||
results_sel_type = self.cfg.get("search_results_selector_type", "css")
|
||||
if results_sel:
|
||||
_wait_for(self.driver, results_sel, "css", timeout=15.0)
|
||||
log.debug(
|
||||
"[%s] Waiting for results container [%s:%s]",
|
||||
name,
|
||||
results_sel_type,
|
||||
results_sel,
|
||||
)
|
||||
_wait_for(
|
||||
self.driver,
|
||||
results_sel,
|
||||
results_sel_type,
|
||||
timeout=15.0,
|
||||
label="results container",
|
||||
)
|
||||
|
||||
human_delay(1.0, 2.0)
|
||||
return StepResult(name=name, success=True, data={"text": text},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.info("[%s] Search OK — %d ms", name, ms)
|
||||
return StepResult(
|
||||
name=name, success=True, data={"text": text}, duration_ms=ms
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.error("[%s] Failed — %s", name, exc)
|
||||
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step: scroll (infinite-scroll) until item selector matches
|
||||
@@ -161,56 +223,270 @@ class DynamicFlow:
|
||||
def _run_scroll_and_find(self, item_id: str) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"scroll_find:{item_id}"
|
||||
log.info("[%s] Scrolling to find item '%s'", name, item_id)
|
||||
try:
|
||||
item_sel = self._build_item_selector(item_id)
|
||||
item_sel_type = self.cfg.get("item_selector_type", "css")
|
||||
item_sel, item_sel_type = self._build_item_selector(item_id)
|
||||
scroll_container = self.cfg.get("scroll_container_selector", "").strip()
|
||||
scroll_container_type = self.cfg.get(
|
||||
"scroll_container_selector_type", "css"
|
||||
)
|
||||
max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
|
||||
pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
|
||||
no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
|
||||
pagination_sel: str = self.cfg.get("pagination_selector", "").strip()
|
||||
pagination_sel_type: str = self.cfg.get("pagination_selector_type", "css")
|
||||
max_pages: int = int(self.cfg.get("max_pages", 10))
|
||||
pagination_wait_ms: int = int(self.cfg.get("pagination_wait_ms", 1500))
|
||||
|
||||
log.debug(
|
||||
"[%s] Selector [%s:%s], container=[%s:'%s'], max_scrolls=%d, pagination=%s",
|
||||
name,
|
||||
item_sel_type,
|
||||
item_sel,
|
||||
scroll_container_type,
|
||||
scroll_container or "window",
|
||||
max_scrolls,
|
||||
(
|
||||
f"[{pagination_sel_type}:{pagination_sel}] max_pages={max_pages}"
|
||||
if pagination_sel
|
||||
else "off"
|
||||
),
|
||||
)
|
||||
|
||||
# Check if already visible before scrolling
|
||||
found = self._find_item_visible(item_sel, item_sel_type)
|
||||
if found:
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "scrolls": 0, "element": found},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
log.info("[%s] Item visible immediately (no scroll needed)", name)
|
||||
return StepResult(
|
||||
name=name,
|
||||
success=True,
|
||||
data={
|
||||
"item_id": item_id,
|
||||
"scrolls": 0,
|
||||
"pages": 0,
|
||||
"element": found,
|
||||
},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000),
|
||||
)
|
||||
|
||||
last_height = self._get_scroll_height(scroll_container)
|
||||
last_height = self._get_scroll_height(
|
||||
scroll_container, scroll_container_type
|
||||
)
|
||||
stale_since: float | None = None
|
||||
total_scrolls = 0
|
||||
page_num = 0
|
||||
scroll_on_page = 0
|
||||
|
||||
for scroll_num in range(1, max_scrolls + 1):
|
||||
self._scroll_down(scroll_container)
|
||||
time.sleep(pause_ms / 1000)
|
||||
|
||||
while scroll_on_page < max_scrolls:
|
||||
# Check for the item BEFORE scrolling so items near the top of a
|
||||
# freshly-paginated page are seen before we jump to the bottom.
|
||||
found = self._find_item_visible(item_sel, item_sel_type)
|
||||
if found:
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "scrolls": scroll_num, "element": found},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.info(
|
||||
"[%s] Found after %d scroll(s) on page %d — %d ms",
|
||||
name,
|
||||
total_scrolls,
|
||||
page_num + 1,
|
||||
ms,
|
||||
)
|
||||
return StepResult(
|
||||
name=name,
|
||||
success=True,
|
||||
data={
|
||||
"item_id": item_id,
|
||||
"scrolls": total_scrolls,
|
||||
"pages": page_num,
|
||||
"element": found,
|
||||
},
|
||||
duration_ms=ms,
|
||||
)
|
||||
|
||||
new_height = self._get_scroll_height(scroll_container)
|
||||
scroll_on_page += 1
|
||||
total_scrolls += 1
|
||||
self._scroll_down(scroll_container, scroll_container_type)
|
||||
time.sleep(pause_ms / 1000)
|
||||
|
||||
new_height = self._get_scroll_height(
|
||||
scroll_container, scroll_container_type
|
||||
)
|
||||
if new_height == last_height:
|
||||
if stale_since is None:
|
||||
stale_since = time.monotonic()
|
||||
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
|
||||
raise DynamicFlowError(
|
||||
f"No new content after {no_new_ms} ms — reached end of page "
|
||||
f"without finding item '{item_id}'"
|
||||
log.debug(
|
||||
"[%s] Page height unchanged at scroll %d — starting stale timer",
|
||||
name,
|
||||
scroll_on_page,
|
||||
)
|
||||
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
|
||||
# Infinite scroll exhausted — try pagination if configured
|
||||
if pagination_sel and page_num < max_pages:
|
||||
if self._try_paginate(
|
||||
name,
|
||||
pagination_sel,
|
||||
pagination_sel_type,
|
||||
pagination_wait_ms,
|
||||
):
|
||||
page_num += 1
|
||||
scroll_on_page = 0
|
||||
stale_since = None
|
||||
last_height = self._get_scroll_height(
|
||||
scroll_container, scroll_container_type
|
||||
)
|
||||
log.info(
|
||||
"[%s] Advanced to pagination page %d",
|
||||
name,
|
||||
page_num + 1,
|
||||
)
|
||||
# Loop continues — item check happens at top of next iteration
|
||||
else:
|
||||
raise DynamicFlowError(
|
||||
f"Pagination button [{pagination_sel_type}:{pagination_sel}] "
|
||||
f"not visible on page {page_num + 1}"
|
||||
)
|
||||
elif pagination_sel and page_num >= max_pages:
|
||||
raise DynamicFlowError(
|
||||
f"Item '{item_id}' not found after {page_num} pagination page(s)"
|
||||
)
|
||||
else:
|
||||
raise DynamicFlowError(
|
||||
f"No new content after {no_new_ms} ms — reached end of page "
|
||||
f"without finding item '{item_id}'"
|
||||
)
|
||||
else:
|
||||
stale_since = None
|
||||
last_height = new_height
|
||||
log.debug(
|
||||
"[%s] Scroll %d (page %d) — new height %d px",
|
||||
name,
|
||||
scroll_on_page,
|
||||
page_num + 1,
|
||||
new_height,
|
||||
)
|
||||
|
||||
raise DynamicFlowError(
|
||||
f"Item '{item_id}' not found after {max_scrolls} scrolls"
|
||||
f"Item '{item_id}' not found after {total_scrolls} scrolls"
|
||||
)
|
||||
|
||||
except DynamicFlowError:
|
||||
raise
|
||||
except DynamicFlowError as exc:
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.error("[%s] Failed — %s", name, exc)
|
||||
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.error("[%s] Unexpected error — %s", name, exc)
|
||||
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Human mouse simulation
|
||||
# ------------------------------------------------------------------
|
||||
def _human_click(self, el: WebElement, name: str = "") -> None:
|
||||
"""
|
||||
Simulate a human-like mouse approach and click:
|
||||
1. Smooth scroll element into viewport
|
||||
2. Dispatch JS mousemove events along a curved path
|
||||
3. Trigger mouseenter / mouseover on the element
|
||||
4. Click via ActionChains with random sub-element offset
|
||||
5. Fallback: synthetic JS mouse event sequence
|
||||
"""
|
||||
# Step 1 — bring element into view smoothly
|
||||
self.driver.execute_script(
|
||||
"arguments[0].scrollIntoView({block:'center', behavior:'smooth'});", el
|
||||
)
|
||||
time.sleep(random.uniform(0.35, 0.7))
|
||||
|
||||
# Step 2 — get element centre in viewport coordinates
|
||||
rect: dict[str, float] = self.driver.execute_script(
|
||||
"""
|
||||
const r = arguments[0].getBoundingClientRect();
|
||||
return {x: r.left, y: r.top, w: r.width, h: r.height,
|
||||
vw: window.innerWidth, vh: window.innerHeight};
|
||||
""",
|
||||
el,
|
||||
)
|
||||
cx = rect["x"] + rect["w"] / 2
|
||||
cy = rect["y"] + rect["h"] / 2
|
||||
vw = rect["vw"]
|
||||
vh = rect["vh"]
|
||||
|
||||
# Step 3 — build a curved path from a random off-element start
|
||||
sx = random.uniform(vw * 0.1, vw * 0.5)
|
||||
sy = random.uniform(vh * 0.1, vh * 0.5)
|
||||
# Control point for quadratic Bézier — pulled perpendicular to the straight line
|
||||
dx, dy = cx - sx, cy - sy
|
||||
perp_x = -dy * random.uniform(0.15, 0.4)
|
||||
perp_y = dx * random.uniform(0.15, 0.4)
|
||||
qx = sx + dx / 2 + perp_x
|
||||
qy = sy + dy / 2 + perp_y
|
||||
|
||||
steps = random.randint(10, 18)
|
||||
prev_mx, prev_my = sx, sy
|
||||
for i in range(1, steps + 1):
|
||||
t = i / steps
|
||||
# Quadratic Bézier point
|
||||
bx = (1 - t) ** 2 * sx + 2 * (1 - t) * t * qx + t**2 * cx
|
||||
by = (1 - t) ** 2 * sy + 2 * (1 - t) * t * qy + t**2 * cy
|
||||
# Add small Gaussian jitter that fades near the target
|
||||
jitter = max(0.0, 1.0 - t) * 4.0
|
||||
mx = bx + random.gauss(0, jitter)
|
||||
my = by + random.gauss(0, jitter)
|
||||
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
document.dispatchEvent(new MouseEvent('mousemove', {
|
||||
bubbles: true, cancelable: true,
|
||||
clientX: arguments[0], clientY: arguments[1]
|
||||
}));
|
||||
""",
|
||||
int(mx),
|
||||
int(my),
|
||||
)
|
||||
# Variable inter-step pause — faster in the middle, slower near target
|
||||
speed = 0.5 + 0.5 * math.sin(math.pi * t)
|
||||
time.sleep(random.uniform(0.008, 0.025) / max(speed, 0.3))
|
||||
prev_mx, prev_my = mx, my
|
||||
|
||||
# Step 4 — hover events on the element itself
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
arguments[0].dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
|
||||
arguments[0].dispatchEvent(new MouseEvent('mouseover', {bubbles: true}));
|
||||
""",
|
||||
el,
|
||||
)
|
||||
time.sleep(random.uniform(0.1, 0.28))
|
||||
|
||||
# Step 5 — ActionChains click with random offset from centre
|
||||
off_x = int(random.uniform(-rect["w"] * 0.28, rect["w"] * 0.28))
|
||||
off_y = int(random.uniform(-rect["h"] * 0.28, rect["h"] * 0.28))
|
||||
log.debug(
|
||||
"[%s] Human click — target=(%.0f,%.0f) offset=(%d,%d)",
|
||||
name,
|
||||
cx,
|
||||
cy,
|
||||
off_x,
|
||||
off_y,
|
||||
)
|
||||
try:
|
||||
ActionChains(self.driver).move_to_element_with_offset(
|
||||
el, off_x, off_y
|
||||
).pause(random.uniform(0.05, 0.14)).click().perform()
|
||||
log.debug("[%s] ActionChains click OK", name)
|
||||
except Exception as ac_exc:
|
||||
log.warning(
|
||||
"[%s] ActionChains failed (%s) — synthetic JS click", name, ac_exc
|
||||
)
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
const opts = {bubbles: true, cancelable: true,
|
||||
clientX: arguments[1], clientY: arguments[2]};
|
||||
arguments[0].dispatchEvent(new MouseEvent('mousedown', opts));
|
||||
arguments[0].dispatchEvent(new MouseEvent('mouseup', opts));
|
||||
arguments[0].dispatchEvent(new MouseEvent('click', opts));
|
||||
""",
|
||||
el,
|
||||
int(cx + off_x),
|
||||
int(cy + off_y),
|
||||
)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step: click the found element
|
||||
@@ -218,65 +494,205 @@ class DynamicFlow:
|
||||
def _run_click(self, item_id: str, element: Any) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"click:{item_id}"
|
||||
log.info("[%s] Clicking item '%s'", name, item_id)
|
||||
try:
|
||||
if element is None:
|
||||
raise DynamicFlowError("No element reference from scroll step")
|
||||
|
||||
el: WebElement = element
|
||||
# Scroll element into view and click
|
||||
self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
|
||||
human_delay(0.3, 0.7)
|
||||
try:
|
||||
el.click()
|
||||
except Exception:
|
||||
# Fallback: JS click
|
||||
self.driver.execute_script("arguments[0].click();", el)
|
||||
self._human_click(element, name)
|
||||
|
||||
human_delay(0.8, 1.5)
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "url_after": self.driver.current_url},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.info(
|
||||
"[%s] Click OK — landed on %s — %d ms",
|
||||
name,
|
||||
self.driver.current_url,
|
||||
ms,
|
||||
)
|
||||
return StepResult(
|
||||
name=name,
|
||||
success=True,
|
||||
data={"item_id": item_id, "url_after": self.driver.current_url},
|
||||
duration_ms=ms,
|
||||
)
|
||||
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
ms = int((time.monotonic() - t0) * 1000)
|
||||
log.error("[%s] Failed — %s", name, exc)
|
||||
return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Pagination helper
|
||||
# ------------------------------------------------------------------
|
||||
def _try_paginate(
|
||||
self, step_name: str, sel: str, sel_type: str, wait_ms: int
|
||||
) -> bool:
|
||||
"""
|
||||
Find the next-page button and click it reliably.
|
||||
|
||||
Uses a layered approach rather than full human-mouse simulation because
|
||||
pagination buttons are small and offset-based clicking is fragile:
|
||||
1. Native Selenium click (triggers real browser events)
|
||||
2. JS click() on the element
|
||||
3. Synthetic pointer + mouse + click event dispatch
|
||||
"""
|
||||
log.debug("[%s] Looking for pagination button [%s:%s]", step_name, sel_type, sel)
|
||||
|
||||
# Search all matching elements, not just the first visible one,
|
||||
# in case the button is present but momentarily off-screen.
|
||||
try:
|
||||
candidates = _find_all(self.driver, sel, sel_type)
|
||||
except Exception as exc:
|
||||
log.warning("[%s] Pagination selector failed — %s", step_name, exc)
|
||||
return False
|
||||
|
||||
btn: WebElement | None = None
|
||||
for el in candidates:
|
||||
try:
|
||||
if el.is_displayed() and el.is_enabled():
|
||||
btn = el
|
||||
break
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
if btn is None:
|
||||
log.debug("[%s] Pagination button not found / not interactable", step_name)
|
||||
return False
|
||||
|
||||
log.debug("[%s] Pagination button found — attempting click", step_name)
|
||||
try:
|
||||
self.driver.execute_script(
|
||||
"arguments[0].scrollIntoView({block:'nearest', behavior:'instant'});", btn
|
||||
)
|
||||
time.sleep(random.uniform(0.15, 0.35))
|
||||
|
||||
# Layer 1: native Selenium click
|
||||
try:
|
||||
btn.click()
|
||||
log.debug("[%s] Pagination: native click OK", step_name)
|
||||
except Exception as e1:
|
||||
log.debug("[%s] Pagination: native click failed (%s) — trying JS", step_name, e1)
|
||||
# Layer 2: JS .click()
|
||||
try:
|
||||
self.driver.execute_script("arguments[0].click();", btn)
|
||||
log.debug("[%s] Pagination: JS click OK", step_name)
|
||||
except Exception as e2:
|
||||
log.debug("[%s] Pagination: JS click failed (%s) — dispatching events", step_name, e2)
|
||||
# Layer 3: synthetic pointer + mouse + click events
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
const el = arguments[0];
|
||||
const r = el.getBoundingClientRect();
|
||||
const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
|
||||
const opts = {bubbles: true, cancelable: true, clientX: cx, clientY: cy};
|
||||
['pointerdown','mousedown','pointerup','mouseup','click']
|
||||
.forEach(t => el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, opts)));
|
||||
""",
|
||||
btn,
|
||||
)
|
||||
log.debug("[%s] Pagination: synthetic events dispatched", step_name)
|
||||
|
||||
log.info("[%s] Pagination button clicked — waiting %d ms", step_name, wait_ms)
|
||||
time.sleep(wait_ms / 1000)
|
||||
return True
|
||||
|
||||
except Exception as exc:
|
||||
log.warning("[%s] Pagination click failed — %s", step_name, exc)
|
||||
return False
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
def _build_item_selector(self, item_id: str) -> str:
|
||||
def _build_item_selector(self, item_id: str) -> tuple[str, str]:
|
||||
"""
|
||||
Return (selector, effective_type).
|
||||
|
||||
Priority:
|
||||
1. item_url_template — renders the URL pattern and uses XPath
|
||||
contains(@href, ...) to find the matching <a> element.
|
||||
2. item_selector_type == "a-link" — same href-contains approach but
|
||||
driven by item_selector_template instead.
|
||||
3. CSS / XPath — item_selector_template used verbatim.
|
||||
|
||||
Href-based searches always use XPath so that arbitrary URL characters
|
||||
(colons, brackets, dots, slashes) never break the selector.
|
||||
"""
|
||||
url_tpl: str = self.cfg.get("item_url_template", "").strip()
|
||||
if url_tpl:
|
||||
href = url_tpl.replace("{item_id}", item_id)
|
||||
log.debug("item_url_template resolved to href pattern '%s'", href)
|
||||
return f"//a[contains(@href, '{href}')]", "xpath"
|
||||
|
||||
sel_type: str = self.cfg.get("item_selector_type", "css")
|
||||
template: str = self.cfg.get("item_selector_template", "")
|
||||
|
||||
if sel_type == "a-link":
|
||||
href_fragment = (
|
||||
template.replace("{item_id}", item_id) if template else item_id
|
||||
)
|
||||
return f"//a[contains(@href, '{href_fragment}')]", "xpath"
|
||||
|
||||
if not template:
|
||||
raise DynamicFlowError("item_selector_template is not configured")
|
||||
return template.replace("{item_id}", item_id)
|
||||
return template.replace("{item_id}", item_id), sel_type
|
||||
|
||||
def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
|
||||
def _find_item_visible(
|
||||
self, selector: str, selector_type: str
|
||||
) -> WebElement | None:
|
||||
try:
|
||||
els = _find_all(self.driver, selector, selector_type)
|
||||
for el in els:
|
||||
except Exception as exc:
|
||||
log.debug("Selector error [%s:%s]: %s", selector_type, selector, exc)
|
||||
return None
|
||||
for el in els:
|
||||
try:
|
||||
if el.is_displayed():
|
||||
return el
|
||||
except NoSuchElementException:
|
||||
pass
|
||||
except Exception:
|
||||
continue
|
||||
return None
|
||||
|
||||
def _get_scroll_height(self, container_sel: str) -> int:
|
||||
def _get_scroll_height(
|
||||
self, container_sel: str, container_type: str = "css"
|
||||
) -> int:
|
||||
body_h = int(self.driver.execute_script(
|
||||
"return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)"
|
||||
))
|
||||
if container_sel:
|
||||
try:
|
||||
el = _find(self.driver, container_sel, "css")
|
||||
return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
|
||||
except Exception:
|
||||
pass
|
||||
return int(self.driver.execute_script("return document.body.scrollHeight"))
|
||||
|
||||
def _scroll_down(self, container_sel: str) -> None:
|
||||
if container_sel:
|
||||
try:
|
||||
el = _find(self.driver, container_sel, "css")
|
||||
self.driver.execute_script(
|
||||
"arguments[0].scrollTop = arguments[0].scrollHeight", el
|
||||
el = _find(self.driver, container_sel, container_type)
|
||||
container_h = int(self.driver.execute_script("return arguments[0].scrollHeight", el))
|
||||
return max(body_h, container_h)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Could not read scrollHeight from container [%s:'%s']: %s — using body",
|
||||
container_type, container_sel, exc,
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
|
||||
return body_h
|
||||
|
||||
def _scroll_down(self, container_sel: str, container_type: str = "css") -> None:
|
||||
if container_sel:
|
||||
try:
|
||||
el = _find(self.driver, container_sel, container_type)
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
const el = arguments[0];
|
||||
el.scrollTop = el.scrollHeight;
|
||||
el.dispatchEvent(new Event('scroll', {bubbles: true}));
|
||||
""",
|
||||
el,
|
||||
)
|
||||
except Exception as exc:
|
||||
log.warning(
|
||||
"Could not scroll container [%s:'%s']: %s — falling through to window",
|
||||
container_type, container_sel, exc,
|
||||
)
|
||||
# Always advance the window too — most infinite-scroll triggers
|
||||
# listen to window scroll events, not container scroll events.
|
||||
self.driver.execute_script(
|
||||
"""
|
||||
window.scrollTo(0, document.body.scrollHeight);
|
||||
window.dispatchEvent(new Event('scroll', {bubbles: true}));
|
||||
document.dispatchEvent(new Event('scroll', {bubbles: true}));
|
||||
"""
|
||||
)
|
||||
|
||||
@@ -2,11 +2,9 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from config import config
|
||||
from crawler.driver import human_delay, make_driver
|
||||
from crawler.driver import human_delay
|
||||
|
||||
|
||||
class FreshSessionScenario:
|
||||
@@ -19,8 +17,9 @@ class FreshSessionScenario:
|
||||
explicitly delete all cookies after loading to be safe.
|
||||
"""
|
||||
|
||||
def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
|
||||
def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
|
||||
self.driver = driver
|
||||
self.url = url
|
||||
self.wait = WebDriverWait(driver, timeout)
|
||||
|
||||
def run(self) -> dict[str, object]:
|
||||
@@ -32,7 +31,7 @@ class FreshSessionScenario:
|
||||
"try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
|
||||
)
|
||||
|
||||
self.driver.get(config.target_url)
|
||||
self.driver.get(self.url)
|
||||
human_delay(2.0, 4.0)
|
||||
|
||||
# Wait for the page to reach a ready state
|
||||
|
||||
@@ -10,7 +10,6 @@ from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from config import config
|
||||
from crawler.driver import human_delay
|
||||
|
||||
|
||||
@@ -32,11 +31,13 @@ class OTPLoginScenario:
|
||||
driver: uc.Chrome,
|
||||
phone: str,
|
||||
otp_resolver: Callable[[str], str],
|
||||
url: str,
|
||||
timeout: int = 30,
|
||||
) -> None:
|
||||
self.driver = driver
|
||||
self.phone = phone
|
||||
self.otp_resolver = otp_resolver
|
||||
self.url = url
|
||||
self.wait = WebDriverWait(driver, timeout)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
@@ -50,7 +51,7 @@ class OTPLoginScenario:
|
||||
|
||||
def run(self) -> dict[str, object]:
|
||||
"""Execute the OTP login. Returns a result dict."""
|
||||
self.driver.get(config.target_url)
|
||||
self.driver.get(self.url)
|
||||
human_delay(1.5, 3.0)
|
||||
|
||||
try:
|
||||
|
||||
@@ -0,0 +1,214 @@
|
||||
"""Huey task definitions and shared batch state for the admin panel."""
|
||||
from __future__ import annotations
|
||||
|
||||
import threading
|
||||
import time
|
||||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||||
from typing import Any
|
||||
|
||||
from huey import MemoryHuey
|
||||
|
||||
huey = MemoryHuey("seed", immediate=False)
|
||||
|
||||
# ── Shared state (all accessed under _lock) ───────────────────────────────────
|
||||
_lock = threading.Lock()
|
||||
|
||||
_current_batch_id: str | None = None
|
||||
_current_result: Any = None # Huey Result — kept for revocation
|
||||
|
||||
_batch_meta: dict[str, Any] = {} # batch_id → aggregate metadata
|
||||
_worker_slots: dict[str, dict] = {} # f"{batch_id}:{slot:04d}" → worker info
|
||||
_stop_events: dict[str, threading.Event] = {}
|
||||
|
||||
|
||||
# ── Public helpers ─────────────────────────────────────────────────────────────
|
||||
|
||||
def get_current_batch_id() -> str | None:
|
||||
with _lock:
|
||||
return _current_batch_id
|
||||
|
||||
|
||||
def set_current(batch_id: str, result: Any) -> None:
|
||||
global _current_batch_id, _current_result
|
||||
with _lock:
|
||||
_current_batch_id = batch_id
|
||||
_current_result = result
|
||||
|
||||
|
||||
def stop_current() -> bool:
|
||||
"""Signal the running batch to stop. Returns True if there was something to stop."""
|
||||
with _lock:
|
||||
bid = _current_batch_id
|
||||
result = _current_result
|
||||
|
||||
if bid is None:
|
||||
return False
|
||||
|
||||
# Revoke task if it hasn't started yet (still pending in Huey queue)
|
||||
if result is not None:
|
||||
try:
|
||||
result.revoke(revoke_once=True)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# Signal running workers to stop after their current step
|
||||
ev = _stop_events.get(bid)
|
||||
if ev:
|
||||
ev.set()
|
||||
|
||||
with _lock:
|
||||
if bid in _batch_meta:
|
||||
_batch_meta[bid]["stopping"] = True
|
||||
|
||||
return True
|
||||
|
||||
|
||||
def batch_status() -> dict[str, Any]:
|
||||
with _lock:
|
||||
bid = _current_batch_id
|
||||
if bid is None:
|
||||
return {"batch_id": None, "running": False}
|
||||
|
||||
meta = dict(_batch_meta.get(bid, {}))
|
||||
prefix = f"{bid}:"
|
||||
workers = [dict(v) for k, v in _worker_slots.items() if k.startswith(prefix)]
|
||||
|
||||
meta["batch_id"] = bid
|
||||
meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0))
|
||||
meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running")
|
||||
return meta
|
||||
|
||||
|
||||
# ── Huey task ─────────────────────────────────────────────────────────────────
|
||||
|
||||
@huey.task(name="run_batch")
|
||||
def run_batch_task(
|
||||
batch_id: str,
|
||||
cfg: dict[str, Any],
|
||||
workers: int,
|
||||
total_runs: int,
|
||||
headless: bool,
|
||||
stagger_ms: int,
|
||||
) -> dict[str, Any]:
|
||||
import asyncio
|
||||
|
||||
from crawler.driver import driver_session
|
||||
from crawler.dynamic_flow import DynamicFlow
|
||||
from crawler.scenarios.fresh_session import FreshSessionScenario
|
||||
from db.repositories.runs import runs_repo
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
stop_ev = threading.Event()
|
||||
with _lock:
|
||||
_stop_events[batch_id] = stop_ev
|
||||
_batch_meta[batch_id] = {
|
||||
"running": True,
|
||||
"stopping": False,
|
||||
"total": total_runs,
|
||||
"completed": 0,
|
||||
"succeeded": 0,
|
||||
"failed": 0,
|
||||
"configured_workers": workers,
|
||||
"started_at": time.time(),
|
||||
"finished_at": None,
|
||||
"error": None,
|
||||
}
|
||||
|
||||
def _run_one(slot: int) -> bool:
|
||||
slot_key = f"{batch_id}:{slot:04d}"
|
||||
|
||||
if stop_ev.is_set():
|
||||
with _lock:
|
||||
_worker_slots[slot_key] = {
|
||||
"slot": slot, "status": "skipped",
|
||||
"started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped",
|
||||
}
|
||||
return False
|
||||
|
||||
with _lock:
|
||||
_worker_slots[slot_key] = {
|
||||
"slot": slot, "status": "running",
|
||||
"started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None,
|
||||
}
|
||||
|
||||
t0 = time.monotonic()
|
||||
identifier = f"batch-{batch_id[:8]}-{slot:04d}"
|
||||
log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
|
||||
|
||||
try:
|
||||
target_url: str = cfg.get("target_url") or ""
|
||||
with driver_session(headless=headless) as driver:
|
||||
FreshSessionScenario(driver, target_url).run()
|
||||
result = DynamicFlow(driver, cfg).run() if cfg else None
|
||||
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
success = result.success if result else True
|
||||
reason = result.failure_reason if result else None
|
||||
steps = [
|
||||
{"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
|
||||
for s in (result.steps if result else [])
|
||||
]
|
||||
asyncio.run(runs_repo.insert("batch", identifier, success, reason, steps, total_ms))
|
||||
|
||||
status = "ok" if success else "failed"
|
||||
with _lock:
|
||||
_worker_slots[slot_key].update({
|
||||
"status": status, "finished_at": time.time(), "duration_ms": total_ms,
|
||||
})
|
||||
log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms)
|
||||
return success
|
||||
|
||||
except Exception as exc:
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
|
||||
with _lock:
|
||||
_worker_slots[slot_key].update({
|
||||
"status": "error", "error": str(exc),
|
||||
"finished_at": time.time(), "duration_ms": total_ms,
|
||||
})
|
||||
log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc)
|
||||
return False
|
||||
|
||||
succeeded = 0
|
||||
failed = 0
|
||||
|
||||
try:
|
||||
with ThreadPoolExecutor(
|
||||
max_workers=workers,
|
||||
thread_name_prefix=f"batch-{batch_id[:8]}",
|
||||
) as pool:
|
||||
futures = []
|
||||
for i in range(total_runs):
|
||||
if stop_ev.is_set():
|
||||
break
|
||||
futures.append(pool.submit(_run_one, i))
|
||||
if stagger_ms > 0 and i < total_runs - 1:
|
||||
if stop_ev.wait(timeout=stagger_ms / 1000):
|
||||
break # stop was signalled during the stagger delay
|
||||
|
||||
for fut in as_completed(futures):
|
||||
try:
|
||||
ok = fut.result()
|
||||
except Exception:
|
||||
ok = False
|
||||
if ok:
|
||||
succeeded += 1
|
||||
else:
|
||||
failed += 1
|
||||
with _lock:
|
||||
_batch_meta[batch_id]["completed"] = succeeded + failed
|
||||
_batch_meta[batch_id]["succeeded"] = succeeded
|
||||
_batch_meta[batch_id]["failed"] = failed
|
||||
|
||||
except Exception as exc:
|
||||
with _lock:
|
||||
_batch_meta[batch_id]["error"] = str(exc)
|
||||
|
||||
finally:
|
||||
with _lock:
|
||||
_batch_meta[batch_id]["running"] = False
|
||||
_batch_meta[batch_id]["finished_at"] = time.time()
|
||||
|
||||
return {"batch_id": batch_id, "succeeded": succeeded, "failed": failed}
|
||||
+30
-3
@@ -37,14 +37,22 @@ CREATE TABLE IF NOT EXISTS flow_configs (
|
||||
search_box_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_submit_selector TEXT NOT NULL DEFAULT '',
|
||||
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_results_selector TEXT NOT NULL DEFAULT '',
|
||||
scroll_container_selector TEXT NOT NULL DEFAULT '',
|
||||
search_results_selector TEXT NOT NULL DEFAULT '',
|
||||
search_results_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
scroll_container_selector TEXT NOT NULL DEFAULT '',
|
||||
scroll_container_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
max_scrolls INTEGER NOT NULL DEFAULT 30,
|
||||
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
|
||||
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000,
|
||||
pagination_selector TEXT NOT NULL DEFAULT '',
|
||||
pagination_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
max_pages INTEGER NOT NULL DEFAULT 10,
|
||||
pagination_wait_ms INTEGER NOT NULL DEFAULT 1500,
|
||||
item_selector_template TEXT NOT NULL DEFAULT '',
|
||||
item_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
target_item_ids_json TEXT NOT NULL DEFAULT '[]'
|
||||
item_url_template TEXT NOT NULL DEFAULT '',
|
||||
target_item_ids_json TEXT NOT NULL DEFAULT '[]',
|
||||
target_url TEXT NOT NULL DEFAULT ''
|
||||
);
|
||||
"""
|
||||
|
||||
@@ -55,10 +63,29 @@ def init_db() -> None:
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.executescript(_DDL)
|
||||
# Additive migrations for columns added after initial release
|
||||
_migrate(conn)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
def _migrate(conn: sqlite3.Connection) -> None:
|
||||
existing = {row[1] for row in conn.execute("PRAGMA table_info(flow_configs)")}
|
||||
additions = {
|
||||
"search_results_selector_type": "TEXT NOT NULL DEFAULT 'css'",
|
||||
"item_url_template": "TEXT NOT NULL DEFAULT ''",
|
||||
"scroll_container_selector_type": "TEXT NOT NULL DEFAULT 'css'",
|
||||
"pagination_selector": "TEXT NOT NULL DEFAULT ''",
|
||||
"pagination_selector_type": "TEXT NOT NULL DEFAULT 'css'",
|
||||
"max_pages": "INTEGER NOT NULL DEFAULT 10",
|
||||
"pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
|
||||
"target_url": "TEXT NOT NULL DEFAULT ''",
|
||||
}
|
||||
for col, definition in additions.items():
|
||||
if col not in existing:
|
||||
conn.execute(f"ALTER TABLE flow_configs ADD COLUMN {col} {definition}")
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_db() -> AsyncGenerator[aiosqlite.Connection, None]:
|
||||
"""Async context manager — yields an open, Row-factory-enabled connection."""
|
||||
|
||||
@@ -11,9 +11,13 @@ FlowConfigRow = dict[str, Any]
|
||||
_FIELDS = [
|
||||
"name", "is_active",
|
||||
"search_texts_json", "search_box_selector", "search_box_selector_type",
|
||||
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
|
||||
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
||||
"item_selector_template", "item_selector_type", "target_item_ids_json",
|
||||
"search_submit_selector", "search_submit_selector_type",
|
||||
"search_results_selector", "search_results_selector_type",
|
||||
"scroll_container_selector", "scroll_container_selector_type",
|
||||
"max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
||||
"pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms",
|
||||
"item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json",
|
||||
"target_url",
|
||||
]
|
||||
|
||||
|
||||
|
||||
@@ -0,0 +1,109 @@
|
||||
"""Central logging configuration for the Seed project.
|
||||
|
||||
Usage anywhere in the codebase:
|
||||
from logger import get_logger
|
||||
log = get_logger(__name__)
|
||||
log.info("message")
|
||||
log.warning("watch out")
|
||||
log.error("something broke")
|
||||
|
||||
Environment variables:
|
||||
LOG_LEVEL — DEBUG | INFO | WARNING | ERROR | CRITICAL (default: INFO)
|
||||
LOG_FILE — path to write a rotating log file (optional)
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import logging
|
||||
import logging.handlers
|
||||
import os
|
||||
import sys
|
||||
from typing import Final
|
||||
|
||||
_LEVEL_MAP: Final = {
|
||||
"DEBUG": logging.DEBUG,
|
||||
"INFO": logging.INFO,
|
||||
"WARNING": logging.WARNING,
|
||||
"ERROR": logging.ERROR,
|
||||
"CRITICAL": logging.CRITICAL,
|
||||
}
|
||||
|
||||
_LOG_LEVEL: int = _LEVEL_MAP.get(os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
|
||||
_LOG_FILE: str | None = os.getenv("LOG_FILE")
|
||||
|
||||
# ── ANSI colour codes (console only) ─────────────────────────────────────────
|
||||
_RESET = "\033[0m"
|
||||
_BOLD = "\033[1m"
|
||||
_GREY = "\033[90m"
|
||||
_CYAN = "\033[96m"
|
||||
_YELLOW = "\033[93m"
|
||||
_RED = "\033[91m"
|
||||
_BRED = "\033[1;91m"
|
||||
|
||||
_LEVEL_COLOURS: Final[dict[int, str]] = {
|
||||
logging.DEBUG: _GREY,
|
||||
logging.INFO: _CYAN,
|
||||
logging.WARNING: _YELLOW,
|
||||
logging.ERROR: _RED,
|
||||
logging.CRITICAL: _BRED,
|
||||
}
|
||||
|
||||
|
||||
class _ColourFormatter(logging.Formatter):
|
||||
_FMT = "{colour}{level:<8}{reset} {grey}{asctime} {name}{reset} {msg}"
|
||||
|
||||
def format(self, record: logging.LogRecord) -> str:
|
||||
colour = _LEVEL_COLOURS.get(record.levelno, "")
|
||||
level = record.levelname
|
||||
grey = _GREY if sys.stderr.isatty() else ""
|
||||
reset = _RESET if sys.stderr.isatty() else ""
|
||||
col = colour if sys.stderr.isatty() else ""
|
||||
|
||||
self.datefmt = "%H:%M:%S"
|
||||
base = super().format(record)
|
||||
return self._FMT.format(
|
||||
colour=col, level=level, reset=reset,
|
||||
grey=grey, asctime=self.formatTime(record, self.datefmt),
|
||||
name=record.name, msg=record.getMessage(),
|
||||
) + (f"\n{record.exc_text}" if record.exc_info and self.formatException(record.exc_info) else "")
|
||||
|
||||
|
||||
_FILE_FMT = logging.Formatter(
|
||||
fmt="%(asctime)s %(levelname)-8s %(name)s %(message)s",
|
||||
datefmt="%Y-%m-%dT%H:%M:%S",
|
||||
)
|
||||
|
||||
|
||||
def _build_root_logger() -> logging.Logger:
|
||||
root = logging.getLogger("seed")
|
||||
root.setLevel(_LOG_LEVEL)
|
||||
|
||||
if root.handlers:
|
||||
return root # already configured (e.g. imported twice)
|
||||
|
||||
# Console handler
|
||||
console = logging.StreamHandler(sys.stderr)
|
||||
console.setLevel(_LOG_LEVEL)
|
||||
console.setFormatter(_ColourFormatter())
|
||||
root.addHandler(console)
|
||||
|
||||
# Optional rotating file handler
|
||||
if _LOG_FILE:
|
||||
fh = logging.handlers.RotatingFileHandler(
|
||||
_LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8"
|
||||
)
|
||||
fh.setLevel(_LOG_LEVEL)
|
||||
fh.setFormatter(_FILE_FMT)
|
||||
root.addHandler(fh)
|
||||
|
||||
root.propagate = False
|
||||
return root
|
||||
|
||||
|
||||
_root = _build_root_logger()
|
||||
|
||||
|
||||
def get_logger(name: str) -> logging.Logger:
|
||||
"""Return a child logger namespaced under 'seed'."""
|
||||
if name.startswith("seed.") or name == "seed":
|
||||
return logging.getLogger(name)
|
||||
return logging.getLogger(f"seed.{name}")
|
||||
@@ -8,15 +8,18 @@ import time
|
||||
|
||||
import uvicorn
|
||||
|
||||
from config import config
|
||||
from db import flow_config_repo, init_db
|
||||
from db.repositories.runs import runs_repo
|
||||
from config import config
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def _load_active_flow() -> dict[str, object]:
|
||||
cfg = asyncio.run(flow_config_repo.get_active())
|
||||
if cfg is None:
|
||||
print("[WARN] No active flow config — running without dynamic flow steps.", file=sys.stderr)
|
||||
log.warning("No active flow config — running without dynamic flow steps.")
|
||||
return cfg or {}
|
||||
|
||||
|
||||
@@ -32,7 +35,7 @@ def _record(
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
if exc is not None:
|
||||
asyncio.run(runs_repo.insert(scenario, identifier, False, str(exc), [], total_ms))
|
||||
print(f"[FAIL] {identifier}: {exc}", file=sys.stderr)
|
||||
log.error("%s failed: %s", identifier, exc)
|
||||
return
|
||||
|
||||
if isinstance(flow_result, DynamicFlowResult):
|
||||
@@ -44,12 +47,13 @@ def _record(
|
||||
scenario, identifier, flow_result.success,
|
||||
flow_result.failure_reason, steps_data, total_ms,
|
||||
))
|
||||
status = "OK" if flow_result.success else f"FAIL ({flow_result.failure_reason})"
|
||||
if flow_result.success:
|
||||
log.info("%s OK — %d ms", identifier, total_ms)
|
||||
else:
|
||||
log.error("%s FAIL (%s) — %d ms", identifier, flow_result.failure_reason, total_ms)
|
||||
else:
|
||||
asyncio.run(runs_repo.insert(scenario, identifier, True, None, [], total_ms))
|
||||
status = "OK (no flow config)"
|
||||
|
||||
print(f"[{status}] {identifier} — {total_ms} ms")
|
||||
log.info("%s OK (no flow config) — %d ms", identifier, total_ms)
|
||||
|
||||
|
||||
def run_otp(phone: str) -> None:
|
||||
@@ -58,14 +62,14 @@ def run_otp(phone: str) -> None:
|
||||
from crawler.scenarios.otp_login import OTPLoginScenario
|
||||
|
||||
def otp_resolver(p: str) -> str:
|
||||
# TODO: wire up your SMS gateway / webhook here.
|
||||
raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
|
||||
|
||||
flow_cfg = _load_active_flow()
|
||||
target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with driver_session() as driver:
|
||||
OTPLoginScenario(driver, phone, otp_resolver).run()
|
||||
OTPLoginScenario(driver, phone, otp_resolver, target_url).run()
|
||||
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
|
||||
except Exception as exc:
|
||||
_record("otp", phone, None, exc, t0)
|
||||
@@ -79,10 +83,11 @@ def run_fresh() -> None:
|
||||
from crawler.scenarios.fresh_session import FreshSessionScenario
|
||||
|
||||
flow_cfg = _load_active_flow()
|
||||
target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with driver_session() as driver:
|
||||
FreshSessionScenario(driver).run()
|
||||
FreshSessionScenario(driver, target_url).run()
|
||||
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
|
||||
except Exception as exc:
|
||||
_record("fresh", "fresh", None, exc, t0)
|
||||
@@ -90,6 +95,34 @@ def run_fresh() -> None:
|
||||
_record("fresh", "fresh", result, None, t0)
|
||||
|
||||
|
||||
def run_batch(workers: int, total_runs: int, stagger_ms: int, headless: bool) -> None:
|
||||
from crawler.batch import run_batch as _run_batch
|
||||
|
||||
flow_cfg = _load_active_flow()
|
||||
if not flow_cfg:
|
||||
log.error("No active flow config — activate one in the admin panel first.")
|
||||
sys.exit(1)
|
||||
|
||||
completed_count = [0]
|
||||
|
||||
def _progress(completed: int, succeeded: int, failed: int) -> None:
|
||||
completed_count[0] = completed
|
||||
log.info("Progress: %d/%d (ok=%d fail=%d)", completed, total_runs, succeeded, failed)
|
||||
|
||||
result = _run_batch(
|
||||
cfg=flow_cfg,
|
||||
workers=workers,
|
||||
total_runs=total_runs,
|
||||
headless=headless,
|
||||
stagger_ms=stagger_ms,
|
||||
on_progress=_progress,
|
||||
)
|
||||
log.info(
|
||||
"Batch complete — succeeded=%d/%d failed=%d",
|
||||
result["succeeded"], result["total"], result["failed"],
|
||||
)
|
||||
|
||||
|
||||
def run_admin() -> None:
|
||||
init_db()
|
||||
from admin.main import app
|
||||
@@ -107,6 +140,12 @@ def main() -> None:
|
||||
|
||||
sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
|
||||
|
||||
batch_p = sub.add_parser("batch", help="Run many parallel fresh-session runners")
|
||||
batch_p.add_argument("--workers", type=int, default=3, help="Parallel Chrome instances (default: 3)")
|
||||
batch_p.add_argument("--runs", type=int, default=10, help="Total runs to complete (default: 10)")
|
||||
batch_p.add_argument("--stagger", type=int, default=500, help="Ms delay between worker launches (default: 500)")
|
||||
batch_p.add_argument("--no-headless", dest="headless", action="store_false", help="Show browser windows")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.cmd == "admin":
|
||||
@@ -114,12 +153,19 @@ def main() -> None:
|
||||
elif args.cmd == "otp":
|
||||
phones = [args.phone] if args.phone else config.phone_numbers
|
||||
if not phones:
|
||||
print("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone", file=sys.stderr)
|
||||
log.error("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone")
|
||||
sys.exit(1)
|
||||
for phone in phones:
|
||||
run_otp(phone)
|
||||
elif args.cmd == "fresh":
|
||||
run_fresh()
|
||||
elif args.cmd == "batch":
|
||||
run_batch(
|
||||
workers=args.workers,
|
||||
total_runs=args.runs,
|
||||
stagger_ms=args.stagger,
|
||||
headless=args.headless,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -14,6 +14,7 @@ dependencies = [
|
||||
"pydantic>=2.7.0",
|
||||
"python-dotenv>=1.0.1",
|
||||
"setuptools>=70.0.0",
|
||||
"huey>=2.5.0",
|
||||
]
|
||||
|
||||
[tool.pyright]
|
||||
|
||||
@@ -0,0 +1,106 @@
|
||||
"""
|
||||
Download and patch ChromeDriver once. Run via: make patch-driver
|
||||
|
||||
Uses the official Chrome for Testing API to fetch the exact ChromeDriver
|
||||
version matching the installed Chrome, then patches it with undetected_chromedriver.
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import io
|
||||
import shutil
|
||||
import stat
|
||||
import subprocess
|
||||
import sys
|
||||
import urllib.request
|
||||
import zipfile
|
||||
from pathlib import Path
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
|
||||
from config import config
|
||||
from logger import get_logger
|
||||
|
||||
log = get_logger(__name__)
|
||||
|
||||
|
||||
def _get_chrome_full_version() -> tuple[int, str]:
|
||||
candidates = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
|
||||
if config.chrome_binary:
|
||||
candidates.insert(0, config.chrome_binary)
|
||||
|
||||
for binary in candidates:
|
||||
if not shutil.which(binary) and not Path(binary).exists():
|
||||
continue
|
||||
try:
|
||||
out = subprocess.check_output(
|
||||
[binary, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
|
||||
).strip()
|
||||
full = out.split()[-1]
|
||||
major = int(full.split(".")[0])
|
||||
log.info("Detected Chrome %s via '%s'", full, binary)
|
||||
return major, full
|
||||
except Exception:
|
||||
continue
|
||||
|
||||
log.critical("Could not detect Chrome version. Set CHROME_BINARY in .env")
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _latest_chromedriver_version(major: int) -> str:
|
||||
url = f"https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_{major}"
|
||||
try:
|
||||
with urllib.request.urlopen(url, timeout=15) as resp:
|
||||
version = resp.read().decode().strip()
|
||||
log.info("Latest ChromeDriver for Chrome %d: %s", major, version)
|
||||
return version
|
||||
except Exception as exc:
|
||||
log.critical("Could not fetch ChromeDriver version for Chrome %d: %s", major, exc)
|
||||
sys.exit(1)
|
||||
|
||||
|
||||
def _download_chromedriver(version: str, dest: Path) -> None:
|
||||
zip_url = (
|
||||
f"https://storage.googleapis.com/chrome-for-testing-public"
|
||||
f"/{version}/linux64/chromedriver-linux64.zip"
|
||||
)
|
||||
log.info("Downloading ChromeDriver %s …", version)
|
||||
try:
|
||||
with urllib.request.urlopen(zip_url, timeout=60) as resp:
|
||||
data = resp.read()
|
||||
except Exception as exc:
|
||||
log.critical("Download failed: %s", exc)
|
||||
sys.exit(1)
|
||||
|
||||
with zipfile.ZipFile(io.BytesIO(data)) as zf:
|
||||
binary_name = next(
|
||||
n for n in zf.namelist() if n.endswith("/chromedriver") or n == "chromedriver"
|
||||
)
|
||||
dest.write_bytes(zf.read(binary_name))
|
||||
|
||||
dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
|
||||
log.info("Extracted to %s", dest)
|
||||
|
||||
|
||||
def _patch(dest: Path) -> None:
|
||||
log.info("Patching binary …")
|
||||
patcher = uc.Patcher(executable_path=str(dest))
|
||||
patcher.patch_exe()
|
||||
log.info("Patch applied successfully")
|
||||
|
||||
|
||||
def main() -> None:
|
||||
dest = Path(config.chromedriver_path)
|
||||
dest.parent.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
major, _ = _get_chrome_full_version()
|
||||
cd_version = _latest_chromedriver_version(major)
|
||||
|
||||
_download_chromedriver(cd_version, dest)
|
||||
_patch(dest)
|
||||
|
||||
log.info("ChromeDriver %s ready at %s", cd_version, dest)
|
||||
log.info("Re-run 'make patch-driver' only when Chrome updates to a new major version.")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
Reference in New Issue
Block a user