This commit is contained in:
2026-06-15 19:42:03 +03:30
commit abff09dc0c
17 changed files with 1800 additions and 0 deletions
+22
View File
@@ -0,0 +1,22 @@
# Copy to .env and fill in
TARGET_URL=https://example.com
# Admin panel credentials
ADMIN_USERNAME=admin
ADMIN_PASSWORD=changeme
ADMIN_HOST=127.0.0.1
ADMIN_PORT=8000
# SQLite DB path
DB_PATH=data/tracker.db
# Chrome: set to false to watch the browser window
HEADLESS=true
# CHROME_BINARY=/usr/bin/chromium # optional: explicit Chrome path
# Scenario 1 — comma-separated phone numbers
PHONE_NUMBERS=+989100000001,+989100000002,+989100000003
# Flow steps — comma-separated list matching keys in STEP_REGISTRY
FLOW_STEPS=step_home,step_browse
+6
View File
@@ -0,0 +1,6 @@
.venv/
__pycache__/
*.pyc
.env
data/
*.db
View File
+28
View File
@@ -0,0 +1,28 @@
"""FastAPI admin application."""
from __future__ import annotations
from pathlib import Path
from fastapi import FastAPI
from fastapi.responses import FileResponse
from fastapi.staticfiles import StaticFiles
from admin.models import init_db
from admin.routes import router
app = FastAPI(title="Seed Admin", docs_url=None, redoc_url=None)
app.include_router(router)
_STATIC = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
@app.on_event("startup")
def on_startup() -> None:
init_db()
@app.get("/")
def dashboard() -> FileResponse:
return FileResponse(str(_STATIC / "dashboard.html"))
+241
View File
@@ -0,0 +1,241 @@
"""SQLite schema and DB helpers (aiosqlite)."""
from __future__ import annotations
import json
import os
import sqlite3
from pathlib import Path
from typing import Any
DB_PATH = os.getenv("DB_PATH", "data/tracker.db")
CREATE_DDL = """
CREATE TABLE IF NOT EXISTS runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
scenario TEXT NOT NULL,
identifier TEXT NOT NULL,
success INTEGER NOT NULL,
failure_reason TEXT,
steps_json TEXT NOT NULL DEFAULT '[]',
total_ms INTEGER NOT NULL DEFAULT 0
);
CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario);
CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at);
CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier);
-- Dynamic flow configurations editable from the admin panel
CREATE TABLE IF NOT EXISTS flow_configs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
name TEXT NOT NULL,
is_active INTEGER NOT NULL DEFAULT 0,
-- Search step
search_texts_json TEXT NOT NULL DEFAULT '[]', -- JSON string[]
search_box_selector TEXT NOT NULL DEFAULT '',
search_box_selector_type TEXT NOT NULL DEFAULT 'css', -- 'css' | 'xpath'
search_submit_selector TEXT NOT NULL DEFAULT '', -- optional — leave blank to use Enter key
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
search_results_selector TEXT NOT NULL DEFAULT '', -- container that appears after results load
-- Scroll step
scroll_container_selector TEXT NOT NULL DEFAULT '', -- element to scroll; blank = window
max_scrolls INTEGER NOT NULL DEFAULT 30,
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000, -- bail if DOM height unchanged after this
-- Item selection
item_selector_template TEXT NOT NULL DEFAULT '',
-- Template may contain {item_id} placeholder, e.g. "[data-id='{item_id}']"
-- or a plain selector like ".result-card" (picks first match)
item_selector_type TEXT NOT NULL DEFAULT 'css',
target_item_ids_json TEXT NOT NULL DEFAULT '[]' -- JSON string[] — one click per id
);
"""
def init_db() -> None:
"""Create the DB file and tables (sync, called at startup)."""
path = Path(DB_PATH)
path.parent.mkdir(parents=True, exist_ok=True)
conn = sqlite3.connect(str(path))
conn.executescript(CREATE_DDL)
conn.commit()
conn.close()
# ---------------------------------------------------------------------------
# Async helpers (used by FastAPI routes via aiosqlite)
# ---------------------------------------------------------------------------
async def insert_run(
scenario: str,
identifier: str,
success: bool,
failure_reason: str | None,
steps: list[dict[str, object]],
total_ms: int,
) -> int:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
cur = await db.execute(
"""INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms)
VALUES (?, ?, ?, ?, ?, ?)""",
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
)
await db.commit()
return int(cur.lastrowid or 0)
# ---------------------------------------------------------------------------
# flow_configs CRUD
# ---------------------------------------------------------------------------
FlowConfigRow = dict[str, Any]
async def list_flow_configs() -> list[FlowConfigRow]:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
db.row_factory = aiosqlite.Row
rows = await (await db.execute(
"SELECT * FROM flow_configs ORDER BY id DESC"
)).fetchall()
return [_parse_flow_row(dict(r)) for r in rows]
async def get_flow_config(cfg_id: int) -> FlowConfigRow | None:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
db.row_factory = aiosqlite.Row
row = await (await db.execute(
"SELECT * FROM flow_configs WHERE id=?", (cfg_id,)
)).fetchone()
return _parse_flow_row(dict(row)) if row else None
async def get_active_flow_config() -> FlowConfigRow | None:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
db.row_factory = aiosqlite.Row
row = await (await db.execute(
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1"
)).fetchone()
return _parse_flow_row(dict(row)) if row else None
async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int:
import aiosqlite
fields = [
"name", "is_active",
"search_texts_json", "search_box_selector", "search_box_selector_type",
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
"item_selector_template", "item_selector_type", "target_item_ids_json",
]
# Serialize list fields
for key in ("search_texts_json", "target_item_ids_json"):
if key in data and isinstance(data[key], list):
data[key] = json.dumps(data[key])
async with aiosqlite.connect(DB_PATH) as db:
if data.get("is_active"):
await db.execute("UPDATE flow_configs SET is_active=0")
if cfg_id is None:
placeholders = ", ".join("?" for _ in fields)
cols = ", ".join(fields)
values = [data.get(f) for f in fields]
cur = await db.execute(
f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values
)
row_id = int(cur.lastrowid or 0)
else:
set_clause = ", ".join(f"{f}=?" for f in fields)
values_u = [data.get(f) for f in fields] + [cfg_id]
await db.execute(f"UPDATE flow_configs SET {set_clause} WHERE id=?", values_u)
row_id = cfg_id
await db.commit()
return row_id
async def delete_flow_config(cfg_id: int) -> None:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
await db.commit()
async def activate_flow_config(cfg_id: int) -> None:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
await db.execute("UPDATE flow_configs SET is_active=0")
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
await db.commit()
def _parse_flow_row(row: dict[str, Any]) -> FlowConfigRow:
for key in ("search_texts_json", "target_item_ids_json"):
if key in row and isinstance(row[key], str):
try:
row[key] = json.loads(row[key])
except (json.JSONDecodeError, TypeError):
row[key] = []
return row
async def get_stats() -> dict[str, object]:
import aiosqlite
async with aiosqlite.connect(DB_PATH) as db:
db.row_factory = aiosqlite.Row
total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0]
unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0]
successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0]
failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0]
failure_reasons_raw = await (
await db.execute(
"""SELECT failure_reason, COUNT(*) as cnt
FROM runs WHERE success=0 AND failure_reason IS NOT NULL
GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10"""
)
).fetchall()
failure_reasons = [{"reason": r["failure_reason"], "count": r["cnt"]} for r in failure_reasons_raw]
scenario_breakdown_raw = await (
await db.execute(
"""SELECT scenario, COUNT(*) as total,
SUM(success) as ok,
COUNT(*) - SUM(success) as fail
FROM runs GROUP BY scenario"""
)
).fetchall()
scenario_breakdown = [
{"scenario": r["scenario"], "total": r["total"], "success": r["ok"], "failure": r["fail"]}
for r in scenario_breakdown_raw
]
recent_raw = await (
await db.execute(
"""SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms
FROM runs ORDER BY id DESC LIMIT 50"""
)
).fetchall()
recent = [dict(r) for r in recent_raw]
return {
"total_runs": total,
"unique_identifiers": unique_ids,
"successes": successes,
"failures": failures,
"success_rate": round(successes / total * 100, 1) if total else 0.0,
"failure_reasons": failure_reasons,
"scenario_breakdown": scenario_breakdown,
"recent_runs": recent,
}
+135
View File
@@ -0,0 +1,135 @@
"""FastAPI routes — dashboard API + auth."""
from __future__ import annotations
import secrets
from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel
from admin.models import (
activate_flow_config,
delete_flow_config,
get_flow_config,
get_stats,
list_flow_configs,
upsert_flow_config,
)
from config import config
router = APIRouter()
security = HTTPBasic()
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
if not (ok_user and ok_pass):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
detail="Invalid credentials",
headers={"WWW-Authenticate": "Basic"},
)
return credentials.username
Auth = Annotated[str, Depends(require_auth)]
# ---------------------------------------------------------------------------
# Stats
# ---------------------------------------------------------------------------
@router.get("/api/stats")
async def stats(_: Auth) -> dict[str, object]:
return await get_stats()
# ---------------------------------------------------------------------------
# Flow configs
# ---------------------------------------------------------------------------
class FlowConfigIn(BaseModel):
name: str
is_active: bool = False
search_texts: list[str] = []
search_box_selector: str = ""
search_box_selector_type: str = "css"
search_submit_selector: str = ""
search_submit_selector_type: str = "css"
search_results_selector: str = ""
scroll_container_selector: str = ""
max_scrolls: int = 30
scroll_pause_ms: int = 1200
no_new_content_timeout_ms: int = 3000
item_selector_template: str = ""
item_selector_type: str = "css"
target_item_ids: list[str] = []
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
return {
"name": body.name,
"is_active": int(body.is_active),
"search_texts_json": body.search_texts,
"search_box_selector": body.search_box_selector,
"search_box_selector_type": body.search_box_selector_type,
"search_submit_selector": body.search_submit_selector,
"search_submit_selector_type": body.search_submit_selector_type,
"search_results_selector": body.search_results_selector,
"scroll_container_selector": body.scroll_container_selector,
"max_scrolls": body.max_scrolls,
"scroll_pause_ms": body.scroll_pause_ms,
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
"item_selector_template": body.item_selector_template,
"item_selector_type": body.item_selector_type,
"target_item_ids_json": body.target_item_ids,
}
@router.get("/api/flow-configs")
async def list_configs(_: Auth) -> list[dict[str, object]]:
return await list_flow_configs() # type: ignore[return-value]
@router.get("/api/flow-configs/{cfg_id}")
async def get_config(cfg_id: int, _: Auth) -> dict[str, object]:
row = await get_flow_config(cfg_id)
if not row:
raise HTTPException(status_code=404, detail="Not found")
return row # type: ignore[return-value]
@router.post("/api/flow-configs", status_code=201)
async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]:
new_id = await upsert_flow_config(_to_db_dict(body))
row = await get_flow_config(new_id)
return row or {} # type: ignore[return-value]
@router.put("/api/flow-configs/{cfg_id}")
async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]:
existing = await get_flow_config(cfg_id)
if not existing:
raise HTTPException(status_code=404, detail="Not found")
await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id)
row = await get_flow_config(cfg_id)
return row or {} # type: ignore[return-value]
@router.post("/api/flow-configs/{cfg_id}/activate", status_code=200)
async def activate_config(cfg_id: int, _: Auth) -> dict[str, object]:
existing = await get_flow_config(cfg_id)
if not existing:
raise HTTPException(status_code=404, detail="Not found")
await activate_flow_config(cfg_id)
return {"activated": cfg_id}
@router.delete("/api/flow-configs/{cfg_id}", status_code=204)
async def delete_config(cfg_id: int, _: Auth) -> None:
existing = await get_flow_config(cfg_id)
if not existing:
raise HTTPException(status_code=404, detail="Not found")
await delete_flow_config(cfg_id)
+558
View File
@@ -0,0 +1,558 @@
<!DOCTYPE html>
<html lang="en">
<head>
<meta charset="UTF-8" />
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
<title>Seed — Admin</title>
<style>
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
:root {
--bg: #0f1117; --surface: #1a1d27; --surface2: #222536;
--border: #2a2d3e; --text: #e2e8f0; --muted: #8892a4;
--accent: #6366f1; --green: #22c55e; --red: #ef4444; --yellow: #eab308;
}
body { background: var(--bg); color: var(--text); font-family: 'Inter', system-ui, sans-serif; font-size: 14px; }
/* ── Header ─────────────────────────────────── */
header {
display: flex; align-items: center; justify-content: space-between;
padding: 14px 24px; border-bottom: 1px solid var(--border); background: var(--surface);
position: sticky; top: 0; z-index: 50;
}
header h1 { font-size: 17px; font-weight: 600; }
.badge { background: var(--accent); color: #fff; border-radius: 4px; padding: 2px 8px; font-size: 11px; font-weight: 600; }
#last-updated { color: var(--muted); font-size: 12px; }
/* ── Tabs ────────────────────────────────────── */
.tabs { display: flex; gap: 2px; padding: 0 24px; background: var(--surface); border-bottom: 1px solid var(--border); }
.tab { padding: 11px 18px; cursor: pointer; font-size: 13px; font-weight: 500; color: var(--muted); border-bottom: 2px solid transparent; transition: color .15s; }
.tab:hover { color: var(--text); }
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
.tab-panel { display: none; }
.tab-panel.active { display: block; }
main { padding: 24px; max-width: 1400px; margin: 0 auto; }
/* ── Cards ───────────────────────────────────── */
.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 16px; margin-bottom: 28px; }
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 18px 16px; }
.card .label { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .5px; margin-bottom: 8px; }
.card .value { font-size: 30px; font-weight: 700; line-height: 1; }
.card .value.green { color: var(--green); } .card .value.red { color: var(--red); } .card .value.accent { color: var(--accent); }
.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 28px; }
@media (max-width: 900px) { .grid-2 { grid-template-columns: 1fr; } }
.panel { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 20px; }
.section-title { font-size: 11px; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: .5px; margin-bottom: 14px; }
/* ── Tables ──────────────────────────────────── */
table { width: 100%; border-collapse: collapse; }
th { text-align: left; color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .5px; padding: 8px 10px; border-bottom: 1px solid var(--border); }
td { padding: 9px 10px; border-bottom: 1px solid var(--border); vertical-align: middle; }
tr:last-child td { border-bottom: none; }
tr:hover td { background: rgba(255,255,255,.02); }
.pill { display: inline-block; border-radius: 999px; padding: 2px 10px; font-size: 11px; font-weight: 600; }
.pill.ok { background: rgba(34,197,94,.15); color: var(--green); }
.pill.err { background: rgba(239,68,68,.15); color: var(--red); }
.pill.active-badge { background: rgba(99,102,241,.2); color: var(--accent); }
.reason-row { display: flex; justify-content: space-between; align-items: center; padding: 8px 0; border-bottom: 1px solid var(--border); }
.reason-row:last-child { border-bottom: none; }
.reason-text { color: var(--text); font-size: 13px; flex: 1; margin-right: 12px; word-break: break-all; }
.reason-count { color: var(--red); font-weight: 700; }
.breakdown-row { display: flex; align-items: center; gap: 10px; padding: 8px 0; border-bottom: 1px solid var(--border); }
.breakdown-row:last-child { border-bottom: none; }
.bar-bg { flex: 1; background: var(--border); border-radius: 4px; height: 8px; overflow: hidden; }
.bar-fill { height: 100%; background: var(--accent); border-radius: 4px; }
.empty { color: var(--muted); text-align: center; padding: 32px 0; font-size: 13px; }
/* ── Buttons ─────────────────────────────────── */
.btn { border: none; border-radius: 7px; padding: 9px 16px; font-size: 13px; font-weight: 600; cursor: pointer; transition: opacity .15s; }
.btn:hover { opacity: .85; }
.btn-primary { background: var(--accent); color: #fff; }
.btn-ghost { background: transparent; border: 1px solid var(--border); color: var(--text); }
.btn-danger { background: rgba(239,68,68,.15); color: var(--red); border: 1px solid rgba(239,68,68,.3); }
.btn-sm { padding: 5px 10px; font-size: 12px; border-radius: 5px; }
#refresh-btn { background: none; border: 1px solid var(--border); color: var(--text); border-radius: 6px; padding: 6px 12px; cursor: pointer; font-size: 13px; }
/* ── Flow Config panel ───────────────────────── */
.config-list { display: flex; flex-direction: column; gap: 12px; margin-bottom: 24px; }
.config-card {
background: var(--surface); border: 1px solid var(--border); border-radius: 10px;
padding: 16px 18px; display: flex; align-items: flex-start; gap: 14px;
}
.config-card.is-active { border-color: var(--accent); }
.config-info { flex: 1; }
.config-name { font-weight: 600; font-size: 15px; margin-bottom: 4px; display: flex; align-items: center; gap: 8px; }
.config-meta { color: var(--muted); font-size: 12px; }
.config-actions { display: flex; gap: 8px; align-items: center; flex-shrink: 0; }
/* ── Form ────────────────────────────────────── */
.form-overlay {
position: fixed; inset: 0; background: rgba(0,0,0,.65); backdrop-filter: blur(4px);
display: flex; align-items: flex-start; justify-content: center;
z-index: 200; overflow-y: auto; padding: 40px 16px;
}
.form-overlay.hidden { display: none; }
.form-box {
background: var(--surface); border: 1px solid var(--border); border-radius: 14px;
padding: 32px 28px; width: 680px; max-width: 100%;
}
.form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
.form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
.fg { margin-bottom: 14px; }
.fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
.fg input, .fg select, .fg textarea {
width: 100%; background: var(--bg); border: 1px solid var(--border); border-radius: 7px;
padding: 9px 11px; color: var(--text); font-size: 13px; font-family: inherit;
}
.fg textarea { resize: vertical; min-height: 64px; font-family: monospace; }
.fg input:focus, .fg select:focus, .fg textarea:focus { outline: none; border-color: var(--accent); }
.fg .hint { color: var(--muted); font-size: 11px; margin-top: 4px; }
.form-footer { display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; }
/* ── Auth modal ──────────────────────────────── */
#auth-modal {
position: fixed; inset: 0; background: rgba(0,0,0,.75); backdrop-filter: blur(4px);
display: flex; align-items: center; justify-content: center; z-index: 300;
}
#auth-modal.hidden { display: none; }
.modal-box { background: var(--surface); border: 1px solid var(--border); border-radius: 14px; padding: 36px 32px; width: 320px; }
.modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
.error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
.error-msg.hidden { display: none; }
</style>
</head>
<body>
<!-- Auth Modal -->
<div id="auth-modal">
<div class="modal-box">
<h2>Seed Admin</h2>
<div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
<div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
<button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
<p class="error-msg hidden" id="auth-error">Invalid credentials</p>
</div>
</div>
<!-- Flow Config Form Overlay -->
<div class="form-overlay hidden" id="config-overlay">
<div class="form-box">
<h2 id="form-title">New Flow Config</h2>
<input type="hidden" id="cfg-id" value="" />
<div class="fg">
<label>Config Name</label>
<input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
</div>
<div class="fg">
<label style="display:flex;align-items:center;gap:8px;cursor:pointer">
<input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
</label>
</div>
<!-- Search -->
<div class="form-section">Search Step</div>
<div class="fg">
<label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
<textarea id="cfg-search-texts" placeholder="shoes&#10;blue sneakers&#10;running shoes"></textarea>
</div>
<div class="form-row">
<div class="fg">
<label>Search Box Selector</label>
<input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
</div>
<div class="fg">
<label>Type</label>
<select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
</div>
</div>
<div class="form-row">
<div class="fg">
<label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
<input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
</div>
<div class="fg">
<label>Type</label>
<select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
</div>
</div>
<div class="fg">
<label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
<input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
</div>
<!-- Scroll -->
<div class="form-section">Scroll Step (Infinite Scroll)</div>
<div class="fg">
<label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
<input id="cfg-scroll-container" type="text" placeholder=".product-list" />
</div>
<div class="form-row">
<div class="fg">
<label>Max Scrolls</label>
<input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
</div>
<div class="fg">
<label>Scroll Pause (ms)</label>
<input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
</div>
</div>
<div class="fg">
<label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
<input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
</div>
<!-- Item -->
<div class="form-section">Item Selection</div>
<div class="fg">
<label>Item Selector Template</label>
<input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
<p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
</div>
<div class="fg">
<label>Selector Type</label>
<select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
</div>
<div class="fg">
<label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
<textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
</div>
<div class="form-footer">
<button class="btn btn-ghost" id="form-cancel">Cancel</button>
<button class="btn btn-primary" id="form-save">Save Config</button>
</div>
</div>
</div>
<!-- ═══════════════════════════════════ APP ═══════════════════════════════════ -->
<header>
<div style="display:flex;align-items:center;gap:10px">
<h1>Seed</h1><span class="badge">LIVE</span>
</div>
<div style="display:flex;align-items:center;gap:14px">
<span id="last-updated"></span>
<button id="refresh-btn">↻ Refresh</button>
</div>
</header>
<div class="tabs">
<div class="tab active" data-tab="dashboard">Dashboard</div>
<div class="tab" data-tab="flows">Flow Configs</div>
</div>
<!-- ── Dashboard Tab ──────────────────────────────────────────── -->
<div class="tab-panel active" id="tab-dashboard">
<main>
<div class="cards">
<div class="card"><div class="label">Total Runs</div><div class="value accent" id="c-total"></div></div>
<div class="card"><div class="label">Unique Users</div><div class="value" id="c-unique"></div></div>
<div class="card"><div class="label">Successes</div><div class="value green" id="c-success"></div></div>
<div class="card"><div class="label">Failures</div><div class="value red" id="c-failures"></div></div>
<div class="card"><div class="label">Success Rate</div><div class="value" id="c-rate"></div></div>
</div>
<div class="grid-2">
<div class="panel">
<div class="section-title">Scenario Breakdown</div>
<div id="scenario-breakdown"><p class="empty">No data yet.</p></div>
</div>
<div class="panel">
<div class="section-title">Top Failure Reasons</div>
<div id="failure-reasons"><p class="empty">No failures recorded.</p></div>
</div>
</div>
<div class="panel">
<div class="section-title" style="margin-bottom:14px">Recent Runs</div>
<div style="overflow-x:auto">
<table>
<thead><tr><th>#</th><th>Time</th><th>Scenario</th><th>Identifier</th><th>Status</th><th>Duration</th><th>Failure Reason</th></tr></thead>
<tbody id="runs-tbody"><tr><td colspan="7" class="empty">No runs yet.</td></tr></tbody>
</table>
</div>
</div>
</main>
</div>
<!-- ── Flow Configs Tab ───────────────────────────────────────── -->
<div class="tab-panel" id="tab-flows">
<main>
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:20px">
<div>
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
<div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
</div>
<button class="btn btn-primary" id="btn-new-config">+ New Config</button>
</div>
<div class="config-list" id="config-list">
<p class="empty">No configs yet. Create one to get started.</p>
</div>
</main>
</div>
<script>
/* ─────────────────────────── Auth ─────────────────────────── */
let _creds = null;
function b64(u, p) { return btoa(`${u}:${p}`); }
function authHeader() { return { 'Authorization': `Basic ${b64(_creds.u, _creds.p)}` }; }
async function apiFetch(path, opts = {}) {
const res = await fetch(path, { ...opts, headers: { ...authHeader(), ...(opts.headers || {}) } });
if (res.status === 401) { showAuth(); return null; }
return res;
}
function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
document.getElementById('login-btn').addEventListener('click', async () => {
const u = document.getElementById('inp-user').value.trim();
const p = document.getElementById('inp-pass').value;
const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
if (res.ok) {
_creds = { u, p };
document.getElementById('auth-error').classList.add('hidden');
hideAuth();
await Promise.all([loadStats(), loadConfigs()]);
startAutoRefresh();
} else {
document.getElementById('auth-error').classList.remove('hidden');
}
});
document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
/* ─────────────────────────── Tabs ─────────────────────────── */
document.querySelectorAll('.tab').forEach(tab => {
tab.addEventListener('click', () => {
document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active'));
tab.classList.add('active');
document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
if (tab.dataset.tab === 'flows') loadConfigs();
});
});
/* ─────────────────────────── Dashboard ──────────────────────── */
async function loadStats() {
const res = await apiFetch('/api/stats');
if (!res || !res.ok) return;
renderStats(await res.json());
}
function renderStats(data) {
document.getElementById('c-total').textContent = data.total_runs;
document.getElementById('c-unique').textContent = data.unique_identifiers;
document.getElementById('c-success').textContent = data.successes;
document.getElementById('c-failures').textContent = data.failures;
document.getElementById('c-rate').textContent = `${data.success_rate}%`;
document.getElementById('last-updated').textContent = `Updated ${new Date().toLocaleTimeString()}`;
const sbEl = document.getElementById('scenario-breakdown');
sbEl.innerHTML = data.scenario_breakdown.length === 0
? '<p class="empty">No data yet.</p>'
: data.scenario_breakdown.map(s => {
const pct = s.total ? Math.round(s.success / s.total * 100) : 0;
return `<div class="breakdown-row">
<span style="width:70px;font-weight:600">${s.scenario}</span>
<div class="bar-bg"><div class="bar-fill" style="width:${pct}%"></div></div>
<span style="width:38px;text-align:right;color:var(--muted);font-size:12px">${pct}%</span>
<span style="width:70px;text-align:right;color:var(--muted);font-size:12px">${s.success}/${s.total}</span>
</div>`;
}).join('');
const frEl = document.getElementById('failure-reasons');
frEl.innerHTML = data.failure_reasons.length === 0
? '<p class="empty">No failures recorded.</p>'
: data.failure_reasons.map(r =>
`<div class="reason-row"><span class="reason-text">${esc(r.reason)}</span><span class="reason-count">${r.count}</span></div>`
).join('');
const tbody = document.getElementById('runs-tbody');
tbody.innerHTML = !data.recent_runs.length
? '<tr><td colspan="7" class="empty">No runs yet.</td></tr>'
: data.recent_runs.map(r => `<tr>
<td style="color:var(--muted)">${r.id}</td>
<td style="color:var(--muted);white-space:nowrap">${fmtTime(r.created_at)}</td>
<td><span style="text-transform:uppercase;font-size:11px;font-weight:600">${esc(r.scenario)}</span></td>
<td style="font-family:monospace">${esc(r.identifier)}</td>
<td><span class="pill ${r.success ? 'ok' : 'err'}">${r.success ? 'OK' : 'FAIL'}</span></td>
<td style="color:var(--muted)">${r.total_ms ? r.total_ms + ' ms' : ''}</td>
<td style="color:var(--red);font-size:12px;max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${r.failure_reason ? esc(r.failure_reason) : ''}</td>
</tr>`).join('');
}
document.getElementById('refresh-btn').addEventListener('click', loadStats);
let _timer = null;
function startAutoRefresh() {
if (_timer) clearInterval(_timer);
_timer = setInterval(loadStats, 15000);
}
/* ─────────────────────────── Flow Configs ──────────────────── */
let _configs = [];
async function loadConfigs() {
const res = await apiFetch('/api/flow-configs');
if (!res || !res.ok) return;
_configs = await res.json();
renderConfigs();
}
function renderConfigs() {
const el = document.getElementById('config-list');
if (!_configs.length) {
el.innerHTML = '<p class="empty">No configs yet. Create one to get started.</p>';
return;
}
el.innerHTML = _configs.map(c => `
<div class="config-card ${c.is_active ? 'is-active' : ''}" id="cc-${c.id}">
<div class="config-info">
<div class="config-name">
${esc(c.name)}
${c.is_active ? '<span class="pill active-badge">ACTIVE</span>' : ''}
</div>
<div class="config-meta">
Searches: <strong>${(c.search_texts_json || []).length}</strong> text(s) &nbsp;·&nbsp;
Items: <strong>${(c.target_item_ids_json || []).length}</strong> ID(s) &nbsp;·&nbsp;
Max scrolls: <strong>${c.max_scrolls}</strong> &nbsp;·&nbsp;
Search box: <code style="font-size:11px">${esc(c.search_box_selector || '—')}</code>
</div>
<div class="config-meta" style="margin-top:4px">
Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
</div>
</div>
<div class="config-actions">
${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
<button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
<button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
</div>
</div>
`).join('');
}
async function activateConfig(id) {
const res = await apiFetch(`/api/flow-configs/${id}/activate`, { method: 'POST' });
if (res && res.ok) await loadConfigs();
}
async function deleteConfig(id) {
const cfg = _configs.find(c => c.id === id);
if (!confirm(`Delete "${cfg?.name}"?`)) return;
const res = await apiFetch(`/api/flow-configs/${id}`, { method: 'DELETE' });
if (res && (res.ok || res.status === 204)) await loadConfigs();
}
/* ─── Form ─── */
function openNewForm() {
document.getElementById('form-title').textContent = 'New Flow Config';
document.getElementById('cfg-id').value = '';
document.getElementById('cfg-name').value = '';
document.getElementById('cfg-active').checked = false;
document.getElementById('cfg-search-texts').value = '';
document.getElementById('cfg-search-sel').value = '';
document.getElementById('cfg-search-sel-type').value = 'css';
document.getElementById('cfg-submit-sel').value = '';
document.getElementById('cfg-submit-sel-type').value = 'css';
document.getElementById('cfg-results-sel').value = '';
document.getElementById('cfg-scroll-container').value = '';
document.getElementById('cfg-max-scrolls').value = '30';
document.getElementById('cfg-scroll-pause').value = '1200';
document.getElementById('cfg-no-content-timeout').value = '3000';
document.getElementById('cfg-item-sel').value = '';
document.getElementById('cfg-item-sel-type').value = 'css';
document.getElementById('cfg-item-ids').value = '';
document.getElementById('config-overlay').classList.remove('hidden');
}
function openEditForm(id) {
const c = _configs.find(x => x.id === id);
if (!c) return;
document.getElementById('form-title').textContent = 'Edit Flow Config';
document.getElementById('cfg-id').value = id;
document.getElementById('cfg-name').value = c.name || '';
document.getElementById('cfg-active').checked = !!c.is_active;
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
document.getElementById('config-overlay').classList.remove('hidden');
}
function closeForm() {
document.getElementById('config-overlay').classList.add('hidden');
}
function collectForm() {
return {
name: document.getElementById('cfg-name').value.trim(),
is_active: document.getElementById('cfg-active').checked,
search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean),
search_box_selector: document.getElementById('cfg-search-sel').value.trim(),
search_box_selector_type: document.getElementById('cfg-search-sel-type').value,
search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
item_selector_type: document.getElementById('cfg-item-sel-type').value,
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
};
}
document.getElementById('btn-new-config').addEventListener('click', openNewForm);
document.getElementById('form-cancel').addEventListener('click', closeForm);
document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
document.getElementById('form-save').addEventListener('click', async () => {
const body = collectForm();
if (!body.name) { alert('Config name is required.'); return; }
const cfgId = document.getElementById('cfg-id').value;
const isEdit = !!cfgId;
const url = isEdit ? `/api/flow-configs/${cfgId}` : '/api/flow-configs';
const method = isEdit ? 'PUT' : 'POST';
const res = await apiFetch(url, {
method,
headers: { 'Content-Type': 'application/json' },
body: JSON.stringify(body),
});
if (res && (res.ok || res.status === 201)) {
closeForm();
await loadConfigs();
} else {
alert('Failed to save config. Check console.');
console.error(await res?.text());
}
});
/* ─────────────────────────── Utils ────────────────────────── */
function esc(s) {
return String(s ?? '').replace(/&/g,'&amp;').replace(/</g,'&lt;').replace(/>/g,'&gt;').replace(/"/g,'&quot;');
}
function fmtTime(iso) {
if (!iso) return '';
return new Date(iso).toLocaleString();
}
</script>
</body>
</html>
+38
View File
@@ -0,0 +1,38 @@
from __future__ import annotations
import os
from dataclasses import dataclass, field
from dotenv import load_dotenv
load_dotenv()
@dataclass
class Config:
target_url: str = os.getenv("TARGET_URL", "https://example.com")
admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
admin_port: int = int(os.getenv("ADMIN_PORT", "8000"))
db_path: str = os.getenv("DB_PATH", "data/tracker.db")
headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
chrome_binary: str | None = os.getenv("CHROME_BINARY")
# Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
phone_numbers: list[str] = field(default_factory=list)
# Flow steps — ordered list of step names to execute after login
# Override in .env as: FLOW_STEPS=step_home,step_browse,step_checkout
flow_steps: list[str] = field(default_factory=lambda: ["step_home"])
def __post_init__(self) -> None:
raw_phones = os.getenv("PHONE_NUMBERS", "")
if raw_phones:
self.phone_numbers = [p.strip() for p in raw_phones.split(",") if p.strip()]
raw_steps = os.getenv("FLOW_STEPS", "")
if raw_steps:
self.flow_steps = [s.strip() for s in raw_steps.split(",") if s.strip()]
config = Config()
View File
+83
View File
@@ -0,0 +1,83 @@
"""Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
from __future__ import annotations
import random
import time
from contextlib import contextmanager
from typing import Generator
import undetected_chromedriver as uc
from selenium.webdriver.chrome.options import Options
from selenium_stealth import stealth
from config import config
_USER_AGENTS = [
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
]
def _build_options(user_agent: str, headless: bool) -> Options:
opts = Options()
if headless:
opts.add_argument("--headless=new")
opts.add_argument(f"--user-agent={user_agent}")
opts.add_argument("--no-sandbox")
opts.add_argument("--disable-dev-shm-usage")
opts.add_argument("--disable-blink-features=AutomationControlled")
opts.add_argument("--disable-infobars")
opts.add_argument("--window-size=1920,1080")
opts.add_experimental_option("excludeSwitches", ["enable-automation"])
opts.add_experimental_option("useAutomationExtension", False)
if config.chrome_binary:
opts.binary_location = config.chrome_binary
return opts
def make_driver(headless: bool | None = None) -> uc.Chrome:
"""Return a stealthed undetected Chrome instance."""
use_headless = config.headless if headless is None else headless
ua = random.choice(_USER_AGENTS)
opts = _build_options(ua, use_headless)
driver = uc.Chrome(options=opts, use_subprocess=True)
stealth(
driver,
languages=["en-US", "en"],
vendor="Google Inc.",
platform="Win32",
webgl_vendor="Intel Inc.",
renderer="Intel Iris OpenGL Engine",
fix_hairline=True,
)
# Mask navigator.webdriver via CDP
driver.execute_cdp_cmd(
"Page.addScriptToEvaluateOnNewDocument",
{
"source": """
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en']});
"""
},
)
return driver
@contextmanager
def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
"""Context manager that guarantees driver.quit() on exit."""
driver = make_driver(headless)
try:
yield driver
finally:
driver.quit()
def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
"""Sleep for a random human-like interval."""
time.sleep(random.uniform(lo, hi))
+282
View File
@@ -0,0 +1,282 @@
"""
Dynamic flow executor — reads the active flow config from DB at runtime and runs:
1. search_step : type each search text into the search box
2. scroll_step : infinite-scroll until target item is visible or max_scrolls reached
3. click_step : click each target item id
"""
from __future__ import annotations
import asyncio
import time
from dataclasses import dataclass, field
from typing import Any
import undetected_chromedriver as uc
from selenium.common.exceptions import NoSuchElementException, TimeoutException
from selenium.webdriver.common.action_chains import ActionChains
from selenium.webdriver.common.by import By
from selenium.webdriver.common.keys import Keys
from selenium.webdriver.remote.webelement import WebElement
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
class DynamicFlowError(Exception):
pass
@dataclass
class StepResult:
name: str
success: bool
data: dict[str, Any] = field(default_factory=dict)
error: str | None = None
duration_ms: int = 0
@dataclass
class DynamicFlowResult:
config_id: int
config_name: str
steps: list[StepResult] = field(default_factory=list)
@property
def success(self) -> bool:
return all(s.success for s in self.steps)
@property
def failure_reason(self) -> str | None:
for s in self.steps:
if not s.success:
return f"{s.name}: {s.error}"
return None
def _by(selector_type: str) -> str:
return By.XPATH if selector_type.lower() == "xpath" else By.CSS_SELECTOR
def _find(driver: uc.Chrome, selector: str, selector_type: str) -> WebElement:
return driver.find_element(_by(selector_type), selector)
def _find_all(driver: uc.Chrome, selector: str, selector_type: str) -> list[WebElement]:
return driver.find_elements(_by(selector_type), selector)
def _wait_for(
driver: uc.Chrome,
selector: str,
selector_type: str,
timeout: float = 15.0,
) -> WebElement:
return WebDriverWait(driver, timeout).until(
EC.presence_of_element_located((_by(selector_type), selector))
)
class DynamicFlow:
def __init__(self, driver: uc.Chrome, cfg: dict[str, Any]) -> None:
self.driver = driver
self.cfg = cfg
def run(self) -> DynamicFlowResult:
result = DynamicFlowResult(
config_id=self.cfg["id"],
config_name=self.cfg["name"],
)
search_texts: list[str] = self.cfg.get("search_texts_json") or []
item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
for text in search_texts:
step = self._run_search(text)
result.steps.append(step)
if not step.success:
return result
human_delay(0.8, 1.5)
for item_id in item_ids:
scroll_step = self._run_scroll_and_find(item_id)
result.steps.append(scroll_step)
if not scroll_step.success:
continue # try next item_id, don't abort the whole flow
click_step = self._run_click(item_id, scroll_step.data.get("element"))
result.steps.append(click_step)
human_delay(1.0, 2.5)
return result
# ------------------------------------------------------------------
# Step: type into search box
# ------------------------------------------------------------------
def _run_search(self, text: str) -> StepResult:
t0 = time.monotonic()
name = f"search:{text}"
try:
sel = self.cfg["search_box_selector"]
sel_type = self.cfg["search_box_selector_type"]
if not sel:
raise DynamicFlowError("search_box_selector is not configured")
box = _wait_for(self.driver, sel, sel_type)
box.clear()
human_delay(0.3, 0.6)
# Type character by character for a human feel
for ch in text:
box.send_keys(ch)
time.sleep(0.04)
human_delay(0.4, 0.8)
# Submit: explicit button or Enter
submit_sel = self.cfg.get("search_submit_selector", "")
if submit_sel:
submit_type = self.cfg.get("search_submit_selector_type", "css")
btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
btn.click()
else:
box.send_keys(Keys.RETURN)
# Wait for results container if configured
results_sel = self.cfg.get("search_results_selector", "")
if results_sel:
_wait_for(self.driver, results_sel, "css", timeout=15.0)
human_delay(1.0, 2.0)
return StepResult(name=name, success=True, data={"text": text},
duration_ms=int((time.monotonic() - t0) * 1000))
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
# ------------------------------------------------------------------
# Step: scroll (infinite-scroll) until item selector matches
# ------------------------------------------------------------------
def _run_scroll_and_find(self, item_id: str) -> StepResult:
t0 = time.monotonic()
name = f"scroll_find:{item_id}"
try:
item_sel = self._build_item_selector(item_id)
item_sel_type = self.cfg.get("item_selector_type", "css")
scroll_container = self.cfg.get("scroll_container_selector", "").strip()
max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
# Check if already visible before scrolling
found = self._find_item_visible(item_sel, item_sel_type)
if found:
return StepResult(name=name, success=True,
data={"item_id": item_id, "scrolls": 0, "element": found},
duration_ms=int((time.monotonic() - t0) * 1000))
last_height = self._get_scroll_height(scroll_container)
stale_since: float | None = None
for scroll_num in range(1, max_scrolls + 1):
self._scroll_down(scroll_container)
time.sleep(pause_ms / 1000)
found = self._find_item_visible(item_sel, item_sel_type)
if found:
return StepResult(name=name, success=True,
data={"item_id": item_id, "scrolls": scroll_num, "element": found},
duration_ms=int((time.monotonic() - t0) * 1000))
new_height = self._get_scroll_height(scroll_container)
if new_height == last_height:
if stale_since is None:
stale_since = time.monotonic()
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
raise DynamicFlowError(
f"No new content after {no_new_ms} ms — reached end of page "
f"without finding item '{item_id}'"
)
else:
stale_since = None
last_height = new_height
raise DynamicFlowError(
f"Item '{item_id}' not found after {max_scrolls} scrolls"
)
except DynamicFlowError:
raise
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
# ------------------------------------------------------------------
# Step: click the found element
# ------------------------------------------------------------------
def _run_click(self, item_id: str, element: Any) -> StepResult:
t0 = time.monotonic()
name = f"click:{item_id}"
try:
if element is None:
raise DynamicFlowError("No element reference from scroll step")
el: WebElement = element
# Scroll element into view and click
self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
human_delay(0.3, 0.7)
try:
el.click()
except Exception:
# Fallback: JS click
self.driver.execute_script("arguments[0].click();", el)
human_delay(0.8, 1.5)
return StepResult(name=name, success=True,
data={"item_id": item_id, "url_after": self.driver.current_url},
duration_ms=int((time.monotonic() - t0) * 1000))
except Exception as exc:
return StepResult(name=name, success=False, error=str(exc),
duration_ms=int((time.monotonic() - t0) * 1000))
# ------------------------------------------------------------------
# Helpers
# ------------------------------------------------------------------
def _build_item_selector(self, item_id: str) -> str:
template: str = self.cfg.get("item_selector_template", "")
if not template:
raise DynamicFlowError("item_selector_template is not configured")
return template.replace("{item_id}", item_id)
def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
try:
els = _find_all(self.driver, selector, selector_type)
for el in els:
if el.is_displayed():
return el
except NoSuchElementException:
pass
return None
def _get_scroll_height(self, container_sel: str) -> int:
if container_sel:
try:
el = _find(self.driver, container_sel, "css")
return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
except Exception:
pass
return int(self.driver.execute_script("return document.body.scrollHeight"))
def _scroll_down(self, container_sel: str) -> None:
if container_sel:
try:
el = _find(self.driver, container_sel, "css")
self.driver.execute_script(
"arguments[0].scrollTop = arguments[0].scrollHeight", el
)
return
except Exception:
pass
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+104
View File
@@ -0,0 +1,104 @@
"""Executes a pre-defined sequence of flow steps and records results."""
from __future__ import annotations
import time
from dataclasses import dataclass, field
from typing import Callable
import undetected_chromedriver as uc
from config import config
from crawler.driver import human_delay
# A flow step is a callable that receives the driver and returns arbitrary data.
StepFn = Callable[[uc.Chrome], dict[str, object]]
# Registry — add your custom step functions here.
# Keys must match the step names in config.flow_steps.
STEP_REGISTRY: dict[str, StepFn] = {}
def register_step(name: str) -> Callable[[StepFn], StepFn]:
"""Decorator to register a function as a named flow step."""
def decorator(fn: StepFn) -> StepFn:
STEP_REGISTRY[name] = fn
return fn
return decorator
@dataclass
class StepResult:
name: str
success: bool
data: dict[str, object] = field(default_factory=dict)
error: str | None = None
duration_ms: int = 0
@dataclass
class FlowResult:
scenario: str
identifier: str # phone number or "fresh"
steps: list[StepResult] = field(default_factory=list)
@property
def success(self) -> bool:
return all(s.success for s in self.steps)
@property
def failure_reason(self) -> str | None:
for s in self.steps:
if not s.success:
return f"{s.name}: {s.error}"
return None
class FlowRunner:
def __init__(self, driver: uc.Chrome) -> None:
self.driver = driver
def run(self, scenario: str, identifier: str) -> FlowResult:
result = FlowResult(scenario=scenario, identifier=identifier)
for step_name in config.flow_steps:
fn = STEP_REGISTRY.get(step_name)
if fn is None:
result.steps.append(
StepResult(
name=step_name,
success=False,
error=f"Step '{step_name}' not found in registry",
)
)
break
t0 = time.monotonic()
try:
data = fn(self.driver)
duration = int((time.monotonic() - t0) * 1000)
result.steps.append(StepResult(name=step_name, success=True, data=data, duration_ms=duration))
human_delay(0.5, 1.5)
except Exception as exc:
duration = int((time.monotonic() - t0) * 1000)
result.steps.append(
StepResult(name=step_name, success=False, error=str(exc), duration_ms=duration)
)
break # abort remaining steps on first failure
return result
# ---------------------------------------------------------------------------
# Example steps — replace with your actual flow logic
# ---------------------------------------------------------------------------
@register_step("step_home")
def step_home(driver: uc.Chrome) -> dict[str, object]:
return {"url": driver.current_url, "title": driver.title}
@register_step("step_browse")
def step_browse(driver: uc.Chrome) -> dict[str, object]:
# Example: navigate to a listing page
human_delay(1.0, 2.0)
return {"url": driver.current_url}
View File
+52
View File
@@ -0,0 +1,52 @@
"""Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
from __future__ import annotations
import undetected_chromedriver as uc
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.support.ui import WebDriverWait
from config import config
from crawler.driver import human_delay, make_driver
class FreshSessionScenario:
"""
Opens the target URL in a brand-new browser profile (no stored cookies,
localStorage, or cache) so each run appears as a first-time visitor.
Because undetected-chromedriver already creates an isolated temp profile
per instance, simply spinning up a new driver is sufficient — but we also
explicitly delete all cookies after loading to be safe.
"""
def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
self.driver = driver
self.wait = WebDriverWait(driver, timeout)
def run(self) -> dict[str, object]:
"""Navigate to the target, clear any residual state, return page info."""
self.driver.delete_all_cookies()
# Clear localStorage / sessionStorage via JS
self.driver.execute_script(
"try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
)
self.driver.get(config.target_url)
human_delay(2.0, 4.0)
# Wait for the page to reach a ready state
self.wait.until(
lambda d: d.execute_script("return document.readyState") == "complete"
)
title = self.driver.title
current_url = self.driver.current_url
cookies = self.driver.get_cookies()
return {
"title": title,
"url": current_url,
"cookie_count": len(cookies),
"cookies": cookies,
}
+106
View File
@@ -0,0 +1,106 @@
"""Scenario 1 — SMS-OTP login with a batch of phone numbers."""
from __future__ import annotations
import time
from typing import Callable
import undetected_chromedriver as uc
from selenium.common.exceptions import TimeoutException
from selenium.webdriver.common.by import By
from selenium.webdriver.support import expected_conditions as EC
from selenium.webdriver.support.ui import WebDriverWait
from config import config
from crawler.driver import human_delay
class OTPLoginError(Exception):
pass
class OTPLoginScenario:
"""
Drives an SMS-OTP login flow.
The caller supplies an `otp_resolver` — a callable that receives the
phone number and returns the OTP string. Wire it up to your SMS
gateway / SIM-management API.
"""
def __init__(
self,
driver: uc.Chrome,
phone: str,
otp_resolver: Callable[[str], str],
timeout: int = 30,
) -> None:
self.driver = driver
self.phone = phone
self.otp_resolver = otp_resolver
self.wait = WebDriverWait(driver, timeout)
# ------------------------------------------------------------------
# Override these selectors to match the target site's actual HTML.
# ------------------------------------------------------------------
_PHONE_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[type='tel'], input[name='phone']")
_SEND_OTP_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
_OTP_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[name='otp'], input[autocomplete='one-time-code']")
_CONFIRM_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
_SUCCESS_INDICATOR = (By.CSS_SELECTOR, "[data-testid='home'], .dashboard, #main-content")
def run(self) -> dict[str, object]:
"""Execute the OTP login. Returns a result dict."""
self.driver.get(config.target_url)
human_delay(1.5, 3.0)
try:
self._enter_phone()
self._request_otp()
otp = self._resolve_otp()
self._enter_otp(otp)
self._confirm_login()
self._wait_for_success()
except TimeoutException as exc:
raise OTPLoginError(f"Timeout during OTP login for {self.phone}") from exc
return {"phone": self.phone, "cookies": self.driver.get_cookies()}
def _enter_phone(self) -> None:
field = self.wait.until(EC.element_to_be_clickable(self._PHONE_FIELD_SELECTOR))
field.clear()
human_delay(0.3, 0.7)
for char in self.phone:
field.send_keys(char)
time.sleep(0.05)
def _request_otp(self) -> None:
btn = self.wait.until(EC.element_to_be_clickable(self._SEND_OTP_BTN_SELECTOR))
human_delay(0.5, 1.2)
btn.click()
def _resolve_otp(self) -> str:
# Poll for the OTP from the external resolver (SMS gateway / webhook)
for attempt in range(12):
human_delay(5.0, 8.0)
otp = self.otp_resolver(self.phone)
if otp:
return otp
if attempt == 11:
raise OTPLoginError(f"OTP not received for {self.phone} after 12 attempts")
return "" # unreachable but satisfies type checker
def _enter_otp(self, otp: str) -> None:
field = self.wait.until(EC.element_to_be_clickable(self._OTP_FIELD_SELECTOR))
field.clear()
human_delay(0.3, 0.7)
for char in otp:
field.send_keys(char)
time.sleep(0.08)
def _confirm_login(self) -> None:
btn = self.wait.until(EC.element_to_be_clickable(self._CONFIRM_BTN_SELECTOR))
human_delay(0.5, 1.0)
btn.click()
def _wait_for_success(self) -> None:
self.wait.until(EC.presence_of_element_located(self._SUCCESS_INDICATOR))
+121
View File
@@ -0,0 +1,121 @@
"""Entry point — run crawl scenarios or start the admin panel."""
from __future__ import annotations
import argparse
import asyncio
import sys
import time
import uvicorn
from admin.models import get_active_flow_config, init_db, insert_run
from config import config
from crawler.driver import driver_session
from crawler.dynamic_flow import DynamicFlow
from crawler.scenarios.fresh_session import FreshSessionScenario
from crawler.scenarios.otp_login import OTPLoginScenario
def _load_active_flow() -> dict[str, object]:
cfg = asyncio.run(get_active_flow_config())
if cfg is None:
print("[WARN] No active flow config — running without dynamic flow steps.", file=sys.stderr)
return cfg or {}
def _record(
scenario: str,
identifier: str,
flow_result: object | None,
exc: Exception | None,
t0: float,
) -> None:
from crawler.dynamic_flow import DynamicFlowResult
total_ms = int((time.monotonic() - t0) * 1000)
if exc is not None:
asyncio.run(insert_run(scenario, identifier, False, str(exc), [], total_ms))
print(f"[FAIL] {identifier}: {exc}", file=sys.stderr)
return
if isinstance(flow_result, DynamicFlowResult):
steps_data = [
{"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
for s in flow_result.steps
]
asyncio.run(insert_run(
scenario, identifier, flow_result.success,
flow_result.failure_reason, steps_data, total_ms,
))
status = "OK" if flow_result.success else f"FAIL ({flow_result.failure_reason})"
else:
asyncio.run(insert_run(scenario, identifier, True, None, [], total_ms))
status = "OK (no flow config)"
print(f"[{status}] {identifier}{total_ms} ms")
def run_otp(phone: str) -> None:
def otp_resolver(p: str) -> str:
# TODO: wire up your SMS gateway / webhook here.
raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
flow_cfg = _load_active_flow()
t0 = time.monotonic()
try:
with driver_session() as driver:
OTPLoginScenario(driver, phone, otp_resolver).run()
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
except Exception as exc:
_record("otp", phone, None, exc, t0)
return
_record("otp", phone, result, None, t0)
def run_fresh() -> None:
flow_cfg = _load_active_flow()
t0 = time.monotonic()
try:
with driver_session() as driver:
FreshSessionScenario(driver).run()
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
except Exception as exc:
_record("fresh", "fresh", None, exc, t0)
return
_record("fresh", "fresh", result, None, t0)
def run_admin() -> None:
init_db()
from admin.main import app
uvicorn.run(app, host=config.admin_host, port=config.admin_port)
def main() -> None:
parser = argparse.ArgumentParser(description="Seed crawler & admin")
sub = parser.add_subparsers(dest="cmd", required=True)
sub.add_parser("admin", help="Start the admin panel")
otp_p = sub.add_parser("otp", help="Run Scenario 1 (SMS-OTP login)")
otp_p.add_argument("--phone", help="Single phone number (default: all from config)")
sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
args = parser.parse_args()
if args.cmd == "admin":
run_admin()
elif args.cmd == "otp":
phones = [args.phone] if args.phone else config.phone_numbers
if not phones:
print("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone", file=sys.stderr)
sys.exit(1)
for phone in phones:
run_otp(phone)
elif args.cmd == "fresh":
run_fresh()
if __name__ == "__main__":
main()
+24
View File
@@ -0,0 +1,24 @@
[project]
name = "seed"
version = "0.1.0"
requires-python = ">=3.11"
dependencies = [
"undetected-chromedriver>=3.5.5",
"selenium>=4.18.0",
"selenium-stealth>=1.0.6",
"fastapi>=0.111.0",
"uvicorn[standard]>=0.30.0",
"aiosqlite>=0.20.0",
"httpx>=0.27.0",
"python-multipart>=0.0.9",
"pydantic>=2.7.0",
"python-dotenv>=1.0.1",
]
[tool.pyright]
pythonVersion = "3.11"
typeCheckingMode = "strict"
reportMissingImports = true
reportMissingTypeStubs = false
venvPath = "."
venv = ".venv"