init
This commit is contained in:
@@ -0,0 +1,22 @@
|
||||
# Copy to .env and fill in
|
||||
|
||||
TARGET_URL=https://example.com
|
||||
|
||||
# Admin panel credentials
|
||||
ADMIN_USERNAME=admin
|
||||
ADMIN_PASSWORD=changeme
|
||||
ADMIN_HOST=127.0.0.1
|
||||
ADMIN_PORT=8000
|
||||
|
||||
# SQLite DB path
|
||||
DB_PATH=data/tracker.db
|
||||
|
||||
# Chrome: set to false to watch the browser window
|
||||
HEADLESS=true
|
||||
# CHROME_BINARY=/usr/bin/chromium # optional: explicit Chrome path
|
||||
|
||||
# Scenario 1 — comma-separated phone numbers
|
||||
PHONE_NUMBERS=+989100000001,+989100000002,+989100000003
|
||||
|
||||
# Flow steps — comma-separated list matching keys in STEP_REGISTRY
|
||||
FLOW_STEPS=step_home,step_browse
|
||||
@@ -0,0 +1,6 @@
|
||||
.venv/
|
||||
__pycache__/
|
||||
*.pyc
|
||||
.env
|
||||
data/
|
||||
*.db
|
||||
@@ -0,0 +1,28 @@
|
||||
"""FastAPI admin application."""
|
||||
from __future__ import annotations
|
||||
|
||||
from pathlib import Path
|
||||
|
||||
from fastapi import FastAPI
|
||||
from fastapi.responses import FileResponse
|
||||
from fastapi.staticfiles import StaticFiles
|
||||
|
||||
from admin.models import init_db
|
||||
from admin.routes import router
|
||||
|
||||
app = FastAPI(title="Seed Admin", docs_url=None, redoc_url=None)
|
||||
|
||||
app.include_router(router)
|
||||
|
||||
_STATIC = Path(__file__).parent / "static"
|
||||
app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
|
||||
|
||||
|
||||
@app.on_event("startup")
|
||||
def on_startup() -> None:
|
||||
init_db()
|
||||
|
||||
|
||||
@app.get("/")
|
||||
def dashboard() -> FileResponse:
|
||||
return FileResponse(str(_STATIC / "dashboard.html"))
|
||||
+241
@@ -0,0 +1,241 @@
|
||||
"""SQLite schema and DB helpers (aiosqlite)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DB_PATH = os.getenv("DB_PATH", "data/tracker.db")
|
||||
|
||||
CREATE_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
scenario TEXT NOT NULL,
|
||||
identifier TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
failure_reason TEXT,
|
||||
steps_json TEXT NOT NULL DEFAULT '[]',
|
||||
total_ms INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier);
|
||||
|
||||
-- Dynamic flow configurations editable from the admin panel
|
||||
CREATE TABLE IF NOT EXISTS flow_configs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
name TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
-- Search step
|
||||
search_texts_json TEXT NOT NULL DEFAULT '[]', -- JSON string[]
|
||||
search_box_selector TEXT NOT NULL DEFAULT '',
|
||||
search_box_selector_type TEXT NOT NULL DEFAULT 'css', -- 'css' | 'xpath'
|
||||
search_submit_selector TEXT NOT NULL DEFAULT '', -- optional — leave blank to use Enter key
|
||||
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_results_selector TEXT NOT NULL DEFAULT '', -- container that appears after results load
|
||||
|
||||
-- Scroll step
|
||||
scroll_container_selector TEXT NOT NULL DEFAULT '', -- element to scroll; blank = window
|
||||
max_scrolls INTEGER NOT NULL DEFAULT 30,
|
||||
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
|
||||
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000, -- bail if DOM height unchanged after this
|
||||
|
||||
-- Item selection
|
||||
item_selector_template TEXT NOT NULL DEFAULT '',
|
||||
-- Template may contain {item_id} placeholder, e.g. "[data-id='{item_id}']"
|
||||
-- or a plain selector like ".result-card" (picks first match)
|
||||
item_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
target_item_ids_json TEXT NOT NULL DEFAULT '[]' -- JSON string[] — one click per id
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
"""Create the DB file and tables (sync, called at startup)."""
|
||||
path = Path(DB_PATH)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.executescript(CREATE_DDL)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async helpers (used by FastAPI routes via aiosqlite)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def insert_run(
|
||||
scenario: str,
|
||||
identifier: str,
|
||||
success: bool,
|
||||
failure_reason: str | None,
|
||||
steps: list[dict[str, object]],
|
||||
total_ms: int,
|
||||
) -> int:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
cur = await db.execute(
|
||||
"""INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
|
||||
)
|
||||
await db.commit()
|
||||
return int(cur.lastrowid or 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# flow_configs CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FlowConfigRow = dict[str, Any]
|
||||
|
||||
|
||||
async def list_flow_configs() -> list[FlowConfigRow]:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
rows = await (await db.execute(
|
||||
"SELECT * FROM flow_configs ORDER BY id DESC"
|
||||
)).fetchall()
|
||||
return [_parse_flow_row(dict(r)) for r in rows]
|
||||
|
||||
|
||||
async def get_flow_config(cfg_id: int) -> FlowConfigRow | None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE id=?", (cfg_id,)
|
||||
)).fetchone()
|
||||
return _parse_flow_row(dict(row)) if row else None
|
||||
|
||||
|
||||
async def get_active_flow_config() -> FlowConfigRow | None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1"
|
||||
)).fetchone()
|
||||
return _parse_flow_row(dict(row)) if row else None
|
||||
|
||||
|
||||
async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int:
|
||||
import aiosqlite
|
||||
|
||||
fields = [
|
||||
"name", "is_active",
|
||||
"search_texts_json", "search_box_selector", "search_box_selector_type",
|
||||
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
|
||||
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
||||
"item_selector_template", "item_selector_type", "target_item_ids_json",
|
||||
]
|
||||
|
||||
# Serialize list fields
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if key in data and isinstance(data[key], list):
|
||||
data[key] = json.dumps(data[key])
|
||||
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
if data.get("is_active"):
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
|
||||
if cfg_id is None:
|
||||
placeholders = ", ".join("?" for _ in fields)
|
||||
cols = ", ".join(fields)
|
||||
values = [data.get(f) for f in fields]
|
||||
cur = await db.execute(
|
||||
f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values
|
||||
)
|
||||
row_id = int(cur.lastrowid or 0)
|
||||
else:
|
||||
set_clause = ", ".join(f"{f}=?" for f in fields)
|
||||
values_u = [data.get(f) for f in fields] + [cfg_id]
|
||||
await db.execute(f"UPDATE flow_configs SET {set_clause} WHERE id=?", values_u)
|
||||
row_id = cfg_id
|
||||
|
||||
await db.commit()
|
||||
return row_id
|
||||
|
||||
|
||||
async def delete_flow_config(cfg_id: int) -> None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def activate_flow_config(cfg_id: int) -> None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
|
||||
def _parse_flow_row(row: dict[str, Any]) -> FlowConfigRow:
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if key in row and isinstance(row[key], str):
|
||||
try:
|
||||
row[key] = json.loads(row[key])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
row[key] = []
|
||||
return row
|
||||
|
||||
|
||||
async def get_stats() -> dict[str, object]:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
|
||||
total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0]
|
||||
unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0]
|
||||
successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0]
|
||||
failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0]
|
||||
|
||||
failure_reasons_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT failure_reason, COUNT(*) as cnt
|
||||
FROM runs WHERE success=0 AND failure_reason IS NOT NULL
|
||||
GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10"""
|
||||
)
|
||||
).fetchall()
|
||||
failure_reasons = [{"reason": r["failure_reason"], "count": r["cnt"]} for r in failure_reasons_raw]
|
||||
|
||||
scenario_breakdown_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT scenario, COUNT(*) as total,
|
||||
SUM(success) as ok,
|
||||
COUNT(*) - SUM(success) as fail
|
||||
FROM runs GROUP BY scenario"""
|
||||
)
|
||||
).fetchall()
|
||||
scenario_breakdown = [
|
||||
{"scenario": r["scenario"], "total": r["total"], "success": r["ok"], "failure": r["fail"]}
|
||||
for r in scenario_breakdown_raw
|
||||
]
|
||||
|
||||
recent_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms
|
||||
FROM runs ORDER BY id DESC LIMIT 50"""
|
||||
)
|
||||
).fetchall()
|
||||
recent = [dict(r) for r in recent_raw]
|
||||
|
||||
return {
|
||||
"total_runs": total,
|
||||
"unique_identifiers": unique_ids,
|
||||
"successes": successes,
|
||||
"failures": failures,
|
||||
"success_rate": round(successes / total * 100, 1) if total else 0.0,
|
||||
"failure_reasons": failure_reasons,
|
||||
"scenario_breakdown": scenario_breakdown,
|
||||
"recent_runs": recent,
|
||||
}
|
||||
+135
@@ -0,0 +1,135 @@
|
||||
"""FastAPI routes — dashboard API + auth."""
|
||||
from __future__ import annotations
|
||||
|
||||
import secrets
|
||||
from typing import Annotated, Any
|
||||
|
||||
from fastapi import APIRouter, Depends, HTTPException, status
|
||||
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
||||
from pydantic import BaseModel
|
||||
|
||||
from admin.models import (
|
||||
activate_flow_config,
|
||||
delete_flow_config,
|
||||
get_flow_config,
|
||||
get_stats,
|
||||
list_flow_configs,
|
||||
upsert_flow_config,
|
||||
)
|
||||
from config import config
|
||||
|
||||
router = APIRouter()
|
||||
security = HTTPBasic()
|
||||
|
||||
|
||||
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
|
||||
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
|
||||
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
|
||||
if not (ok_user and ok_pass):
|
||||
raise HTTPException(
|
||||
status_code=status.HTTP_401_UNAUTHORIZED,
|
||||
detail="Invalid credentials",
|
||||
headers={"WWW-Authenticate": "Basic"},
|
||||
)
|
||||
return credentials.username
|
||||
|
||||
|
||||
Auth = Annotated[str, Depends(require_auth)]
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Stats
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@router.get("/api/stats")
|
||||
async def stats(_: Auth) -> dict[str, object]:
|
||||
return await get_stats()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Flow configs
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
class FlowConfigIn(BaseModel):
|
||||
name: str
|
||||
is_active: bool = False
|
||||
search_texts: list[str] = []
|
||||
search_box_selector: str = ""
|
||||
search_box_selector_type: str = "css"
|
||||
search_submit_selector: str = ""
|
||||
search_submit_selector_type: str = "css"
|
||||
search_results_selector: str = ""
|
||||
scroll_container_selector: str = ""
|
||||
max_scrolls: int = 30
|
||||
scroll_pause_ms: int = 1200
|
||||
no_new_content_timeout_ms: int = 3000
|
||||
item_selector_template: str = ""
|
||||
item_selector_type: str = "css"
|
||||
target_item_ids: list[str] = []
|
||||
|
||||
|
||||
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
||||
return {
|
||||
"name": body.name,
|
||||
"is_active": int(body.is_active),
|
||||
"search_texts_json": body.search_texts,
|
||||
"search_box_selector": body.search_box_selector,
|
||||
"search_box_selector_type": body.search_box_selector_type,
|
||||
"search_submit_selector": body.search_submit_selector,
|
||||
"search_submit_selector_type": body.search_submit_selector_type,
|
||||
"search_results_selector": body.search_results_selector,
|
||||
"scroll_container_selector": body.scroll_container_selector,
|
||||
"max_scrolls": body.max_scrolls,
|
||||
"scroll_pause_ms": body.scroll_pause_ms,
|
||||
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
|
||||
"item_selector_template": body.item_selector_template,
|
||||
"item_selector_type": body.item_selector_type,
|
||||
"target_item_ids_json": body.target_item_ids,
|
||||
}
|
||||
|
||||
|
||||
@router.get("/api/flow-configs")
|
||||
async def list_configs(_: Auth) -> list[dict[str, object]]:
|
||||
return await list_flow_configs() # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.get("/api/flow-configs/{cfg_id}")
|
||||
async def get_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
||||
row = await get_flow_config(cfg_id)
|
||||
if not row:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
return row # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.post("/api/flow-configs", status_code=201)
|
||||
async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
||||
new_id = await upsert_flow_config(_to_db_dict(body))
|
||||
row = await get_flow_config(new_id)
|
||||
return row or {} # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.put("/api/flow-configs/{cfg_id}")
|
||||
async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
||||
existing = await get_flow_config(cfg_id)
|
||||
if not existing:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id)
|
||||
row = await get_flow_config(cfg_id)
|
||||
return row or {} # type: ignore[return-value]
|
||||
|
||||
|
||||
@router.post("/api/flow-configs/{cfg_id}/activate", status_code=200)
|
||||
async def activate_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
||||
existing = await get_flow_config(cfg_id)
|
||||
if not existing:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
await activate_flow_config(cfg_id)
|
||||
return {"activated": cfg_id}
|
||||
|
||||
|
||||
@router.delete("/api/flow-configs/{cfg_id}", status_code=204)
|
||||
async def delete_config(cfg_id: int, _: Auth) -> None:
|
||||
existing = await get_flow_config(cfg_id)
|
||||
if not existing:
|
||||
raise HTTPException(status_code=404, detail="Not found")
|
||||
await delete_flow_config(cfg_id)
|
||||
@@ -0,0 +1,558 @@
|
||||
<!DOCTYPE html>
|
||||
<html lang="en">
|
||||
<head>
|
||||
<meta charset="UTF-8" />
|
||||
<meta name="viewport" content="width=device-width, initial-scale=1.0" />
|
||||
<title>Seed — Admin</title>
|
||||
<style>
|
||||
*, *::before, *::after { box-sizing: border-box; margin: 0; padding: 0; }
|
||||
:root {
|
||||
--bg: #0f1117; --surface: #1a1d27; --surface2: #222536;
|
||||
--border: #2a2d3e; --text: #e2e8f0; --muted: #8892a4;
|
||||
--accent: #6366f1; --green: #22c55e; --red: #ef4444; --yellow: #eab308;
|
||||
}
|
||||
body { background: var(--bg); color: var(--text); font-family: 'Inter', system-ui, sans-serif; font-size: 14px; }
|
||||
|
||||
/* ── Header ─────────────────────────────────── */
|
||||
header {
|
||||
display: flex; align-items: center; justify-content: space-between;
|
||||
padding: 14px 24px; border-bottom: 1px solid var(--border); background: var(--surface);
|
||||
position: sticky; top: 0; z-index: 50;
|
||||
}
|
||||
header h1 { font-size: 17px; font-weight: 600; }
|
||||
.badge { background: var(--accent); color: #fff; border-radius: 4px; padding: 2px 8px; font-size: 11px; font-weight: 600; }
|
||||
#last-updated { color: var(--muted); font-size: 12px; }
|
||||
|
||||
/* ── Tabs ────────────────────────────────────── */
|
||||
.tabs { display: flex; gap: 2px; padding: 0 24px; background: var(--surface); border-bottom: 1px solid var(--border); }
|
||||
.tab { padding: 11px 18px; cursor: pointer; font-size: 13px; font-weight: 500; color: var(--muted); border-bottom: 2px solid transparent; transition: color .15s; }
|
||||
.tab:hover { color: var(--text); }
|
||||
.tab.active { color: var(--accent); border-bottom-color: var(--accent); }
|
||||
|
||||
.tab-panel { display: none; }
|
||||
.tab-panel.active { display: block; }
|
||||
|
||||
main { padding: 24px; max-width: 1400px; margin: 0 auto; }
|
||||
|
||||
/* ── Cards ───────────────────────────────────── */
|
||||
.cards { display: grid; grid-template-columns: repeat(auto-fit, minmax(170px, 1fr)); gap: 16px; margin-bottom: 28px; }
|
||||
.card { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 18px 16px; }
|
||||
.card .label { color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .5px; margin-bottom: 8px; }
|
||||
.card .value { font-size: 30px; font-weight: 700; line-height: 1; }
|
||||
.card .value.green { color: var(--green); } .card .value.red { color: var(--red); } .card .value.accent { color: var(--accent); }
|
||||
|
||||
.grid-2 { display: grid; grid-template-columns: 1fr 1fr; gap: 20px; margin-bottom: 28px; }
|
||||
@media (max-width: 900px) { .grid-2 { grid-template-columns: 1fr; } }
|
||||
.panel { background: var(--surface); border: 1px solid var(--border); border-radius: 10px; padding: 20px; }
|
||||
.section-title { font-size: 11px; font-weight: 600; color: var(--muted); text-transform: uppercase; letter-spacing: .5px; margin-bottom: 14px; }
|
||||
|
||||
/* ── Tables ──────────────────────────────────── */
|
||||
table { width: 100%; border-collapse: collapse; }
|
||||
th { text-align: left; color: var(--muted); font-size: 11px; text-transform: uppercase; letter-spacing: .5px; padding: 8px 10px; border-bottom: 1px solid var(--border); }
|
||||
td { padding: 9px 10px; border-bottom: 1px solid var(--border); vertical-align: middle; }
|
||||
tr:last-child td { border-bottom: none; }
|
||||
tr:hover td { background: rgba(255,255,255,.02); }
|
||||
|
||||
.pill { display: inline-block; border-radius: 999px; padding: 2px 10px; font-size: 11px; font-weight: 600; }
|
||||
.pill.ok { background: rgba(34,197,94,.15); color: var(--green); }
|
||||
.pill.err { background: rgba(239,68,68,.15); color: var(--red); }
|
||||
.pill.active-badge { background: rgba(99,102,241,.2); color: var(--accent); }
|
||||
|
||||
.reason-row { display: flex; justify-content: space-between; align-items: center; padding: 8px 0; border-bottom: 1px solid var(--border); }
|
||||
.reason-row:last-child { border-bottom: none; }
|
||||
.reason-text { color: var(--text); font-size: 13px; flex: 1; margin-right: 12px; word-break: break-all; }
|
||||
.reason-count { color: var(--red); font-weight: 700; }
|
||||
.breakdown-row { display: flex; align-items: center; gap: 10px; padding: 8px 0; border-bottom: 1px solid var(--border); }
|
||||
.breakdown-row:last-child { border-bottom: none; }
|
||||
.bar-bg { flex: 1; background: var(--border); border-radius: 4px; height: 8px; overflow: hidden; }
|
||||
.bar-fill { height: 100%; background: var(--accent); border-radius: 4px; }
|
||||
.empty { color: var(--muted); text-align: center; padding: 32px 0; font-size: 13px; }
|
||||
|
||||
/* ── Buttons ─────────────────────────────────── */
|
||||
.btn { border: none; border-radius: 7px; padding: 9px 16px; font-size: 13px; font-weight: 600; cursor: pointer; transition: opacity .15s; }
|
||||
.btn:hover { opacity: .85; }
|
||||
.btn-primary { background: var(--accent); color: #fff; }
|
||||
.btn-ghost { background: transparent; border: 1px solid var(--border); color: var(--text); }
|
||||
.btn-danger { background: rgba(239,68,68,.15); color: var(--red); border: 1px solid rgba(239,68,68,.3); }
|
||||
.btn-sm { padding: 5px 10px; font-size: 12px; border-radius: 5px; }
|
||||
#refresh-btn { background: none; border: 1px solid var(--border); color: var(--text); border-radius: 6px; padding: 6px 12px; cursor: pointer; font-size: 13px; }
|
||||
|
||||
/* ── Flow Config panel ───────────────────────── */
|
||||
.config-list { display: flex; flex-direction: column; gap: 12px; margin-bottom: 24px; }
|
||||
.config-card {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: 10px;
|
||||
padding: 16px 18px; display: flex; align-items: flex-start; gap: 14px;
|
||||
}
|
||||
.config-card.is-active { border-color: var(--accent); }
|
||||
.config-info { flex: 1; }
|
||||
.config-name { font-weight: 600; font-size: 15px; margin-bottom: 4px; display: flex; align-items: center; gap: 8px; }
|
||||
.config-meta { color: var(--muted); font-size: 12px; }
|
||||
.config-actions { display: flex; gap: 8px; align-items: center; flex-shrink: 0; }
|
||||
|
||||
/* ── Form ────────────────────────────────────── */
|
||||
.form-overlay {
|
||||
position: fixed; inset: 0; background: rgba(0,0,0,.65); backdrop-filter: blur(4px);
|
||||
display: flex; align-items: flex-start; justify-content: center;
|
||||
z-index: 200; overflow-y: auto; padding: 40px 16px;
|
||||
}
|
||||
.form-overlay.hidden { display: none; }
|
||||
.form-box {
|
||||
background: var(--surface); border: 1px solid var(--border); border-radius: 14px;
|
||||
padding: 32px 28px; width: 680px; max-width: 100%;
|
||||
}
|
||||
.form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
|
||||
.form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
|
||||
.form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
|
||||
.fg { margin-bottom: 14px; }
|
||||
.fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
|
||||
.fg input, .fg select, .fg textarea {
|
||||
width: 100%; background: var(--bg); border: 1px solid var(--border); border-radius: 7px;
|
||||
padding: 9px 11px; color: var(--text); font-size: 13px; font-family: inherit;
|
||||
}
|
||||
.fg textarea { resize: vertical; min-height: 64px; font-family: monospace; }
|
||||
.fg input:focus, .fg select:focus, .fg textarea:focus { outline: none; border-color: var(--accent); }
|
||||
.fg .hint { color: var(--muted); font-size: 11px; margin-top: 4px; }
|
||||
.form-footer { display: flex; justify-content: flex-end; gap: 10px; margin-top: 24px; }
|
||||
|
||||
/* ── Auth modal ──────────────────────────────── */
|
||||
#auth-modal {
|
||||
position: fixed; inset: 0; background: rgba(0,0,0,.75); backdrop-filter: blur(4px);
|
||||
display: flex; align-items: center; justify-content: center; z-index: 300;
|
||||
}
|
||||
#auth-modal.hidden { display: none; }
|
||||
.modal-box { background: var(--surface); border: 1px solid var(--border); border-radius: 14px; padding: 36px 32px; width: 320px; }
|
||||
.modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
|
||||
.error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
|
||||
.error-msg.hidden { display: none; }
|
||||
</style>
|
||||
</head>
|
||||
<body>
|
||||
|
||||
<!-- Auth Modal -->
|
||||
<div id="auth-modal">
|
||||
<div class="modal-box">
|
||||
<h2>Seed Admin</h2>
|
||||
<div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
|
||||
<div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
|
||||
<button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
|
||||
<p class="error-msg hidden" id="auth-error">Invalid credentials</p>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- Flow Config Form Overlay -->
|
||||
<div class="form-overlay hidden" id="config-overlay">
|
||||
<div class="form-box">
|
||||
<h2 id="form-title">New Flow Config</h2>
|
||||
<input type="hidden" id="cfg-id" value="" />
|
||||
|
||||
<div class="fg">
|
||||
<label>Config Name</label>
|
||||
<input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label style="display:flex;align-items:center;gap:8px;cursor:pointer">
|
||||
<input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
|
||||
</label>
|
||||
</div>
|
||||
|
||||
<!-- Search -->
|
||||
<div class="form-section">Search Step</div>
|
||||
<div class="fg">
|
||||
<label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
|
||||
<textarea id="cfg-search-texts" placeholder="shoes blue sneakers running shoes"></textarea>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Search Box Selector</label>
|
||||
<input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
|
||||
<input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Type</label>
|
||||
<select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
|
||||
<input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
|
||||
</div>
|
||||
|
||||
<!-- Scroll -->
|
||||
<div class="form-section">Scroll Step (Infinite Scroll)</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
|
||||
<input id="cfg-scroll-container" type="text" placeholder=".product-list" />
|
||||
</div>
|
||||
<div class="form-row">
|
||||
<div class="fg">
|
||||
<label>Max Scrolls</label>
|
||||
<input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Scroll Pause (ms)</label>
|
||||
<input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
|
||||
</div>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
|
||||
<input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
|
||||
</div>
|
||||
|
||||
<!-- Item -->
|
||||
<div class="form-section">Item Selection</div>
|
||||
<div class="fg">
|
||||
<label>Item Selector Template</label>
|
||||
<input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
|
||||
<p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Selector Type</label>
|
||||
<select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
|
||||
</div>
|
||||
<div class="fg">
|
||||
<label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
|
||||
<textarea id="cfg-item-ids" placeholder="prod-1234 prod-5678"></textarea>
|
||||
</div>
|
||||
|
||||
<div class="form-footer">
|
||||
<button class="btn btn-ghost" id="form-cancel">Cancel</button>
|
||||
<button class="btn btn-primary" id="form-save">Save Config</button>
|
||||
</div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<!-- ═══════════════════════════════════ APP ═══════════════════════════════════ -->
|
||||
<header>
|
||||
<div style="display:flex;align-items:center;gap:10px">
|
||||
<h1>Seed</h1><span class="badge">LIVE</span>
|
||||
</div>
|
||||
<div style="display:flex;align-items:center;gap:14px">
|
||||
<span id="last-updated">–</span>
|
||||
<button id="refresh-btn">↻ Refresh</button>
|
||||
</div>
|
||||
</header>
|
||||
|
||||
<div class="tabs">
|
||||
<div class="tab active" data-tab="dashboard">Dashboard</div>
|
||||
<div class="tab" data-tab="flows">Flow Configs</div>
|
||||
</div>
|
||||
|
||||
<!-- ── Dashboard Tab ──────────────────────────────────────────── -->
|
||||
<div class="tab-panel active" id="tab-dashboard">
|
||||
<main>
|
||||
<div class="cards">
|
||||
<div class="card"><div class="label">Total Runs</div><div class="value accent" id="c-total">–</div></div>
|
||||
<div class="card"><div class="label">Unique Users</div><div class="value" id="c-unique">–</div></div>
|
||||
<div class="card"><div class="label">Successes</div><div class="value green" id="c-success">–</div></div>
|
||||
<div class="card"><div class="label">Failures</div><div class="value red" id="c-failures">–</div></div>
|
||||
<div class="card"><div class="label">Success Rate</div><div class="value" id="c-rate">–</div></div>
|
||||
</div>
|
||||
|
||||
<div class="grid-2">
|
||||
<div class="panel">
|
||||
<div class="section-title">Scenario Breakdown</div>
|
||||
<div id="scenario-breakdown"><p class="empty">No data yet.</p></div>
|
||||
</div>
|
||||
<div class="panel">
|
||||
<div class="section-title">Top Failure Reasons</div>
|
||||
<div id="failure-reasons"><p class="empty">No failures recorded.</p></div>
|
||||
</div>
|
||||
</div>
|
||||
|
||||
<div class="panel">
|
||||
<div class="section-title" style="margin-bottom:14px">Recent Runs</div>
|
||||
<div style="overflow-x:auto">
|
||||
<table>
|
||||
<thead><tr><th>#</th><th>Time</th><th>Scenario</th><th>Identifier</th><th>Status</th><th>Duration</th><th>Failure Reason</th></tr></thead>
|
||||
<tbody id="runs-tbody"><tr><td colspan="7" class="empty">No runs yet.</td></tr></tbody>
|
||||
</table>
|
||||
</div>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<!-- ── Flow Configs Tab ───────────────────────────────────────── -->
|
||||
<div class="tab-panel" id="tab-flows">
|
||||
<main>
|
||||
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:20px">
|
||||
<div>
|
||||
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
|
||||
<div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
|
||||
</div>
|
||||
<button class="btn btn-primary" id="btn-new-config">+ New Config</button>
|
||||
</div>
|
||||
|
||||
<div class="config-list" id="config-list">
|
||||
<p class="empty">No configs yet. Create one to get started.</p>
|
||||
</div>
|
||||
</main>
|
||||
</div>
|
||||
|
||||
<script>
|
||||
/* ─────────────────────────── Auth ─────────────────────────── */
|
||||
let _creds = null;
|
||||
function b64(u, p) { return btoa(`${u}:${p}`); }
|
||||
function authHeader() { return { 'Authorization': `Basic ${b64(_creds.u, _creds.p)}` }; }
|
||||
|
||||
async function apiFetch(path, opts = {}) {
|
||||
const res = await fetch(path, { ...opts, headers: { ...authHeader(), ...(opts.headers || {}) } });
|
||||
if (res.status === 401) { showAuth(); return null; }
|
||||
return res;
|
||||
}
|
||||
|
||||
function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
|
||||
function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
|
||||
|
||||
document.getElementById('login-btn').addEventListener('click', async () => {
|
||||
const u = document.getElementById('inp-user').value.trim();
|
||||
const p = document.getElementById('inp-pass').value;
|
||||
const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
|
||||
if (res.ok) {
|
||||
_creds = { u, p };
|
||||
document.getElementById('auth-error').classList.add('hidden');
|
||||
hideAuth();
|
||||
await Promise.all([loadStats(), loadConfigs()]);
|
||||
startAutoRefresh();
|
||||
} else {
|
||||
document.getElementById('auth-error').classList.remove('hidden');
|
||||
}
|
||||
});
|
||||
document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
|
||||
|
||||
/* ─────────────────────────── Tabs ─────────────────────────── */
|
||||
document.querySelectorAll('.tab').forEach(tab => {
|
||||
tab.addEventListener('click', () => {
|
||||
document.querySelectorAll('.tab').forEach(t => t.classList.remove('active'));
|
||||
document.querySelectorAll('.tab-panel').forEach(p => p.classList.remove('active'));
|
||||
tab.classList.add('active');
|
||||
document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
|
||||
if (tab.dataset.tab === 'flows') loadConfigs();
|
||||
});
|
||||
});
|
||||
|
||||
/* ─────────────────────────── Dashboard ──────────────────────── */
|
||||
async function loadStats() {
|
||||
const res = await apiFetch('/api/stats');
|
||||
if (!res || !res.ok) return;
|
||||
renderStats(await res.json());
|
||||
}
|
||||
|
||||
function renderStats(data) {
|
||||
document.getElementById('c-total').textContent = data.total_runs;
|
||||
document.getElementById('c-unique').textContent = data.unique_identifiers;
|
||||
document.getElementById('c-success').textContent = data.successes;
|
||||
document.getElementById('c-failures').textContent = data.failures;
|
||||
document.getElementById('c-rate').textContent = `${data.success_rate}%`;
|
||||
document.getElementById('last-updated').textContent = `Updated ${new Date().toLocaleTimeString()}`;
|
||||
|
||||
const sbEl = document.getElementById('scenario-breakdown');
|
||||
sbEl.innerHTML = data.scenario_breakdown.length === 0
|
||||
? '<p class="empty">No data yet.</p>'
|
||||
: data.scenario_breakdown.map(s => {
|
||||
const pct = s.total ? Math.round(s.success / s.total * 100) : 0;
|
||||
return `<div class="breakdown-row">
|
||||
<span style="width:70px;font-weight:600">${s.scenario}</span>
|
||||
<div class="bar-bg"><div class="bar-fill" style="width:${pct}%"></div></div>
|
||||
<span style="width:38px;text-align:right;color:var(--muted);font-size:12px">${pct}%</span>
|
||||
<span style="width:70px;text-align:right;color:var(--muted);font-size:12px">${s.success}/${s.total}</span>
|
||||
</div>`;
|
||||
}).join('');
|
||||
|
||||
const frEl = document.getElementById('failure-reasons');
|
||||
frEl.innerHTML = data.failure_reasons.length === 0
|
||||
? '<p class="empty">No failures recorded.</p>'
|
||||
: data.failure_reasons.map(r =>
|
||||
`<div class="reason-row"><span class="reason-text">${esc(r.reason)}</span><span class="reason-count">${r.count}</span></div>`
|
||||
).join('');
|
||||
|
||||
const tbody = document.getElementById('runs-tbody');
|
||||
tbody.innerHTML = !data.recent_runs.length
|
||||
? '<tr><td colspan="7" class="empty">No runs yet.</td></tr>'
|
||||
: data.recent_runs.map(r => `<tr>
|
||||
<td style="color:var(--muted)">${r.id}</td>
|
||||
<td style="color:var(--muted);white-space:nowrap">${fmtTime(r.created_at)}</td>
|
||||
<td><span style="text-transform:uppercase;font-size:11px;font-weight:600">${esc(r.scenario)}</span></td>
|
||||
<td style="font-family:monospace">${esc(r.identifier)}</td>
|
||||
<td><span class="pill ${r.success ? 'ok' : 'err'}">${r.success ? 'OK' : 'FAIL'}</span></td>
|
||||
<td style="color:var(--muted)">${r.total_ms ? r.total_ms + ' ms' : '–'}</td>
|
||||
<td style="color:var(--red);font-size:12px;max-width:260px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${r.failure_reason ? esc(r.failure_reason) : '–'}</td>
|
||||
</tr>`).join('');
|
||||
}
|
||||
|
||||
document.getElementById('refresh-btn').addEventListener('click', loadStats);
|
||||
let _timer = null;
|
||||
function startAutoRefresh() {
|
||||
if (_timer) clearInterval(_timer);
|
||||
_timer = setInterval(loadStats, 15000);
|
||||
}
|
||||
|
||||
/* ─────────────────────────── Flow Configs ──────────────────── */
|
||||
let _configs = [];
|
||||
|
||||
async function loadConfigs() {
|
||||
const res = await apiFetch('/api/flow-configs');
|
||||
if (!res || !res.ok) return;
|
||||
_configs = await res.json();
|
||||
renderConfigs();
|
||||
}
|
||||
|
||||
function renderConfigs() {
|
||||
const el = document.getElementById('config-list');
|
||||
if (!_configs.length) {
|
||||
el.innerHTML = '<p class="empty">No configs yet. Create one to get started.</p>';
|
||||
return;
|
||||
}
|
||||
el.innerHTML = _configs.map(c => `
|
||||
<div class="config-card ${c.is_active ? 'is-active' : ''}" id="cc-${c.id}">
|
||||
<div class="config-info">
|
||||
<div class="config-name">
|
||||
${esc(c.name)}
|
||||
${c.is_active ? '<span class="pill active-badge">ACTIVE</span>' : ''}
|
||||
</div>
|
||||
<div class="config-meta">
|
||||
Searches: <strong>${(c.search_texts_json || []).length}</strong> text(s) ·
|
||||
Items: <strong>${(c.target_item_ids_json || []).length}</strong> ID(s) ·
|
||||
Max scrolls: <strong>${c.max_scrolls}</strong> ·
|
||||
Search box: <code style="font-size:11px">${esc(c.search_box_selector || '—')}</code>
|
||||
</div>
|
||||
<div class="config-meta" style="margin-top:4px">
|
||||
Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
|
||||
</div>
|
||||
</div>
|
||||
<div class="config-actions">
|
||||
${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
|
||||
<button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
|
||||
<button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
|
||||
</div>
|
||||
</div>
|
||||
`).join('');
|
||||
}
|
||||
|
||||
async function activateConfig(id) {
|
||||
const res = await apiFetch(`/api/flow-configs/${id}/activate`, { method: 'POST' });
|
||||
if (res && res.ok) await loadConfigs();
|
||||
}
|
||||
|
||||
async function deleteConfig(id) {
|
||||
const cfg = _configs.find(c => c.id === id);
|
||||
if (!confirm(`Delete "${cfg?.name}"?`)) return;
|
||||
const res = await apiFetch(`/api/flow-configs/${id}`, { method: 'DELETE' });
|
||||
if (res && (res.ok || res.status === 204)) await loadConfigs();
|
||||
}
|
||||
|
||||
/* ─── Form ─── */
|
||||
function openNewForm() {
|
||||
document.getElementById('form-title').textContent = 'New Flow Config';
|
||||
document.getElementById('cfg-id').value = '';
|
||||
document.getElementById('cfg-name').value = '';
|
||||
document.getElementById('cfg-active').checked = false;
|
||||
document.getElementById('cfg-search-texts').value = '';
|
||||
document.getElementById('cfg-search-sel').value = '';
|
||||
document.getElementById('cfg-search-sel-type').value = 'css';
|
||||
document.getElementById('cfg-submit-sel').value = '';
|
||||
document.getElementById('cfg-submit-sel-type').value = 'css';
|
||||
document.getElementById('cfg-results-sel').value = '';
|
||||
document.getElementById('cfg-scroll-container').value = '';
|
||||
document.getElementById('cfg-max-scrolls').value = '30';
|
||||
document.getElementById('cfg-scroll-pause').value = '1200';
|
||||
document.getElementById('cfg-no-content-timeout').value = '3000';
|
||||
document.getElementById('cfg-item-sel').value = '';
|
||||
document.getElementById('cfg-item-sel-type').value = 'css';
|
||||
document.getElementById('cfg-item-ids').value = '';
|
||||
document.getElementById('config-overlay').classList.remove('hidden');
|
||||
}
|
||||
|
||||
function openEditForm(id) {
|
||||
const c = _configs.find(x => x.id === id);
|
||||
if (!c) return;
|
||||
document.getElementById('form-title').textContent = 'Edit Flow Config';
|
||||
document.getElementById('cfg-id').value = id;
|
||||
document.getElementById('cfg-name').value = c.name || '';
|
||||
document.getElementById('cfg-active').checked = !!c.is_active;
|
||||
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
|
||||
document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
|
||||
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
|
||||
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
|
||||
document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
|
||||
document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
|
||||
document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
|
||||
document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
|
||||
document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
|
||||
document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
|
||||
document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
|
||||
document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
|
||||
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
|
||||
document.getElementById('config-overlay').classList.remove('hidden');
|
||||
}
|
||||
|
||||
function closeForm() {
|
||||
document.getElementById('config-overlay').classList.add('hidden');
|
||||
}
|
||||
|
||||
function collectForm() {
|
||||
return {
|
||||
name: document.getElementById('cfg-name').value.trim(),
|
||||
is_active: document.getElementById('cfg-active').checked,
|
||||
search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean),
|
||||
search_box_selector: document.getElementById('cfg-search-sel').value.trim(),
|
||||
search_box_selector_type: document.getElementById('cfg-search-sel-type').value,
|
||||
search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
|
||||
search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
|
||||
search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
|
||||
scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
|
||||
max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
|
||||
scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
|
||||
no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
|
||||
item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
|
||||
item_selector_type: document.getElementById('cfg-item-sel-type').value,
|
||||
target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
|
||||
};
|
||||
}
|
||||
|
||||
document.getElementById('btn-new-config').addEventListener('click', openNewForm);
|
||||
document.getElementById('form-cancel').addEventListener('click', closeForm);
|
||||
document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
|
||||
|
||||
document.getElementById('form-save').addEventListener('click', async () => {
|
||||
const body = collectForm();
|
||||
if (!body.name) { alert('Config name is required.'); return; }
|
||||
|
||||
const cfgId = document.getElementById('cfg-id').value;
|
||||
const isEdit = !!cfgId;
|
||||
const url = isEdit ? `/api/flow-configs/${cfgId}` : '/api/flow-configs';
|
||||
const method = isEdit ? 'PUT' : 'POST';
|
||||
|
||||
const res = await apiFetch(url, {
|
||||
method,
|
||||
headers: { 'Content-Type': 'application/json' },
|
||||
body: JSON.stringify(body),
|
||||
});
|
||||
if (res && (res.ok || res.status === 201)) {
|
||||
closeForm();
|
||||
await loadConfigs();
|
||||
} else {
|
||||
alert('Failed to save config. Check console.');
|
||||
console.error(await res?.text());
|
||||
}
|
||||
});
|
||||
|
||||
/* ─────────────────────────── Utils ────────────────────────── */
|
||||
function esc(s) {
|
||||
return String(s ?? '').replace(/&/g,'&').replace(/</g,'<').replace(/>/g,'>').replace(/"/g,'"');
|
||||
}
|
||||
function fmtTime(iso) {
|
||||
if (!iso) return '–';
|
||||
return new Date(iso).toLocaleString();
|
||||
}
|
||||
</script>
|
||||
</body>
|
||||
</html>
|
||||
@@ -0,0 +1,38 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
from dataclasses import dataclass, field
|
||||
from dotenv import load_dotenv
|
||||
|
||||
load_dotenv()
|
||||
|
||||
|
||||
@dataclass
|
||||
class Config:
|
||||
target_url: str = os.getenv("TARGET_URL", "https://example.com")
|
||||
admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
|
||||
admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
|
||||
admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
|
||||
admin_port: int = int(os.getenv("ADMIN_PORT", "8000"))
|
||||
db_path: str = os.getenv("DB_PATH", "data/tracker.db")
|
||||
headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
|
||||
chrome_binary: str | None = os.getenv("CHROME_BINARY")
|
||||
|
||||
# Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
|
||||
phone_numbers: list[str] = field(default_factory=list)
|
||||
|
||||
# Flow steps — ordered list of step names to execute after login
|
||||
# Override in .env as: FLOW_STEPS=step_home,step_browse,step_checkout
|
||||
flow_steps: list[str] = field(default_factory=lambda: ["step_home"])
|
||||
|
||||
def __post_init__(self) -> None:
|
||||
raw_phones = os.getenv("PHONE_NUMBERS", "")
|
||||
if raw_phones:
|
||||
self.phone_numbers = [p.strip() for p in raw_phones.split(",") if p.strip()]
|
||||
|
||||
raw_steps = os.getenv("FLOW_STEPS", "")
|
||||
if raw_steps:
|
||||
self.flow_steps = [s.strip() for s in raw_steps.split(",") if s.strip()]
|
||||
|
||||
|
||||
config = Config()
|
||||
@@ -0,0 +1,83 @@
|
||||
"""Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
|
||||
from __future__ import annotations
|
||||
|
||||
import random
|
||||
import time
|
||||
from contextlib import contextmanager
|
||||
from typing import Generator
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.webdriver.chrome.options import Options
|
||||
from selenium_stealth import stealth
|
||||
|
||||
from config import config
|
||||
|
||||
_USER_AGENTS = [
|
||||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (Macintosh; Intel Mac OS X 10_15_7) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
"Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
|
||||
]
|
||||
|
||||
|
||||
def _build_options(user_agent: str, headless: bool) -> Options:
|
||||
opts = Options()
|
||||
if headless:
|
||||
opts.add_argument("--headless=new")
|
||||
opts.add_argument(f"--user-agent={user_agent}")
|
||||
opts.add_argument("--no-sandbox")
|
||||
opts.add_argument("--disable-dev-shm-usage")
|
||||
opts.add_argument("--disable-blink-features=AutomationControlled")
|
||||
opts.add_argument("--disable-infobars")
|
||||
opts.add_argument("--window-size=1920,1080")
|
||||
opts.add_experimental_option("excludeSwitches", ["enable-automation"])
|
||||
opts.add_experimental_option("useAutomationExtension", False)
|
||||
if config.chrome_binary:
|
||||
opts.binary_location = config.chrome_binary
|
||||
return opts
|
||||
|
||||
|
||||
def make_driver(headless: bool | None = None) -> uc.Chrome:
|
||||
"""Return a stealthed undetected Chrome instance."""
|
||||
use_headless = config.headless if headless is None else headless
|
||||
ua = random.choice(_USER_AGENTS)
|
||||
opts = _build_options(ua, use_headless)
|
||||
|
||||
driver = uc.Chrome(options=opts, use_subprocess=True)
|
||||
|
||||
stealth(
|
||||
driver,
|
||||
languages=["en-US", "en"],
|
||||
vendor="Google Inc.",
|
||||
platform="Win32",
|
||||
webgl_vendor="Intel Inc.",
|
||||
renderer="Intel Iris OpenGL Engine",
|
||||
fix_hairline=True,
|
||||
)
|
||||
|
||||
# Mask navigator.webdriver via CDP
|
||||
driver.execute_cdp_cmd(
|
||||
"Page.addScriptToEvaluateOnNewDocument",
|
||||
{
|
||||
"source": """
|
||||
Object.defineProperty(navigator, 'webdriver', {get: () => undefined});
|
||||
Object.defineProperty(navigator, 'plugins', {get: () => [1, 2, 3, 4, 5]});
|
||||
Object.defineProperty(navigator, 'languages', {get: () => ['en-US', 'en']});
|
||||
"""
|
||||
},
|
||||
)
|
||||
return driver
|
||||
|
||||
|
||||
@contextmanager
|
||||
def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
|
||||
"""Context manager that guarantees driver.quit() on exit."""
|
||||
driver = make_driver(headless)
|
||||
try:
|
||||
yield driver
|
||||
finally:
|
||||
driver.quit()
|
||||
|
||||
|
||||
def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
|
||||
"""Sleep for a random human-like interval."""
|
||||
time.sleep(random.uniform(lo, hi))
|
||||
@@ -0,0 +1,282 @@
|
||||
"""
|
||||
Dynamic flow executor — reads the active flow config from DB at runtime and runs:
|
||||
1. search_step : type each search text into the search box
|
||||
2. scroll_step : infinite-scroll until target item is visible or max_scrolls reached
|
||||
3. click_step : click each target item id
|
||||
"""
|
||||
from __future__ import annotations
|
||||
|
||||
import asyncio
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Any
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.common.exceptions import NoSuchElementException, TimeoutException
|
||||
from selenium.webdriver.common.action_chains import ActionChains
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.common.keys import Keys
|
||||
from selenium.webdriver.remote.webelement import WebElement
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from crawler.driver import human_delay
|
||||
|
||||
|
||||
class DynamicFlowError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepResult:
|
||||
name: str
|
||||
success: bool
|
||||
data: dict[str, Any] = field(default_factory=dict)
|
||||
error: str | None = None
|
||||
duration_ms: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class DynamicFlowResult:
|
||||
config_id: int
|
||||
config_name: str
|
||||
steps: list[StepResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return all(s.success for s in self.steps)
|
||||
|
||||
@property
|
||||
def failure_reason(self) -> str | None:
|
||||
for s in self.steps:
|
||||
if not s.success:
|
||||
return f"{s.name}: {s.error}"
|
||||
return None
|
||||
|
||||
|
||||
def _by(selector_type: str) -> str:
|
||||
return By.XPATH if selector_type.lower() == "xpath" else By.CSS_SELECTOR
|
||||
|
||||
|
||||
def _find(driver: uc.Chrome, selector: str, selector_type: str) -> WebElement:
|
||||
return driver.find_element(_by(selector_type), selector)
|
||||
|
||||
|
||||
def _find_all(driver: uc.Chrome, selector: str, selector_type: str) -> list[WebElement]:
|
||||
return driver.find_elements(_by(selector_type), selector)
|
||||
|
||||
|
||||
def _wait_for(
|
||||
driver: uc.Chrome,
|
||||
selector: str,
|
||||
selector_type: str,
|
||||
timeout: float = 15.0,
|
||||
) -> WebElement:
|
||||
return WebDriverWait(driver, timeout).until(
|
||||
EC.presence_of_element_located((_by(selector_type), selector))
|
||||
)
|
||||
|
||||
|
||||
class DynamicFlow:
|
||||
def __init__(self, driver: uc.Chrome, cfg: dict[str, Any]) -> None:
|
||||
self.driver = driver
|
||||
self.cfg = cfg
|
||||
|
||||
def run(self) -> DynamicFlowResult:
|
||||
result = DynamicFlowResult(
|
||||
config_id=self.cfg["id"],
|
||||
config_name=self.cfg["name"],
|
||||
)
|
||||
|
||||
search_texts: list[str] = self.cfg.get("search_texts_json") or []
|
||||
item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
|
||||
|
||||
for text in search_texts:
|
||||
step = self._run_search(text)
|
||||
result.steps.append(step)
|
||||
if not step.success:
|
||||
return result
|
||||
human_delay(0.8, 1.5)
|
||||
|
||||
for item_id in item_ids:
|
||||
scroll_step = self._run_scroll_and_find(item_id)
|
||||
result.steps.append(scroll_step)
|
||||
if not scroll_step.success:
|
||||
continue # try next item_id, don't abort the whole flow
|
||||
|
||||
click_step = self._run_click(item_id, scroll_step.data.get("element"))
|
||||
result.steps.append(click_step)
|
||||
human_delay(1.0, 2.5)
|
||||
|
||||
return result
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step: type into search box
|
||||
# ------------------------------------------------------------------
|
||||
def _run_search(self, text: str) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"search:{text}"
|
||||
try:
|
||||
sel = self.cfg["search_box_selector"]
|
||||
sel_type = self.cfg["search_box_selector_type"]
|
||||
if not sel:
|
||||
raise DynamicFlowError("search_box_selector is not configured")
|
||||
|
||||
box = _wait_for(self.driver, sel, sel_type)
|
||||
box.clear()
|
||||
human_delay(0.3, 0.6)
|
||||
|
||||
# Type character by character for a human feel
|
||||
for ch in text:
|
||||
box.send_keys(ch)
|
||||
time.sleep(0.04)
|
||||
|
||||
human_delay(0.4, 0.8)
|
||||
|
||||
# Submit: explicit button or Enter
|
||||
submit_sel = self.cfg.get("search_submit_selector", "")
|
||||
if submit_sel:
|
||||
submit_type = self.cfg.get("search_submit_selector_type", "css")
|
||||
btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
|
||||
btn.click()
|
||||
else:
|
||||
box.send_keys(Keys.RETURN)
|
||||
|
||||
# Wait for results container if configured
|
||||
results_sel = self.cfg.get("search_results_selector", "")
|
||||
if results_sel:
|
||||
_wait_for(self.driver, results_sel, "css", timeout=15.0)
|
||||
|
||||
human_delay(1.0, 2.0)
|
||||
return StepResult(name=name, success=True, data={"text": text},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step: scroll (infinite-scroll) until item selector matches
|
||||
# ------------------------------------------------------------------
|
||||
def _run_scroll_and_find(self, item_id: str) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"scroll_find:{item_id}"
|
||||
try:
|
||||
item_sel = self._build_item_selector(item_id)
|
||||
item_sel_type = self.cfg.get("item_selector_type", "css")
|
||||
scroll_container = self.cfg.get("scroll_container_selector", "").strip()
|
||||
max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
|
||||
pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
|
||||
no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
|
||||
|
||||
# Check if already visible before scrolling
|
||||
found = self._find_item_visible(item_sel, item_sel_type)
|
||||
if found:
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "scrolls": 0, "element": found},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
last_height = self._get_scroll_height(scroll_container)
|
||||
stale_since: float | None = None
|
||||
|
||||
for scroll_num in range(1, max_scrolls + 1):
|
||||
self._scroll_down(scroll_container)
|
||||
time.sleep(pause_ms / 1000)
|
||||
|
||||
found = self._find_item_visible(item_sel, item_sel_type)
|
||||
if found:
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "scrolls": scroll_num, "element": found},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
new_height = self._get_scroll_height(scroll_container)
|
||||
if new_height == last_height:
|
||||
if stale_since is None:
|
||||
stale_since = time.monotonic()
|
||||
elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
|
||||
raise DynamicFlowError(
|
||||
f"No new content after {no_new_ms} ms — reached end of page "
|
||||
f"without finding item '{item_id}'"
|
||||
)
|
||||
else:
|
||||
stale_since = None
|
||||
last_height = new_height
|
||||
|
||||
raise DynamicFlowError(
|
||||
f"Item '{item_id}' not found after {max_scrolls} scrolls"
|
||||
)
|
||||
|
||||
except DynamicFlowError:
|
||||
raise
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Step: click the found element
|
||||
# ------------------------------------------------------------------
|
||||
def _run_click(self, item_id: str, element: Any) -> StepResult:
|
||||
t0 = time.monotonic()
|
||||
name = f"click:{item_id}"
|
||||
try:
|
||||
if element is None:
|
||||
raise DynamicFlowError("No element reference from scroll step")
|
||||
|
||||
el: WebElement = element
|
||||
# Scroll element into view and click
|
||||
self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
|
||||
human_delay(0.3, 0.7)
|
||||
try:
|
||||
el.click()
|
||||
except Exception:
|
||||
# Fallback: JS click
|
||||
self.driver.execute_script("arguments[0].click();", el)
|
||||
|
||||
human_delay(0.8, 1.5)
|
||||
return StepResult(name=name, success=True,
|
||||
data={"item_id": item_id, "url_after": self.driver.current_url},
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
except Exception as exc:
|
||||
return StepResult(name=name, success=False, error=str(exc),
|
||||
duration_ms=int((time.monotonic() - t0) * 1000))
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Helpers
|
||||
# ------------------------------------------------------------------
|
||||
def _build_item_selector(self, item_id: str) -> str:
|
||||
template: str = self.cfg.get("item_selector_template", "")
|
||||
if not template:
|
||||
raise DynamicFlowError("item_selector_template is not configured")
|
||||
return template.replace("{item_id}", item_id)
|
||||
|
||||
def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
|
||||
try:
|
||||
els = _find_all(self.driver, selector, selector_type)
|
||||
for el in els:
|
||||
if el.is_displayed():
|
||||
return el
|
||||
except NoSuchElementException:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _get_scroll_height(self, container_sel: str) -> int:
|
||||
if container_sel:
|
||||
try:
|
||||
el = _find(self.driver, container_sel, "css")
|
||||
return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
|
||||
except Exception:
|
||||
pass
|
||||
return int(self.driver.execute_script("return document.body.scrollHeight"))
|
||||
|
||||
def _scroll_down(self, container_sel: str) -> None:
|
||||
if container_sel:
|
||||
try:
|
||||
el = _find(self.driver, container_sel, "css")
|
||||
self.driver.execute_script(
|
||||
"arguments[0].scrollTop = arguments[0].scrollHeight", el
|
||||
)
|
||||
return
|
||||
except Exception:
|
||||
pass
|
||||
self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
|
||||
@@ -0,0 +1,104 @@
|
||||
"""Executes a pre-defined sequence of flow steps and records results."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from dataclasses import dataclass, field
|
||||
from typing import Callable
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
|
||||
from config import config
|
||||
from crawler.driver import human_delay
|
||||
|
||||
# A flow step is a callable that receives the driver and returns arbitrary data.
|
||||
StepFn = Callable[[uc.Chrome], dict[str, object]]
|
||||
|
||||
# Registry — add your custom step functions here.
|
||||
# Keys must match the step names in config.flow_steps.
|
||||
STEP_REGISTRY: dict[str, StepFn] = {}
|
||||
|
||||
|
||||
def register_step(name: str) -> Callable[[StepFn], StepFn]:
|
||||
"""Decorator to register a function as a named flow step."""
|
||||
def decorator(fn: StepFn) -> StepFn:
|
||||
STEP_REGISTRY[name] = fn
|
||||
return fn
|
||||
return decorator
|
||||
|
||||
|
||||
@dataclass
|
||||
class StepResult:
|
||||
name: str
|
||||
success: bool
|
||||
data: dict[str, object] = field(default_factory=dict)
|
||||
error: str | None = None
|
||||
duration_ms: int = 0
|
||||
|
||||
|
||||
@dataclass
|
||||
class FlowResult:
|
||||
scenario: str
|
||||
identifier: str # phone number or "fresh"
|
||||
steps: list[StepResult] = field(default_factory=list)
|
||||
|
||||
@property
|
||||
def success(self) -> bool:
|
||||
return all(s.success for s in self.steps)
|
||||
|
||||
@property
|
||||
def failure_reason(self) -> str | None:
|
||||
for s in self.steps:
|
||||
if not s.success:
|
||||
return f"{s.name}: {s.error}"
|
||||
return None
|
||||
|
||||
|
||||
class FlowRunner:
|
||||
def __init__(self, driver: uc.Chrome) -> None:
|
||||
self.driver = driver
|
||||
|
||||
def run(self, scenario: str, identifier: str) -> FlowResult:
|
||||
result = FlowResult(scenario=scenario, identifier=identifier)
|
||||
|
||||
for step_name in config.flow_steps:
|
||||
fn = STEP_REGISTRY.get(step_name)
|
||||
if fn is None:
|
||||
result.steps.append(
|
||||
StepResult(
|
||||
name=step_name,
|
||||
success=False,
|
||||
error=f"Step '{step_name}' not found in registry",
|
||||
)
|
||||
)
|
||||
break
|
||||
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
data = fn(self.driver)
|
||||
duration = int((time.monotonic() - t0) * 1000)
|
||||
result.steps.append(StepResult(name=step_name, success=True, data=data, duration_ms=duration))
|
||||
human_delay(0.5, 1.5)
|
||||
except Exception as exc:
|
||||
duration = int((time.monotonic() - t0) * 1000)
|
||||
result.steps.append(
|
||||
StepResult(name=step_name, success=False, error=str(exc), duration_ms=duration)
|
||||
)
|
||||
break # abort remaining steps on first failure
|
||||
|
||||
return result
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Example steps — replace with your actual flow logic
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
@register_step("step_home")
|
||||
def step_home(driver: uc.Chrome) -> dict[str, object]:
|
||||
return {"url": driver.current_url, "title": driver.title}
|
||||
|
||||
|
||||
@register_step("step_browse")
|
||||
def step_browse(driver: uc.Chrome) -> dict[str, object]:
|
||||
# Example: navigate to a listing page
|
||||
human_delay(1.0, 2.0)
|
||||
return {"url": driver.current_url}
|
||||
@@ -0,0 +1,52 @@
|
||||
"""Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
|
||||
from __future__ import annotations
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from config import config
|
||||
from crawler.driver import human_delay, make_driver
|
||||
|
||||
|
||||
class FreshSessionScenario:
|
||||
"""
|
||||
Opens the target URL in a brand-new browser profile (no stored cookies,
|
||||
localStorage, or cache) so each run appears as a first-time visitor.
|
||||
|
||||
Because undetected-chromedriver already creates an isolated temp profile
|
||||
per instance, simply spinning up a new driver is sufficient — but we also
|
||||
explicitly delete all cookies after loading to be safe.
|
||||
"""
|
||||
|
||||
def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
|
||||
self.driver = driver
|
||||
self.wait = WebDriverWait(driver, timeout)
|
||||
|
||||
def run(self) -> dict[str, object]:
|
||||
"""Navigate to the target, clear any residual state, return page info."""
|
||||
self.driver.delete_all_cookies()
|
||||
|
||||
# Clear localStorage / sessionStorage via JS
|
||||
self.driver.execute_script(
|
||||
"try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
|
||||
)
|
||||
|
||||
self.driver.get(config.target_url)
|
||||
human_delay(2.0, 4.0)
|
||||
|
||||
# Wait for the page to reach a ready state
|
||||
self.wait.until(
|
||||
lambda d: d.execute_script("return document.readyState") == "complete"
|
||||
)
|
||||
|
||||
title = self.driver.title
|
||||
current_url = self.driver.current_url
|
||||
cookies = self.driver.get_cookies()
|
||||
|
||||
return {
|
||||
"title": title,
|
||||
"url": current_url,
|
||||
"cookie_count": len(cookies),
|
||||
"cookies": cookies,
|
||||
}
|
||||
@@ -0,0 +1,106 @@
|
||||
"""Scenario 1 — SMS-OTP login with a batch of phone numbers."""
|
||||
from __future__ import annotations
|
||||
|
||||
import time
|
||||
from typing import Callable
|
||||
|
||||
import undetected_chromedriver as uc
|
||||
from selenium.common.exceptions import TimeoutException
|
||||
from selenium.webdriver.common.by import By
|
||||
from selenium.webdriver.support import expected_conditions as EC
|
||||
from selenium.webdriver.support.ui import WebDriverWait
|
||||
|
||||
from config import config
|
||||
from crawler.driver import human_delay
|
||||
|
||||
|
||||
class OTPLoginError(Exception):
|
||||
pass
|
||||
|
||||
|
||||
class OTPLoginScenario:
|
||||
"""
|
||||
Drives an SMS-OTP login flow.
|
||||
|
||||
The caller supplies an `otp_resolver` — a callable that receives the
|
||||
phone number and returns the OTP string. Wire it up to your SMS
|
||||
gateway / SIM-management API.
|
||||
"""
|
||||
|
||||
def __init__(
|
||||
self,
|
||||
driver: uc.Chrome,
|
||||
phone: str,
|
||||
otp_resolver: Callable[[str], str],
|
||||
timeout: int = 30,
|
||||
) -> None:
|
||||
self.driver = driver
|
||||
self.phone = phone
|
||||
self.otp_resolver = otp_resolver
|
||||
self.wait = WebDriverWait(driver, timeout)
|
||||
|
||||
# ------------------------------------------------------------------
|
||||
# Override these selectors to match the target site's actual HTML.
|
||||
# ------------------------------------------------------------------
|
||||
_PHONE_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[type='tel'], input[name='phone']")
|
||||
_SEND_OTP_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
|
||||
_OTP_FIELD_SELECTOR = (By.CSS_SELECTOR, "input[name='otp'], input[autocomplete='one-time-code']")
|
||||
_CONFIRM_BTN_SELECTOR = (By.CSS_SELECTOR, "button[type='submit']")
|
||||
_SUCCESS_INDICATOR = (By.CSS_SELECTOR, "[data-testid='home'], .dashboard, #main-content")
|
||||
|
||||
def run(self) -> dict[str, object]:
|
||||
"""Execute the OTP login. Returns a result dict."""
|
||||
self.driver.get(config.target_url)
|
||||
human_delay(1.5, 3.0)
|
||||
|
||||
try:
|
||||
self._enter_phone()
|
||||
self._request_otp()
|
||||
otp = self._resolve_otp()
|
||||
self._enter_otp(otp)
|
||||
self._confirm_login()
|
||||
self._wait_for_success()
|
||||
except TimeoutException as exc:
|
||||
raise OTPLoginError(f"Timeout during OTP login for {self.phone}") from exc
|
||||
|
||||
return {"phone": self.phone, "cookies": self.driver.get_cookies()}
|
||||
|
||||
def _enter_phone(self) -> None:
|
||||
field = self.wait.until(EC.element_to_be_clickable(self._PHONE_FIELD_SELECTOR))
|
||||
field.clear()
|
||||
human_delay(0.3, 0.7)
|
||||
for char in self.phone:
|
||||
field.send_keys(char)
|
||||
time.sleep(0.05)
|
||||
|
||||
def _request_otp(self) -> None:
|
||||
btn = self.wait.until(EC.element_to_be_clickable(self._SEND_OTP_BTN_SELECTOR))
|
||||
human_delay(0.5, 1.2)
|
||||
btn.click()
|
||||
|
||||
def _resolve_otp(self) -> str:
|
||||
# Poll for the OTP from the external resolver (SMS gateway / webhook)
|
||||
for attempt in range(12):
|
||||
human_delay(5.0, 8.0)
|
||||
otp = self.otp_resolver(self.phone)
|
||||
if otp:
|
||||
return otp
|
||||
if attempt == 11:
|
||||
raise OTPLoginError(f"OTP not received for {self.phone} after 12 attempts")
|
||||
return "" # unreachable but satisfies type checker
|
||||
|
||||
def _enter_otp(self, otp: str) -> None:
|
||||
field = self.wait.until(EC.element_to_be_clickable(self._OTP_FIELD_SELECTOR))
|
||||
field.clear()
|
||||
human_delay(0.3, 0.7)
|
||||
for char in otp:
|
||||
field.send_keys(char)
|
||||
time.sleep(0.08)
|
||||
|
||||
def _confirm_login(self) -> None:
|
||||
btn = self.wait.until(EC.element_to_be_clickable(self._CONFIRM_BTN_SELECTOR))
|
||||
human_delay(0.5, 1.0)
|
||||
btn.click()
|
||||
|
||||
def _wait_for_success(self) -> None:
|
||||
self.wait.until(EC.presence_of_element_located(self._SUCCESS_INDICATOR))
|
||||
@@ -0,0 +1,121 @@
|
||||
"""Entry point — run crawl scenarios or start the admin panel."""
|
||||
from __future__ import annotations
|
||||
|
||||
import argparse
|
||||
import asyncio
|
||||
import sys
|
||||
import time
|
||||
|
||||
import uvicorn
|
||||
|
||||
from admin.models import get_active_flow_config, init_db, insert_run
|
||||
from config import config
|
||||
from crawler.driver import driver_session
|
||||
from crawler.dynamic_flow import DynamicFlow
|
||||
from crawler.scenarios.fresh_session import FreshSessionScenario
|
||||
from crawler.scenarios.otp_login import OTPLoginScenario
|
||||
|
||||
|
||||
def _load_active_flow() -> dict[str, object]:
|
||||
cfg = asyncio.run(get_active_flow_config())
|
||||
if cfg is None:
|
||||
print("[WARN] No active flow config — running without dynamic flow steps.", file=sys.stderr)
|
||||
return cfg or {}
|
||||
|
||||
|
||||
def _record(
|
||||
scenario: str,
|
||||
identifier: str,
|
||||
flow_result: object | None,
|
||||
exc: Exception | None,
|
||||
t0: float,
|
||||
) -> None:
|
||||
from crawler.dynamic_flow import DynamicFlowResult
|
||||
|
||||
total_ms = int((time.monotonic() - t0) * 1000)
|
||||
if exc is not None:
|
||||
asyncio.run(insert_run(scenario, identifier, False, str(exc), [], total_ms))
|
||||
print(f"[FAIL] {identifier}: {exc}", file=sys.stderr)
|
||||
return
|
||||
|
||||
if isinstance(flow_result, DynamicFlowResult):
|
||||
steps_data = [
|
||||
{"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
|
||||
for s in flow_result.steps
|
||||
]
|
||||
asyncio.run(insert_run(
|
||||
scenario, identifier, flow_result.success,
|
||||
flow_result.failure_reason, steps_data, total_ms,
|
||||
))
|
||||
status = "OK" if flow_result.success else f"FAIL ({flow_result.failure_reason})"
|
||||
else:
|
||||
asyncio.run(insert_run(scenario, identifier, True, None, [], total_ms))
|
||||
status = "OK (no flow config)"
|
||||
|
||||
print(f"[{status}] {identifier} — {total_ms} ms")
|
||||
|
||||
|
||||
def run_otp(phone: str) -> None:
|
||||
def otp_resolver(p: str) -> str:
|
||||
# TODO: wire up your SMS gateway / webhook here.
|
||||
raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
|
||||
|
||||
flow_cfg = _load_active_flow()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with driver_session() as driver:
|
||||
OTPLoginScenario(driver, phone, otp_resolver).run()
|
||||
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
|
||||
except Exception as exc:
|
||||
_record("otp", phone, None, exc, t0)
|
||||
return
|
||||
_record("otp", phone, result, None, t0)
|
||||
|
||||
|
||||
def run_fresh() -> None:
|
||||
flow_cfg = _load_active_flow()
|
||||
t0 = time.monotonic()
|
||||
try:
|
||||
with driver_session() as driver:
|
||||
FreshSessionScenario(driver).run()
|
||||
result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
|
||||
except Exception as exc:
|
||||
_record("fresh", "fresh", None, exc, t0)
|
||||
return
|
||||
_record("fresh", "fresh", result, None, t0)
|
||||
|
||||
|
||||
def run_admin() -> None:
|
||||
init_db()
|
||||
from admin.main import app
|
||||
uvicorn.run(app, host=config.admin_host, port=config.admin_port)
|
||||
|
||||
|
||||
def main() -> None:
|
||||
parser = argparse.ArgumentParser(description="Seed crawler & admin")
|
||||
sub = parser.add_subparsers(dest="cmd", required=True)
|
||||
|
||||
sub.add_parser("admin", help="Start the admin panel")
|
||||
|
||||
otp_p = sub.add_parser("otp", help="Run Scenario 1 (SMS-OTP login)")
|
||||
otp_p.add_argument("--phone", help="Single phone number (default: all from config)")
|
||||
|
||||
sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
|
||||
|
||||
args = parser.parse_args()
|
||||
|
||||
if args.cmd == "admin":
|
||||
run_admin()
|
||||
elif args.cmd == "otp":
|
||||
phones = [args.phone] if args.phone else config.phone_numbers
|
||||
if not phones:
|
||||
print("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone", file=sys.stderr)
|
||||
sys.exit(1)
|
||||
for phone in phones:
|
||||
run_otp(phone)
|
||||
elif args.cmd == "fresh":
|
||||
run_fresh()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
@@ -0,0 +1,24 @@
|
||||
[project]
|
||||
name = "seed"
|
||||
version = "0.1.0"
|
||||
requires-python = ">=3.11"
|
||||
dependencies = [
|
||||
"undetected-chromedriver>=3.5.5",
|
||||
"selenium>=4.18.0",
|
||||
"selenium-stealth>=1.0.6",
|
||||
"fastapi>=0.111.0",
|
||||
"uvicorn[standard]>=0.30.0",
|
||||
"aiosqlite>=0.20.0",
|
||||
"httpx>=0.27.0",
|
||||
"python-multipart>=0.0.9",
|
||||
"pydantic>=2.7.0",
|
||||
"python-dotenv>=1.0.1",
|
||||
]
|
||||
|
||||
[tool.pyright]
|
||||
pythonVersion = "3.11"
|
||||
typeCheckingMode = "strict"
|
||||
reportMissingImports = true
|
||||
reportMissingTypeStubs = false
|
||||
venvPath = "."
|
||||
venv = ".venv"
|
||||
Reference in New Issue
Block a user