nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

updates :)

Browse Source
This commit is contained in:
Navid Filsaraee
2026-06-20 17:07:48 +03:30
parent 24155ad6bf
commit 64886b2b10
17 changed files with 896 additions and 262 deletions
Download Patch File Download Diff File Expand all files Collapse all files
.gitignore
+1
View File
@@ -6,3 +6,4 @@ data/
*.db *.db
*.swp *.swp
drivers/ drivers/
/logs
Makefile
+10 -1
View File
@@ -11,7 +11,7 @@ GREEN := $(shell tput setaf 2 2>/dev/null)
CYAN := $(shell tput setaf 6 2>/dev/null) CYAN := $(shell tput setaf 6 2>/dev/null)
.PHONY: help install sync lock lint typecheck \ .PHONY: help install sync lock lint typecheck \
admin otp fresh batch \ dev admin worker otp fresh batch \
build up down logs shell \ build up down logs shell \
clean clean
@@ -50,10 +50,19 @@ patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chr
@mkdir -p drivers @mkdir -p drivers
PYTHONPATH=. $(PYTHON) scripts/patch_driver.py PYTHONPATH=. $(PYTHON) scripts/patch_driver.py
dev: ## Start admin panel + RQ worker together (Ctrl-C stops both)
@mkdir -p $(DATA_DIR)
@trap 'kill 0' INT TERM EXIT; \
$(UV) run rq worker --with-scheduler batch & \
$(PYTHON) main.py admin
admin: ## Start the admin panel (http://localhost:8000) admin: ## Start the admin panel (http://localhost:8000)
@mkdir -p $(DATA_DIR) @mkdir -p $(DATA_DIR)
$(PYTHON) main.py admin $(PYTHON) main.py admin
worker: ## Start RQ worker with scheduler (batch queue)
$(UV) run rq worker --with-scheduler batch
otp: ## Run Scenario 1 — SMS-OTP login (PHONE=+98... or all from .env) otp: ## Run Scenario 1 — SMS-OTP login (PHONE=+98... or all from .env)
@mkdir -p $(DATA_DIR) @mkdir -p $(DATA_DIR)
ifdef PHONE ifdef PHONE
admin/main.py
+20 -13
View File
@@ -1,7 +1,9 @@
"""FastAPI admin application.""" """FastAPI admin application."""
from __future__ import annotations from __future__ import annotations
import threading import os
import subprocess
import sys
from pathlib import Path from pathlib import Path
from fastapi import FastAPI from fastapi import FastAPI
@@ -18,24 +20,29 @@ app.include_router(router)
_STATIC = Path(__file__).parent / "static" _STATIC = Path(__file__).parent / "static"
app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
_rq_worker: subprocess.Popen[bytes] | None = None
@app.on_event("startup") @app.on_event("startup")
def on_startup() -> None: def on_startup() -> None:
global _rq_worker
init_db() init_db()
_start_huey_consumer() redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
_rq_worker = subprocess.Popen(
[sys.executable, "-m", "rq", "worker", "--url", redis_url, "batch"],
stdout=subprocess.DEVNULL,
stderr=subprocess.DEVNULL,
)
def _start_huey_consumer() -> None: @app.on_event("shutdown")
from huey.consumer import Consumer def on_shutdown() -> None:
from crawler.tasks import huey if _rq_worker is not None:
_rq_worker.terminate()
class _ThreadConsumer(Consumer): try:
def _set_signal_handlers(self) -> None: _rq_worker.wait(timeout=5)
pass # signal.signal() only works on the main thread except subprocess.TimeoutExpired:
_rq_worker.kill()
consumer = _ThreadConsumer(huey, workers=1, periodic=False)
t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
t.start()
@app.get("/") @app.get("/")
admin/models.py
+1 -9
View File
@@ -4,7 +4,7 @@ from __future__ import annotations
from typing import Any from typing import Any
from db.connection import DB_PATH as DB_PATH, init_db as init_db from db.connection import DB_PATH as DB_PATH, init_db as init_db
from db.repositories import flow_config_repo, runs_repo from db.repositories import flow_config_repo as flow_config_repo, runs_repo
FlowConfigRow = dict[str, Any] FlowConfigRow = dict[str, Any]
@@ -36,10 +36,6 @@ async def get_flow_config(cfg_id: int) -> FlowConfigRow | None:
return await flow_config_repo.get(cfg_id) return await flow_config_repo.get(cfg_id)
async def get_active_flow_config() -> FlowConfigRow | None:
return await flow_config_repo.get_active()
async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int: async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int:
if cfg_id is None: if cfg_id is None:
return await flow_config_repo.create(data) return await flow_config_repo.create(data)
@@ -49,7 +45,3 @@ async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) ->
async def delete_flow_config(cfg_id: int) -> None: async def delete_flow_config(cfg_id: int) -> None:
await flow_config_repo.delete(cfg_id) await flow_config_repo.delete(cfg_id)
async def activate_flow_config(cfg_id: int) -> None:
await flow_config_repo.activate(cfg_id)
admin/routes.py
+74 -29
View File
@@ -2,7 +2,6 @@
from __future__ import annotations from __future__ import annotations
import secrets import secrets
import uuid
from typing import Annotated, Any from typing import Annotated, Any
from fastapi import APIRouter, Depends, HTTPException, status from fastapi import APIRouter, Depends, HTTPException, status
@@ -10,7 +9,6 @@ from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel from pydantic import BaseModel
from admin.models import ( from admin.models import (
activate_flow_config,
delete_flow_config, delete_flow_config,
get_flow_config, get_flow_config,
get_stats, get_stats,
@@ -56,7 +54,6 @@ async def stats(_: Auth) -> dict[str, object]:
class FlowConfigIn(BaseModel): class FlowConfigIn(BaseModel):
name: str name: str
is_active: bool = False
search_texts: list[str] = [] search_texts: list[str] = []
search_box_selector: str = "" search_box_selector: str = ""
search_box_selector_type: str = "css" search_box_selector_type: str = "css"
@@ -83,7 +80,6 @@ class FlowConfigIn(BaseModel):
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
return { return {
"name": body.name, "name": body.name,
"is_active": int(body.is_active),
"search_texts_json": body.search_texts, "search_texts_json": body.search_texts,
"search_box_selector": body.search_box_selector, "search_box_selector": body.search_box_selector,
"search_box_selector_type": body.search_box_selector_type, "search_box_selector_type": body.search_box_selector_type,
@@ -138,15 +134,6 @@ async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, o
return row or {} # type: ignore[return-value] return row or {} # type: ignore[return-value]
@router.post("/api/flow-configs/{cfg_id}/activate", status_code=200)
async def activate_config(cfg_id: int, _: Auth) -> dict[str, object]:
existing = await get_flow_config(cfg_id)
if not existing:
raise HTTPException(status_code=404, detail="Not found")
await activate_flow_config(cfg_id)
return {"activated": cfg_id}
@router.delete("/api/flow-configs/{cfg_id}", status_code=204) @router.delete("/api/flow-configs/{cfg_id}", status_code=204)
async def delete_config(cfg_id: int, _: Auth) -> None: async def delete_config(cfg_id: int, _: Auth) -> None:
existing = await get_flow_config(cfg_id) existing = await get_flow_config(cfg_id)
@@ -159,32 +146,42 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
# Batch runs (managed via Huey task queue) # Batch runs (managed via Huey task queue)
# --------------------------------------------------------------------------- # ---------------------------------------------------------------------------
class BatchRunIn(BaseModel): class BatchTaskIn(BaseModel):
config_id: int
workers: int = 3 workers: int = 3
total_runs: int = 10 total_runs: int = 10
stagger_ms: int = 500 stagger_ms: int = 500
headless: bool = True headless: bool = True
@router.post("/api/run-batch", status_code=202) @router.post("/api/batch-queue", status_code=202)
async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]: async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
from crawler.tasks import batch_status as _status, run_batch_task, set_current from crawler.tasks import enqueue
current = _status() cfg = await get_flow_config(body.config_id)
if current.get("running"): if not cfg:
raise HTTPException(status_code=409, detail="A batch is already running") raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found")
from admin.models import get_active_flow_config result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
active_cfg = await get_active_flow_config() log.info("Task enqueued — config=%d runs=%d workers=%d status=%s",
if not active_cfg: body.config_id, body.total_runs, body.workers, result["status"])
raise HTTPException(status_code=422, detail="No active flow config — activate one first") return result
batch_id = uuid.uuid4().hex
result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
set_current(batch_id, result)
log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers) @router.delete("/api/batch-queue/{queue_id}", status_code=200)
return {"batch_id": batch_id, "status": "queued"} async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
from crawler.tasks import dequeue
removed = dequeue(queue_id)
if not removed:
raise HTTPException(status_code=404, detail="Queue item not found")
return {"removed": queue_id}
@router.post("/api/batch-queue/clear", status_code=200)
async def clear_batch_queue(_: Auth) -> dict[str, object]:
from crawler.tasks import clear_queue
count = clear_queue()
return {"cleared": count}
@router.post("/api/stop-batch", status_code=200) @router.post("/api/stop-batch", status_code=200)
@@ -200,3 +197,51 @@ async def stop_batch(_: Auth) -> dict[str, object]:
async def batch_status(_: Auth) -> dict[str, object]: async def batch_status(_: Auth) -> dict[str, object]:
from crawler.tasks import batch_status as _status from crawler.tasks import batch_status as _status
return _status() # type: ignore[return-value] return _status() # type: ignore[return-value]
@router.get("/api/batch-runs")
async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
from db.repositories.batch_runs import batch_runs_repo
return await batch_runs_repo.list() # type: ignore[return-value]
# ---------------------------------------------------------------------------
# Log viewer
# ---------------------------------------------------------------------------
@router.get("/api/logs")
def get_logs(
_: Auth,
n: int = 200,
level: str = "",
q: str = "",
) -> dict[str, object]:
import os
from logger import _LOG_FILE
path = _LOG_FILE
if not os.path.exists(path):
return {"lines": [], "file": path}
with open(path, encoding="utf-8", errors="replace") as f:
all_lines = f.readlines()
all_lines.reverse() # newest first
level_upper = level.upper()
q_lower = q.lower()
results: list[str] = []
for raw in all_lines:
line = raw.rstrip()
if not line:
continue
if level_upper and level_upper not in line:
continue
if q_lower and q_lower not in line.lower():
continue
results.append(line)
if len(results) >= n:
break
return {"lines": results, "file": path}
admin/static/dashboard.html
+303 -48
View File
@@ -56,7 +56,7 @@
.pill { display: inline-block; border-radius: 999px; padding: 2px 10px; font-size: 11px; font-weight: 600; } .pill { display: inline-block; border-radius: 999px; padding: 2px 10px; font-size: 11px; font-weight: 600; }
.pill.ok { background: rgba(34,197,94,.15); color: var(--green); } .pill.ok { background: rgba(34,197,94,.15); color: var(--green); }
.pill.err { background: rgba(239,68,68,.15); color: var(--red); } .pill.err { background: rgba(239,68,68,.15); color: var(--red); }
.pill.active-badge { background: rgba(99,102,241,.2); color: var(--accent); } .pill.id-badge { background: rgba(99,102,241,.12); color: var(--muted); font-family: monospace; }
.reason-row { display: flex; justify-content: space-between; align-items: center; padding: 8px 0; border-bottom: 1px solid var(--border); } .reason-row { display: flex; justify-content: space-between; align-items: center; padding: 8px 0; border-bottom: 1px solid var(--border); }
.reason-row:last-child { border-bottom: none; } .reason-row:last-child { border-bottom: none; }
@@ -83,7 +83,7 @@
background: var(--surface); border: 1px solid var(--border); border-radius: 10px; background: var(--surface); border: 1px solid var(--border); border-radius: 10px;
padding: 16px 18px; display: flex; align-items: flex-start; gap: 14px; padding: 16px 18px; display: flex; align-items: flex-start; gap: 14px;
} }
.config-card.is-active { border-color: var(--accent); } .config-card { cursor: default; }
.config-info { flex: 1; } .config-info { flex: 1; }
.config-name { font-weight: 600; font-size: 15px; margin-bottom: 4px; display: flex; align-items: center; gap: 8px; } .config-name { font-weight: 600; font-size: 15px; margin-bottom: 4px; display: flex; align-items: center; gap: 8px; }
.config-meta { color: var(--muted); font-size: 12px; } .config-meta { color: var(--muted); font-size: 12px; }
@@ -188,12 +188,6 @@
<summary>Advanced Settings</summary> <summary>Advanced Settings</summary>
<div class="adv-body"> <div class="adv-body">
<div class="fg">
<label style="display:flex;align-items:center;gap:8px;cursor:pointer">
<input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
</label>
</div>
<div class="form-section">Search Step</div> <div class="form-section">Search Step</div>
<div class="form-row"> <div class="form-row">
<div class="fg"> <div class="fg">
@@ -313,6 +307,7 @@
<div class="tab active" data-tab="dashboard">Dashboard</div> <div class="tab active" data-tab="dashboard">Dashboard</div>
<div class="tab" data-tab="flows">Flow Configs</div> <div class="tab" data-tab="flows">Flow Configs</div>
<div class="tab" data-tab="batch">Batch Run</div> <div class="tab" data-tab="batch">Batch Run</div>
<div class="tab" data-tab="logs">Logs</div>
</div> </div>
<!-- ── Dashboard Tab ──────────────────────────────────────────── --> <!-- ── Dashboard Tab ──────────────────────────────────────────── -->
@@ -355,7 +350,7 @@
<div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:20px"> <div style="display:flex;justify-content:space-between;align-items:center;margin-bottom:20px">
<div> <div>
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div> <div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
<div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div> <div style="color:var(--muted);font-size:13px">Select a config in the Batch Run tab to use it for a run.</div>
</div> </div>
<div style="display:flex;gap:8px"> <div style="display:flex;gap:8px">
<button class="btn btn-ghost" id="btn-import-config">↑ Import</button> <button class="btn btn-ghost" id="btn-import-config">↑ Import</button>
@@ -374,13 +369,22 @@
<div class="tab-panel" id="tab-batch"> <div class="tab-panel" id="tab-batch">
<main> <main>
<div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div> <div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div>
<div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using the active flow config.</div> <div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using a selected flow config.</div>
<div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px"> <div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px;align-items:start">
<!-- Left column: controls + queue -->
<div style="display:flex;flex-direction:column;gap:16px">
<!-- Controls --> <!-- Controls -->
<div class="panel" style="align-self:start"> <div class="panel">
<div class="section-title">Configuration</div> <div class="section-title">Configuration</div>
<div class="fg" style="margin-bottom:14px">
<label>Flow Config</label>
<select id="batch-config-select" style="width:100%">
<option value="">— select a config —</option>
</select>
</div>
<div class="fg" style="margin-bottom:14px"> <div class="fg" style="margin-bottom:14px">
<label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label> <label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label>
<input id="batch-workers" type="number" min="1" max="50" value="3" /> <input id="batch-workers" type="number" min="1" max="50" value="3" />
@@ -398,20 +402,37 @@
<label for="batch-headless" style="cursor:pointer;margin:0">Headless</label> <label for="batch-headless" style="cursor:pointer;margin:0">Headless</label>
</div> </div>
<div style="display:flex;gap:10px"> <div style="display:flex;gap:10px">
<button class="btn btn-primary" id="batch-start-btn" style="flex:1">▶ Start</button> <button class="btn btn-primary" id="batch-start-btn" style="flex:1">+ Add to Queue</button>
<button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button> <button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button>
</div> </div>
<div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div> <div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div>
</div> </div>
<!-- Pending Queue -->
<div class="panel" id="queue-panel">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:12px">
<div class="section-title" style="margin:0">Pending Queue <span id="queue-count-badge" style="background:var(--accent);color:#fff;border-radius:10px;padding:1px 7px;font-size:11px;margin-left:6px;display:none"></span></div>
<button class="btn btn-ghost btn-sm" id="queue-clear-btn" style="display:none">Clear All</button>
</div>
<div id="queue-list">
<p class="empty" style="font-size:12px">Queue is empty.</p>
</div>
</div>
</div> <!-- end left column -->
<!-- Status --> <!-- Status -->
<div class="panel"> <div class="panel" style="overflow-y:auto;max-height:680px">
<div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px"> <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
<div class="section-title" style="margin:0">Live Status</div> <div class="section-title" style="margin:0">Live Status</div>
<span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span> <span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span>
</div> </div>
<div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div> <div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div>
<div id="batch-status-body" style="display:none"> <div id="batch-status-body" style="display:none">
<!-- Completion banner -->
<div id="bs-completion-banner" style="display:none;border-radius:7px;padding:8px 12px;font-size:12px;font-weight:600;margin-bottom:14px"></div>
<div style="margin-bottom:16px"> <div style="margin-bottom:16px">
<div style="display:flex;justify-content:space-between;margin-bottom:6px"> <div style="display:flex;justify-content:space-between;margin-bottom:6px">
<span style="color:var(--muted);font-size:12px">Progress</span> <span style="color:var(--muted);font-size:12px">Progress</span>
@@ -436,27 +457,89 @@
</div> </div>
</div> </div>
<div style="font-size:12px;color:var(--muted);margin-bottom:14px"> <div style="font-size:12px;color:var(--muted);margin-bottom:14px">
Config: <strong id="bs-config-name">–</strong> &nbsp;·&nbsp;
Workers: <strong id="bs-workers">–</strong> &nbsp;·&nbsp; Workers: <strong id="bs-workers">–</strong> &nbsp;·&nbsp;
Started: <strong id="bs-started">–</strong> Started: <strong id="bs-started">–</strong>
<span id="bs-finished-row" style="display:none"> &nbsp;·&nbsp; Finished: <strong id="bs-finished">–</strong></span> <span id="bs-finished-row" style="display:none"> &nbsp;·&nbsp; Finished: <strong id="bs-finished">–</strong></span>
</div> </div>
<!-- Per-worker table --> <!-- Per-worker table with scroll -->
<div class="section-title">Workers</div> <div class="section-title">Workers</div>
<table id="bs-worker-table" style="font-size:12px"> <div style="max-height:320px;overflow-y:auto;border:1px solid var(--border);border-radius:7px">
<thead> <table id="bs-worker-table" style="font-size:12px">
<tr> <thead style="position:sticky;top:0;background:var(--surface);z-index:1">
<th>#</th> <tr>
<th>Status</th> <th>#</th>
<th>Duration</th> <th>Status</th>
<th>Error</th> <th>Duration</th>
</tr> <th>Error</th>
</thead> </tr>
<tbody id="bs-worker-tbody"></tbody> </thead>
</table> <tbody id="bs-worker-tbody"></tbody>
</table>
</div>
</div> </div>
</div> </div>
</div> </div>
<!-- Batch run history -->
<div class="panel" style="margin-top:20px;max-width:1100px">
<div class="section-title" style="margin-bottom:14px">Run History</div>
<div style="overflow-x:auto">
<table id="batch-history-table">
<thead>
<tr>
<th>#</th>
<th>Started</th>
<th>Config</th>
<th>Workers</th>
<th>Total</th>
<th>Succeeded</th>
<th>Failed</th>
<th>Duration</th>
<th>Status</th>
</tr>
</thead>
<tbody id="batch-history-tbody">
<tr><td colspan="9" class="empty">No batch runs yet.</td></tr>
</tbody>
</table>
</div>
</div>
</main>
</div>
<!-- ── Logs Tab ───────────────────────────────────────────────── -->
<div class="tab-panel" id="tab-logs">
<main>
<div class="panel" style="margin-bottom:20px">
<div style="display:flex;align-items:center;gap:12px;flex-wrap:wrap;margin-bottom:14px">
<span class="section-title" style="margin:0">Log Viewer</span>
<select id="log-level" style="background:var(--surface2);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:4px 8px;font-size:12px">
<option value="">All levels</option>
<option value="DEBUG">DEBUG</option>
<option value="INFO">INFO</option>
<option value="WARNING">WARNING</option>
<option value="ERROR">ERROR</option>
<option value="CRITICAL">CRITICAL</option>
</select>
<input id="log-search" type="text" placeholder="Filter…" style="background:var(--surface2);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:4px 10px;font-size:12px;width:200px" />
<select id="log-n" style="background:var(--surface2);color:var(--text);border:1px solid var(--border);border-radius:6px;padding:4px 8px;font-size:12px">
<option value="100">100 lines</option>
<option value="200" selected>200 lines</option>
<option value="500">500 lines</option>
<option value="1000">1000 lines</option>
</select>
<button class="btn" onclick="loadLogs()" style="padding:4px 14px;font-size:12px">↻ Refresh</button>
<label style="display:flex;align-items:center;gap:6px;font-size:12px;color:var(--muted);cursor:pointer">
<input type="checkbox" id="log-tail" checked style="accent-color:var(--accent)" /> Auto-refresh (5s)
</label>
<span id="log-file" style="color:var(--muted);font-size:11px;margin-left:auto"></span>
</div>
<div id="log-output" style="background:var(--bg);border:1px solid var(--border);border-radius:8px;padding:14px;height:68vh;overflow-y:auto;font-family:monospace;font-size:12px;line-height:1.6;white-space:pre-wrap;word-break:break-all">
<span style="color:var(--muted)">Loading…</span>
</div>
</div>
</main> </main>
</div> </div>
@@ -548,7 +631,9 @@ document.querySelectorAll('.tab').forEach(tab => {
tab.classList.add('active'); tab.classList.add('active');
document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active'); document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
if (tab.dataset.tab === 'flows') loadConfigs(); if (tab.dataset.tab === 'flows') loadConfigs();
if (tab.dataset.tab === 'batch') pollBatchStatus(); if (tab.dataset.tab === 'batch') { loadConfigs(); pollBatchStatus(); loadBatchHistory(); }
if (tab.dataset.tab === 'logs') { loadLogs(); _startLogPoll(); }
else _stopLogPoll();
}); });
}); });
@@ -616,6 +701,7 @@ async function loadConfigs() {
if (!res || !res.ok) return; if (!res || !res.ok) return;
_configs = await res.json(); _configs = await res.json();
renderConfigs(); renderConfigs();
_refreshBatchConfigSelect();
} }
function renderConfigs() { function renderConfigs() {
@@ -625,11 +711,11 @@ function renderConfigs() {
return; return;
} }
el.innerHTML = _configs.map(c => ` el.innerHTML = _configs.map(c => `
<div class="config-card ${c.is_active ? 'is-active' : ''}" id="cc-${c.id}"> <div class="config-card" id="cc-${c.id}">
<div class="config-info"> <div class="config-info">
<div class="config-name"> <div class="config-name">
${esc(c.name)} ${esc(c.name)}
${c.is_active ? '<span class="pill active-badge">ACTIVE</span>' : ''} <span class="pill id-badge">#${c.id}</span>
</div> </div>
<div class="config-meta"> <div class="config-meta">
Searches: <strong>${(c.search_texts_json || []).length}</strong> text(s) &nbsp;·&nbsp; Searches: <strong>${(c.search_texts_json || []).length}</strong> text(s) &nbsp;·&nbsp;
@@ -644,7 +730,6 @@ function renderConfigs() {
</div> </div>
</div> </div>
<div class="config-actions"> <div class="config-actions">
${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
<button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button> <button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
<button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button> <button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button>
<button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button> <button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button>
@@ -654,9 +739,11 @@ function renderConfigs() {
`).join(''); `).join('');
} }
async function activateConfig(id) { function _refreshBatchConfigSelect() {
const res = await apiFetch(`/api/flow-configs/${id}/activate`, { method: 'POST' }); const sel = document.getElementById('batch-config-select');
if (res && res.ok) await loadConfigs(); const cur = sel.value;
sel.innerHTML = '<option value="">— select a config —</option>' +
_configs.map(c => `<option value="${c.id}" ${String(c.id) === cur ? 'selected' : ''}>${esc(c.name)} (#${c.id})</option>`).join('');
} }
async function deleteConfig(id) { async function deleteConfig(id) {
@@ -705,7 +792,6 @@ async function duplicateConfig(id) {
if (!c) return; if (!c) return;
const body = { const body = {
name: c.name + ' (copy)', name: c.name + ' (copy)',
is_active: false,
search_texts: c.search_texts_json || [], search_texts: c.search_texts_json || [],
search_box_selector: c.search_box_selector || '', search_box_selector: c.search_box_selector || '',
search_box_selector_type: c.search_box_selector_type || 'css', search_box_selector_type: c.search_box_selector_type || 'css',
@@ -743,7 +829,6 @@ document.getElementById('import-file-input').addEventListener('change', async (e
try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; } try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; }
const body = { const body = {
name: data.name || file.name.replace(/\.json$/, ''), name: data.name || file.name.replace(/\.json$/, ''),
is_active: false,
search_texts: data.search_texts || [], search_texts: data.search_texts || [],
search_box_selector: data.search_box_selector || '', search_box_selector: data.search_box_selector || '',
search_box_selector_type: data.search_box_selector_type || 'css', search_box_selector_type: data.search_box_selector_type || 'css',
@@ -772,7 +857,6 @@ document.getElementById('import-file-input').addEventListener('change', async (e
/* ─── Form ─── */ /* ─── Form ─── */
function _resetAdvanced() { function _resetAdvanced() {
document.getElementById('cfg-active').checked = false;
document.getElementById('cfg-search-sel').value = ''; document.getElementById('cfg-search-sel').value = '';
document.getElementById('cfg-search-sel-type').value = 'css'; document.getElementById('cfg-search-sel-type').value = 'css';
document.getElementById('cfg-submit-sel').value = ''; document.getElementById('cfg-submit-sel').value = '';
@@ -815,7 +899,6 @@ function openEditForm(id) {
document.getElementById('cfg-target-url').value = c.target_url || ''; document.getElementById('cfg-target-url').value = c.target_url || '';
document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n'); document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n'); document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
document.getElementById('cfg-active').checked = !!c.is_active;
document.getElementById('cfg-search-sel').value = c.search_box_selector || ''; document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css'; document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
document.getElementById('cfg-submit-sel').value = c.search_submit_selector || ''; document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
@@ -847,7 +930,6 @@ function closeForm() {
function collectForm() { function collectForm() {
return { return {
name: document.getElementById('cfg-name').value.trim(), name: document.getElementById('cfg-name').value.trim(),
is_active: document.getElementById('cfg-active').checked,
search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean), search_texts: document.getElementById('cfg-search-texts').value.split('\n').map(s => s.trim()).filter(Boolean),
search_box_selector: document.getElementById('cfg-search-sel').value.trim(), search_box_selector: document.getElementById('cfg-search-sel').value.trim(),
search_box_selector_type: document.getElementById('cfg-search-sel-type').value, search_box_selector_type: document.getElementById('cfg-search-sel-type').value,
@@ -939,14 +1021,16 @@ function renderBatchStatus(s) {
const idle = document.getElementById('batch-idle-msg'); const idle = document.getElementById('batch-idle-msg');
const body = document.getElementById('batch-status-body'); const body = document.getElementById('batch-status-body');
const errEl = document.getElementById('batch-error'); const errEl = document.getElementById('batch-error');
const startBtn = document.getElementById('batch-start-btn');
const stopBtn = document.getElementById('batch-stop-btn'); const stopBtn = document.getElementById('batch-stop-btn');
const badge = document.getElementById('bs-active-badge'); const badge = document.getElementById('bs-active-badge');
const banner = document.getElementById('bs-completion-banner');
// Always render queue
renderQueue(s.queue || []);
if (!s.batch_id) { if (!s.batch_id) {
idle.style.display = ''; idle.style.display = '';
body.style.display = 'none'; body.style.display = 'none';
startBtn.style.display = '';
stopBtn.style.display = 'none'; stopBtn.style.display = 'none';
badge.style.display = 'none'; badge.style.display = 'none';
return; return;
@@ -956,11 +1040,16 @@ function renderBatchStatus(s) {
body.style.display = ''; body.style.display = '';
const pct = s.total ? Math.round(s.completed / s.total * 100) : 0; const pct = s.total ? Math.round(s.completed / s.total * 100) : 0;
const barColor = !s.running
? (s.stopping ? 'var(--yellow)' : s.error ? 'var(--red)' : 'var(--green)')
: 'var(--accent)';
document.getElementById('batch-progress-bar').style.background = barColor;
document.getElementById('batch-progress-bar').style.width = pct + '%'; document.getElementById('batch-progress-bar').style.width = pct + '%';
document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`; document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`;
document.getElementById('bs-completed').textContent = s.completed ?? 0; document.getElementById('bs-completed').textContent = s.completed ?? 0;
document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0; document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0;
document.getElementById('bs-failed').textContent = s.failed ?? 0; document.getElementById('bs-failed').textContent = s.failed ?? 0;
document.getElementById('bs-config-name').textContent = s.config_name || '–';
document.getElementById('bs-workers').textContent = s.configured_workers ?? '–'; document.getElementById('bs-workers').textContent = s.configured_workers ?? '–';
document.getElementById('bs-started').textContent = fmtTs(s.started_at); document.getElementById('bs-started').textContent = fmtTs(s.started_at);
@@ -972,6 +1061,23 @@ function renderBatchStatus(s) {
finRow.style.display = 'none'; finRow.style.display = 'none';
} }
// Completion banner
if (!s.running && s.finished_at) {
let msg, bg, color;
if (s.stopping) {
msg = '■ Stopped'; bg = 'rgba(234,179,8,.15)'; color = 'var(--yellow)';
} else if (s.error) {
msg = '✕ Error: ' + s.error; bg = 'rgba(239,68,68,.12)'; color = 'var(--red)';
} else {
msg = `✓ Batch complete — ${s.succeeded ?? 0} succeeded, ${s.failed ?? 0} failed`;
bg = 'rgba(34,197,94,.12)'; color = 'var(--green)';
}
banner.textContent = msg;
banner.style.cssText = `display:block;border-radius:7px;padding:8px 12px;font-size:12px;font-weight:600;margin-bottom:14px;background:${bg};color:${color}`;
} else {
banner.style.display = 'none';
}
// Active worker badge // Active worker badge
const active = s.active_workers ?? 0; const active = s.active_workers ?? 0;
if (s.running && active > 0) { if (s.running && active > 0) {
@@ -992,7 +1098,7 @@ function renderBatchStatus(s) {
<td style="font-family:monospace">${w.slot}</td> <td style="font-family:monospace">${w.slot}</td>
<td style="color:${color};font-weight:600">${label}</td> <td style="color:${color};font-weight:600">${label}</td>
<td>${fmtDuration(w.duration_ms)}</td> <td>${fmtDuration(w.duration_ms)}</td>
<td style="max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td> <td style="max-width:200px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td>
</tr>`; </tr>`;
}).join(''); }).join('');
@@ -1004,16 +1110,52 @@ function renderBatchStatus(s) {
} }
if (s.running) { if (s.running) {
startBtn.style.display = 'none';
stopBtn.style.display = ''; stopBtn.style.display = '';
stopBtn.disabled = s.stopping || false; stopBtn.disabled = s.stopping || false;
stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop'; stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop';
} else { } else {
startBtn.style.display = '';
stopBtn.style.display = 'none'; stopBtn.style.display = 'none';
} }
} }
function renderQueue(queue) {
const list = document.getElementById('queue-list');
const badge = document.getElementById('queue-count-badge');
const clearBtn = document.getElementById('queue-clear-btn');
if (!queue.length) {
list.innerHTML = '<p class="empty" style="font-size:12px">Queue is empty.</p>';
badge.style.display = 'none';
clearBtn.style.display = 'none';
return;
}
badge.textContent = queue.length;
badge.style.display = '';
clearBtn.style.display = '';
list.innerHTML = queue.map((t, i) => `
<div style="display:flex;align-items:center;gap:8px;padding:8px 0;border-bottom:1px solid var(--border)">
<span style="color:var(--muted);font-size:11px;width:18px;text-align:right">${i + 1}</span>
<div style="flex:1;min-width:0">
<div style="font-weight:600;font-size:12px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${esc(t.config_name || '–')}</div>
<div style="color:var(--muted);font-size:11px">${t.workers}w · ${t.total_runs} runs · ${t.stagger_ms}ms stagger</div>
</div>
<button class="btn btn-danger btn-sm" onclick="removeQueueItem('${t.queue_id}')">✕</button>
</div>
`).join('').replace(/<div[^>]*>.*?<\/div>\s*$/, s => s.replace('border-bottom:1px solid var(--border)', ''));
}
async function removeQueueItem(queueId) {
const res = await apiFetch(`/api/batch-queue/${queueId}`, { method: 'DELETE' });
if (res && res.ok) await pollBatchStatus();
}
document.getElementById('queue-clear-btn').addEventListener('click', async () => {
const res = await apiFetch('/api/batch-queue/clear', { method: 'POST' });
if (res && res.ok) await pollBatchStatus();
});
async function pollBatchStatus() { async function pollBatchStatus() {
const res = await apiFetch('/api/batch-status'); const res = await apiFetch('/api/batch-status');
if (!res || !res.ok) return; if (!res || !res.ok) return;
@@ -1032,27 +1174,31 @@ function startBatchPoll() {
if (!st.running) { if (!st.running) {
clearInterval(_batchPollTimer); clearInterval(_batchPollTimer);
_batchPollTimer = null; _batchPollTimer = null;
// Delay history reload to allow the DB finish() write to complete
setTimeout(loadBatchHistory, 2000);
} }
}, 1500); }, 1500);
} }
document.getElementById('batch-start-btn').addEventListener('click', async () => { document.getElementById('batch-start-btn').addEventListener('click', async () => {
const configId = parseInt(document.getElementById('batch-config-select').value);
if (!configId) { alert('Please select a flow config.'); return; }
const workers = parseInt(document.getElementById('batch-workers').value) || 3; const workers = parseInt(document.getElementById('batch-workers').value) || 3;
const runs = parseInt(document.getElementById('batch-runs').value) || 10; const runs = parseInt(document.getElementById('batch-runs').value) || 10;
const stagger = parseInt(document.getElementById('batch-stagger').value) || 0; const stagger = parseInt(document.getElementById('batch-stagger').value) || 0;
const headless = document.getElementById('batch-headless').checked; const headless = document.getElementById('batch-headless').checked;
const res = await apiFetch('/api/run-batch', { const res = await apiFetch('/api/batch-queue', {
method: 'POST', method: 'POST',
headers: { 'Content-Type': 'application/json' }, headers: { 'Content-Type': 'application/json' },
body: JSON.stringify({ workers, total_runs: runs, stagger_ms: stagger, headless }), body: JSON.stringify({ config_id: configId, workers, total_runs: runs, stagger_ms: stagger, headless }),
}); });
if (!res) return; if (!res) return;
if (res.status === 409) { alert('A batch is already running.'); return; } if (res.status === 404) { const d = await res.json(); alert(d.detail || 'Flow config not found.'); return; }
if (res.status === 422) { const d = await res.json(); alert(d.detail || 'No active flow config.'); return; } if (!res.ok) { alert('Failed to add task.'); return; }
if (!res.ok) { alert('Failed to start batch.'); return; }
const data = await res.json();
await pollBatchStatus(); await pollBatchStatus();
startBatchPoll(); startBatchPoll();
}); });
@@ -1067,6 +1213,115 @@ document.getElementById('batch-stop-btn').addEventListener('click', async () =>
btn.textContent = '■ Stop'; btn.textContent = '■ Stop';
} }
}); });
/* ──────────────────────── Batch Run History ────────────────────── */
async function loadBatchHistory() {
const res = await apiFetch('/api/batch-runs');
if (!res || !res.ok) return;
renderBatchHistory(await res.json());
}
/* ─────────────────────────── Logs ──────────────────────────── */
let _logPollTimer = null;
const _LEVEL_COLORS = {
DEBUG: '#8892a4',
INFO: '#6ee7b7',
WARNING: '#eab308',
ERROR: '#ef4444',
CRITICAL: '#f97316',
};
function _colorLine(line) {
for (const [lvl, col] of Object.entries(_LEVEL_COLORS)) {
if (line.includes(lvl)) {
return `<span style="color:${col}">${esc(line)}</span>`;
}
}
return `<span style="color:var(--muted)">${esc(line)}</span>`;
}
async function loadLogs() {
const level = document.getElementById('log-level').value;
const q = document.getElementById('log-search').value;
const n = document.getElementById('log-n').value;
const params = new URLSearchParams({ n, level, q });
const res = await apiFetch(`/api/logs?${params}`);
if (!res || !res.ok) return;
const data = await res.json();
document.getElementById('log-file').textContent = data.file || '';
const out = document.getElementById('log-output');
if (!data.lines.length) {
out.innerHTML = '<span style="color:var(--muted)">No log entries.</span>';
return;
}
out.innerHTML = data.lines.map(_colorLine).join('\n');
}
function _startLogPoll() {
_stopLogPoll();
_logPollTimer = setInterval(() => {
if (document.getElementById('log-tail').checked) loadLogs();
}, 5000);
}
function _stopLogPoll() {
if (_logPollTimer) { clearInterval(_logPollTimer); _logPollTimer = null; }
}
document.addEventListener('DOMContentLoaded', () => {
document.getElementById('log-tail').addEventListener('change', e => {
if (e.target.checked) _startLogPoll(); else _stopLogPoll();
});
['log-level', 'log-n'].forEach(id =>
document.getElementById(id).addEventListener('change', loadLogs)
);
let _debounce;
document.getElementById('log-search').addEventListener('input', () => {
clearTimeout(_debounce);
_debounce = setTimeout(loadLogs, 300);
});
});
/* ──────────────────────── Batch Run History ────────────────────── */
function renderBatchHistory(rows) {
const tbody = document.getElementById('batch-history-tbody');
if (!rows.length) {
tbody.innerHTML = '<tr><td colspan="9" class="empty">No batch runs yet.</td></tr>';
return;
}
tbody.innerHTML = rows.map(r => {
const dur = r.finished_at && r.started_at
? fmtDuration((new Date(r.finished_at) - new Date(r.started_at)))
: '–';
let statusLabel, statusColor;
if (!r.finished_at) {
statusLabel = '⟳ running'; statusColor = 'var(--accent)';
} else if (r.stopped) {
statusLabel = '■ stopped'; statusColor = 'var(--yellow)';
} else if (r.error) {
statusLabel = '✕ error'; statusColor = 'var(--red)';
} else if (r.failed > 0 && r.succeeded === 0) {
statusLabel = 'all failed'; statusColor = 'var(--red)';
} else {
statusLabel = '✓ done'; statusColor = 'var(--green)';
}
const configLabel = r.config_name
? `${esc(r.config_name)} <span style="color:var(--muted);font-size:11px">#${r.config_id}</span>`
: `<span style="color:var(--muted)">#${r.config_id ?? '–'}</span>`;
return `<tr>
<td style="color:var(--muted);font-family:monospace;font-size:11px">${r.batch_id.slice(0,8)}</td>
<td style="white-space:nowrap;color:var(--muted)">${fmtTime(r.started_at)}</td>
<td>${configLabel}</td>
<td style="text-align:center">${r.workers}</td>
<td style="text-align:center">${r.total_runs}</td>
<td style="text-align:center;color:var(--green);font-weight:600">${r.succeeded}</td>
<td style="text-align:center;color:var(--red);font-weight:600">${r.failed}</td>
<td style="color:var(--muted)">${dur}</td>
<td style="color:${statusColor};font-weight:600">${statusLabel}</td>
</tr>`;
}).join('');
}
</script> </script>
</body> </body>
</html> </html>
config.py
+2 -1
View File
@@ -2,6 +2,7 @@ from __future__ import annotations
import os import os
from dataclasses import dataclass, field from dataclasses import dataclass, field
from dotenv import load_dotenv from dotenv import load_dotenv
load_dotenv() load_dotenv()
@@ -12,7 +13,7 @@ class Config:
admin_username: str = os.getenv("ADMIN_USERNAME", "admin") admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme") admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1") admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
admin_port: int = int(os.getenv("ADMIN_PORT", "8000")) admin_port: int = int(os.getenv("ADMIN_PORT", "9000"))
db_path: str = os.getenv("DB_PATH", "data/tracker.db") db_path: str = os.getenv("DB_PATH", "data/tracker.db")
headless: bool = os.getenv("HEADLESS", "true").lower() == "true" headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
chrome_binary: str | None = os.getenv("CHROME_BINARY") chrome_binary: str | None = os.getenv("CHROME_BINARY")
crawler/scenarios/fresh_session.py
+3
View File
@@ -31,6 +31,9 @@ class FreshSessionScenario:
"try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}" "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
) )
if not self.url:
return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
self.driver.get(self.url) self.driver.get(self.url)
human_delay(2.0, 4.0) human_delay(2.0, 4.0)
crawler/tasks.py
+299 -113
View File
@@ -1,137 +1,306 @@
"""Huey task definitions and shared batch state for the admin panel.""" """Batch task execution via RQ (Redis Queue) — fully decoupled from FastAPI's event loop."""
from __future__ import annotations from __future__ import annotations
import threading import json
import os
import sqlite3
import time import time
from concurrent.futures import ThreadPoolExecutor, as_completed import uuid
from typing import Any from typing import Any
from huey import MemoryHuey import redis
from rq import Queue
huey = MemoryHuey("seed", immediate=False) from db.connection import DB_PATH
# ── Shared state (all accessed under _lock) ─────────────────────────────────── REDIS_URL = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
_lock = threading.Lock()
_current_batch_id: str | None = None # Redis key helpers
_current_result: Any = None # Huey Result — kept for revocation _K_CURRENT = "seed:current" # string: current batch_id (or empty)
_K_META = "seed:meta:{}" # JSON string: batch aggregate meta
_K_SLOTS = "seed:slots:{}" # hash: slot_key → JSON worker info
_K_STOP = "seed:stop:{}" # string: "1" if stop requested
_K_QUEUE = "seed:pending_queue" # list: JSON-encoded pending items
_batch_meta: dict[str, Any] = {} # batch_id → aggregate metadata
_worker_slots: dict[str, dict] = {} # f"{batch_id}:{slot:04d}" → worker info def _r() -> redis.Redis: # type: ignore[type-arg]
_stop_events: dict[str, threading.Event] = {} return redis.from_url(REDIS_URL, decode_responses=True)
def _q() -> Queue:
return Queue("batch", connection=redis.from_url(REDIS_URL))
# ── Sync DB helpers ────────────────────────────────────────────────────────────
def _db_insert_run(
scenario: str,
identifier: str,
success: bool,
failure_reason: str | None,
steps: list[dict[str, Any]],
total_ms: int,
) -> None:
conn = sqlite3.connect(DB_PATH)
try:
conn.execute(
"INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms)"
" VALUES (?,?,?,?,?,?)",
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
)
conn.commit()
finally:
conn.close()
def _db_insert_batch_run(
batch_id: str,
config_id: int | None,
config_name: str,
workers: int,
total_runs: int,
stagger_ms: int,
headless: bool,
) -> None:
conn = sqlite3.connect(DB_PATH)
try:
conn.execute(
"INSERT INTO batch_runs"
" (batch_id, config_id, config_name, workers, total_runs, stagger_ms, headless)"
" VALUES (?,?,?,?,?,?,?)",
(batch_id, config_id, config_name, workers, total_runs, stagger_ms, int(headless)),
)
conn.commit()
finally:
conn.close()
def _db_finish_batch_run(
batch_id: str,
succeeded: int,
failed: int,
stopped: bool,
error: str | None,
) -> None:
conn = sqlite3.connect(DB_PATH)
try:
conn.execute(
"UPDATE batch_runs"
" SET finished_at=strftime('%Y-%m-%dT%H:%M:%SZ','now'),"
" succeeded=?, failed=?, stopped=?, error=?"
" WHERE batch_id=?",
(succeeded, failed, int(stopped), error, batch_id),
)
conn.commit()
finally:
conn.close()
# ── Public helpers ───────────────────────────────────────────────────────────── # ── Public helpers ─────────────────────────────────────────────────────────────
def get_current_batch_id() -> str | None: def get_queue() -> list[dict[str, Any]]:
with _lock: r = _r()
return _current_batch_id raw = r.lrange(_K_QUEUE, 0, -1)
items = []
for s in raw:
try:
item = json.loads(s)
items.append({k: v for k, v in item.items() if k != "cfg"})
except Exception:
pass
return items
def set_current(batch_id: str, result: Any) -> None: def enqueue(
global _current_batch_id, _current_result cfg: dict[str, Any],
with _lock: workers: int,
_current_batch_id = batch_id total_runs: int,
_current_result = result stagger_ms: int,
headless: bool,
) -> dict[str, Any]:
queue_id = uuid.uuid4().hex
item: dict[str, Any] = {
"queue_id": queue_id,
"cfg": cfg,
"config_id": cfg.get("id"),
"config_name": cfg.get("name") or "",
"workers": workers,
"total_runs": total_runs,
"stagger_ms": stagger_ms,
"headless": headless,
"added_at": time.time(),
}
r = _r()
current = r.get(_K_CURRENT)
if current:
meta_raw = r.get(_K_META.format(current))
if meta_raw:
meta = json.loads(meta_raw)
if meta.get("running"):
r.rpush(_K_QUEUE, json.dumps(item))
return {"queue_id": queue_id, "status": "queued"}
_launch(item)
return {"queue_id": queue_id, "status": "started"}
def dequeue(queue_id: str) -> bool:
r = _r()
raw = r.lrange(_K_QUEUE, 0, -1)
for s in raw:
try:
item = json.loads(s)
if item.get("queue_id") == queue_id:
r.lrem(_K_QUEUE, 1, s)
return True
except Exception:
pass
return False
def clear_queue() -> int:
r = _r()
raw = r.lrange(_K_QUEUE, 0, -1)
count = len(raw)
r.delete(_K_QUEUE)
return count
def stop_current() -> bool: def stop_current() -> bool:
"""Signal the running batch to stop. Returns True if there was something to stop.""" r = _r()
with _lock: current = r.get(_K_CURRENT)
bid = _current_batch_id if not current:
result = _current_result
if bid is None:
return False return False
meta_raw = r.get(_K_META.format(current))
# Revoke task if it hasn't started yet (still pending in Huey queue) if not meta_raw:
if result is not None: return False
try: meta = json.loads(meta_raw)
result.revoke(revoke_once=True) if not meta.get("running"):
except Exception: return False
pass r.set(_K_STOP.format(current), "1")
meta["stopping"] = True
# Signal running workers to stop after their current step r.set(_K_META.format(current), json.dumps(meta))
ev = _stop_events.get(bid)
if ev:
ev.set()
with _lock:
if bid in _batch_meta:
_batch_meta[bid]["stopping"] = True
return True return True
def batch_status() -> dict[str, Any]: def batch_status() -> dict[str, Any]:
with _lock: r = _r()
bid = _current_batch_id bid = r.get(_K_CURRENT)
if bid is None: if not bid:
return {"batch_id": None, "running": False} return {"batch_id": None, "running": False, "queue": get_queue()}
meta = dict(_batch_meta.get(bid, {})) meta_raw = r.get(_K_META.format(bid))
prefix = f"{bid}:" meta = json.loads(meta_raw) if meta_raw else {}
workers = [dict(v) for k, v in _worker_slots.items() if k.startswith(prefix)]
slot_data = r.hgetall(_K_SLOTS.format(bid))
workers = []
for v in slot_data.values():
try:
workers.append(json.loads(v))
except Exception:
pass
meta["batch_id"] = bid meta["batch_id"] = bid
meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0)) meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0))
meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running") meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running")
meta["queue"] = get_queue()
return meta return meta
# ── Huey task ───────────────────────────────────────────────────────────────── # ── Internal ───────────────────────────────────────────────────────────────────
@huey.task(name="run_batch") def _launch(item: dict[str, Any]) -> None:
def run_batch_task( batch_id = uuid.uuid4().hex
batch_id: str, r = _r()
cfg: dict[str, Any], r.set(_K_CURRENT, batch_id)
workers: int,
total_runs: int, q = _q()
headless: bool, q.enqueue(
stagger_ms: int, run_batch_job,
) -> dict[str, Any]: batch_id,
import asyncio item,
job_id=batch_id,
job_timeout=-1,
result_ttl=3600,
)
def _start_next_in_queue() -> None:
r = _r()
raw = r.lpop(_K_QUEUE)
if not raw:
return
try:
item = json.loads(raw)
except Exception:
return
_launch(item)
# ── RQ job (runs in worker process, no asyncio) ────────────────────────────────
def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
"""Top-level RQ job function. Runs in a forked worker process."""
import threading
from concurrent.futures import ThreadPoolExecutor, as_completed
from crawler.driver import driver_session from crawler.driver import driver_session
from crawler.dynamic_flow import DynamicFlow from crawler.dynamic_flow import DynamicFlow
from crawler.scenarios.fresh_session import FreshSessionScenario from crawler.scenarios.fresh_session import FreshSessionScenario
from db.repositories.runs import runs_repo
from logger import get_logger from logger import get_logger
log = get_logger(__name__) log = get_logger(__name__)
stop_ev = threading.Event() r = redis.from_url(REDIS_URL, decode_responses=True)
with _lock:
_stop_events[batch_id] = stop_ev cfg = item["cfg"]
_batch_meta[batch_id] = { workers = item["workers"]
"running": True, total_runs = item["total_runs"]
"stopping": False, stagger_ms = item["stagger_ms"]
"total": total_runs, headless = item["headless"]
"completed": 0, config_id: int | None = int(cfg["id"]) if cfg and cfg.get("id") is not None else None
"succeeded": 0, config_name: str = str(cfg.get("name") or "") if cfg else ""
"failed": 0,
"configured_workers": workers, _db_insert_batch_run(batch_id, config_id, config_name, workers, total_runs, stagger_ms, headless)
"started_at": time.time(),
"finished_at": None, meta: dict[str, Any] = {
"error": None, "running": True,
} "stopping": False,
"total": total_runs,
"completed": 0,
"succeeded": 0,
"failed": 0,
"configured_workers": workers,
"config_id": config_id,
"config_name": config_name,
"started_at": time.time(),
"finished_at": None,
"error": None,
}
r.set(_K_META.format(batch_id), json.dumps(meta))
slots_key = _K_SLOTS.format(batch_id)
stop_key = _K_STOP.format(batch_id)
meta_key = _K_META.format(batch_id)
_slots_lock = threading.Lock()
def _run_one(slot: int) -> bool: def _run_one(slot: int) -> bool:
slot_key = f"{batch_id}:{slot:04d}" slot_key = f"{slot:04d}"
if stop_ev.is_set(): if r.exists(stop_key):
with _lock: r.hset(slots_key, slot_key, json.dumps({
_worker_slots[slot_key] = { "slot": slot, "status": "skipped",
"slot": slot, "status": "skipped", "started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped",
"started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped", }))
}
return False return False
with _lock: r.hset(slots_key, slot_key, json.dumps({
_worker_slots[slot_key] = { "slot": slot, "status": "running",
"slot": slot, "status": "running", "started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None,
"started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None, }))
}
t0 = time.monotonic() t0 = time.monotonic()
identifier = f"batch-{batch_id[:8]}-{slot:04d}" identifier = f"batch-{batch_id[:8]}-{slot:04d}"
@@ -150,24 +319,23 @@ def run_batch_task(
{"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms} {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
for s in (result.steps if result else []) for s in (result.steps if result else [])
] ]
asyncio.run(runs_repo.insert("batch", identifier, success, reason, steps, total_ms)) _db_insert_run("batch", identifier, success, reason, steps, total_ms)
status = "ok" if success else "failed" status = "ok" if success else "failed"
with _lock: r.hset(slots_key, slot_key, json.dumps({
_worker_slots[slot_key].update({ "slot": slot, "status": status,
"status": status, "finished_at": time.time(), "duration_ms": total_ms, "finished_at": time.time(), "duration_ms": total_ms, "error": None,
}) }))
log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms) log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms)
return success return success
except Exception as exc: except Exception as exc:
total_ms = int((time.monotonic() - t0) * 1000) total_ms = int((time.monotonic() - t0) * 1000)
asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms)) _db_insert_run("batch", identifier, False, str(exc), [], total_ms)
with _lock: r.hset(slots_key, slot_key, json.dumps({
_worker_slots[slot_key].update({ "slot": slot, "status": "error", "error": str(exc),
"status": "error", "error": str(exc), "finished_at": time.time(), "duration_ms": total_ms,
"finished_at": time.time(), "duration_ms": total_ms, }))
})
log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc) log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc)
return False return False
@@ -181,12 +349,17 @@ def run_batch_task(
) as pool: ) as pool:
futures = [] futures = []
for i in range(total_runs): for i in range(total_runs):
if stop_ev.is_set(): if r.exists(stop_key):
break break
futures.append(pool.submit(_run_one, i)) futures.append(pool.submit(_run_one, i))
if stagger_ms > 0 and i < total_runs - 1: if stagger_ms > 0 and i < total_runs - 1:
if stop_ev.wait(timeout=stagger_ms / 1000): deadline = time.monotonic() + stagger_ms / 1000
break # stop was signalled during the stagger delay while time.monotonic() < deadline:
if r.exists(stop_key):
break
time.sleep(0.1)
if r.exists(stop_key):
break
for fut in as_completed(futures): for fut in as_completed(futures):
try: try:
@@ -197,18 +370,31 @@ def run_batch_task(
succeeded += 1 succeeded += 1
else: else:
failed += 1 failed += 1
with _lock:
_batch_meta[batch_id]["completed"] = succeeded + failed current_meta_raw = r.get(meta_key)
_batch_meta[batch_id]["succeeded"] = succeeded if current_meta_raw:
_batch_meta[batch_id]["failed"] = failed current_meta = json.loads(current_meta_raw)
current_meta["completed"] = succeeded + failed
current_meta["succeeded"] = succeeded
current_meta["failed"] = failed
r.set(meta_key, json.dumps(current_meta))
except Exception as exc: except Exception as exc:
with _lock: current_meta_raw = r.get(meta_key)
_batch_meta[batch_id]["error"] = str(exc) if current_meta_raw:
current_meta = json.loads(current_meta_raw)
current_meta["error"] = str(exc)
r.set(meta_key, json.dumps(current_meta))
finally: finally:
with _lock: final_meta_raw = r.get(meta_key)
_batch_meta[batch_id]["running"] = False error_msg: str | None = None
_batch_meta[batch_id]["finished_at"] = time.time() if final_meta_raw:
final_meta = json.loads(final_meta_raw)
error_msg = final_meta.get("error")
final_meta["running"] = False
final_meta["finished_at"] = time.time()
r.set(meta_key, json.dumps(final_meta))
return {"batch_id": batch_id, "succeeded": succeeded, "failed": failed} _db_finish_batch_run(batch_id, succeeded, failed, bool(r.exists(stop_key)), error_msg)
_start_next_in_queue()
db/connection.py
+17
View File
@@ -54,6 +54,23 @@ CREATE TABLE IF NOT EXISTS flow_configs (
target_item_ids_json TEXT NOT NULL DEFAULT '[]', target_item_ids_json TEXT NOT NULL DEFAULT '[]',
target_url TEXT NOT NULL DEFAULT '' target_url TEXT NOT NULL DEFAULT ''
); );
CREATE TABLE IF NOT EXISTS batch_runs (
id INTEGER PRIMARY KEY AUTOINCREMENT,
batch_id TEXT NOT NULL UNIQUE,
config_id INTEGER,
config_name TEXT NOT NULL DEFAULT '',
workers INTEGER NOT NULL DEFAULT 1,
total_runs INTEGER NOT NULL DEFAULT 0,
stagger_ms INTEGER NOT NULL DEFAULT 0,
headless INTEGER NOT NULL DEFAULT 1,
started_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
finished_at TEXT,
succeeded INTEGER NOT NULL DEFAULT 0,
failed INTEGER NOT NULL DEFAULT 0,
stopped INTEGER NOT NULL DEFAULT 0,
error TEXT
);
""" """
db/repositories/__init__.py
+2
View File
@@ -1,7 +1,9 @@
from db.repositories.batch_runs import BatchRunsRepository, batch_runs_repo
from db.repositories.flow_configs import FlowConfigRepository, flow_config_repo from db.repositories.flow_configs import FlowConfigRepository, flow_config_repo
from db.repositories.runs import RunsRepository, runs_repo from db.repositories.runs import RunsRepository, runs_repo
__all__ = [ __all__ = [
"RunsRepository", "runs_repo", "RunsRepository", "runs_repo",
"FlowConfigRepository", "flow_config_repo", "FlowConfigRepository", "flow_config_repo",
"BatchRunsRepository", "batch_runs_repo",
] ]
db/repositories/batch_runs.py
+55
View File
@@ -0,0 +1,55 @@
"""Repository for the `batch_runs` table."""
from __future__ import annotations
from typing import Any
from db.connection import get_db
class BatchRunsRepository:
async def insert(
self,
batch_id: str,
config_id: int | None,
config_name: str,
workers: int,
total_runs: int,
stagger_ms: int,
headless: bool,
) -> None:
async with get_db() as db:
await db.execute(
"""INSERT INTO batch_runs
(batch_id, config_id, config_name, workers, total_runs, stagger_ms, headless)
VALUES (?, ?, ?, ?, ?, ?, ?)""",
(batch_id, config_id, config_name, workers, total_runs, stagger_ms, int(headless)),
)
await db.commit()
async def finish(
self,
batch_id: str,
succeeded: int,
failed: int,
stopped: bool,
error: str | None,
) -> None:
async with get_db() as db:
await db.execute(
"""UPDATE batch_runs
SET finished_at = strftime('%Y-%m-%dT%H:%M:%SZ', 'now'),
succeeded = ?, failed = ?, stopped = ?, error = ?
WHERE batch_id = ?""",
(succeeded, failed, int(stopped), error, batch_id),
)
await db.commit()
async def list(self, limit: int = 100) -> list[dict[str, Any]]:
async with get_db() as db:
rows = await (await db.execute(
"SELECT * FROM batch_runs ORDER BY id DESC LIMIT ?", (limit,)
)).fetchall()
return [dict(r) for r in rows]
batch_runs_repo = BatchRunsRepository()
db/repositories/flow_configs.py
+3 -13
View File
@@ -9,7 +9,7 @@ from db.connection import get_db
FlowConfigRow = dict[str, Any] FlowConfigRow = dict[str, Any]
_FIELDS = [ _FIELDS = [
"name", "is_active", "name",
"search_texts_json", "search_box_selector", "search_box_selector_type", "search_texts_json", "search_box_selector", "search_box_selector_type",
"search_submit_selector", "search_submit_selector_type", "search_submit_selector", "search_submit_selector_type",
"search_results_selector", "search_results_selector_type", "search_results_selector", "search_results_selector_type",
@@ -54,18 +54,16 @@ class FlowConfigRepository:
)).fetchone() )).fetchone()
return _deserialise(dict(row)) if row else None return _deserialise(dict(row)) if row else None
async def get_active(self) -> FlowConfigRow | None: async def get_first(self) -> FlowConfigRow | None:
async with get_db() as db: async with get_db() as db:
row = await (await db.execute( row = await (await db.execute(
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1" "SELECT * FROM flow_configs ORDER BY id ASC LIMIT 1"
)).fetchone() )).fetchone()
return _deserialise(dict(row)) if row else None return _deserialise(dict(row)) if row else None
async def create(self, data: dict[str, Any]) -> int: async def create(self, data: dict[str, Any]) -> int:
data = _serialise(data) data = _serialise(data)
async with get_db() as db: async with get_db() as db:
if data.get("is_active"):
await db.execute("UPDATE flow_configs SET is_active=0")
placeholders = ", ".join("?" for _ in _FIELDS) placeholders = ", ".join("?" for _ in _FIELDS)
cols = ", ".join(_FIELDS) cols = ", ".join(_FIELDS)
values = [data.get(f) for f in _FIELDS] values = [data.get(f) for f in _FIELDS]
@@ -78,8 +76,6 @@ class FlowConfigRepository:
async def update(self, cfg_id: int, data: dict[str, Any]) -> None: async def update(self, cfg_id: int, data: dict[str, Any]) -> None:
data = _serialise(data) data = _serialise(data)
async with get_db() as db: async with get_db() as db:
if data.get("is_active"):
await db.execute("UPDATE flow_configs SET is_active=0")
set_clause = ", ".join(f"{f}=?" for f in _FIELDS) set_clause = ", ".join(f"{f}=?" for f in _FIELDS)
values = [data.get(f) for f in _FIELDS] + [cfg_id] values = [data.get(f) for f in _FIELDS] + [cfg_id]
await db.execute( await db.execute(
@@ -87,12 +83,6 @@ class FlowConfigRepository:
) )
await db.commit() await db.commit()
async def activate(self, cfg_id: int) -> None:
async with get_db() as db:
await db.execute("UPDATE flow_configs SET is_active=0")
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
await db.commit()
async def delete(self, cfg_id: int) -> None: async def delete(self, cfg_id: int) -> None:
async with get_db() as db: async with get_db() as db:
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,)) await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
logger.py
+10 -10
View File
@@ -9,7 +9,7 @@ Usage anywhere in the codebase:
Environment variables: Environment variables:
LOG_LEVEL — DEBUG | INFO | WARNING | ERROR | CRITICAL (default: INFO) LOG_LEVEL — DEBUG | INFO | WARNING | ERROR | CRITICAL (default: INFO)
LOG_FILE — path to write a rotating log file (optional) LOG_FILE — path to write a rotating log file (default: logs/seed.log)
""" """
from __future__ import annotations from __future__ import annotations
@@ -28,7 +28,7 @@ _LEVEL_MAP: Final = {
} }
_LOG_LEVEL: int = _LEVEL_MAP.get(os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO) _LOG_LEVEL: int = _LEVEL_MAP.get(os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
_LOG_FILE: str | None = os.getenv("LOG_FILE") _LOG_FILE: str = os.getenv("LOG_FILE", "logs/seed.log")
# ── ANSI colour codes (console only) ───────────────────────────────────────── # ── ANSI colour codes (console only) ─────────────────────────────────────────
_RESET = "\033[0m" _RESET = "\033[0m"
@@ -86,14 +86,14 @@ def _build_root_logger() -> logging.Logger:
console.setFormatter(_ColourFormatter()) console.setFormatter(_ColourFormatter())
root.addHandler(console) root.addHandler(console)
# Optional rotating file handler # Rotating file handler (always on)
if _LOG_FILE: os.makedirs(os.path.dirname(_LOG_FILE), exist_ok=True)
fh = logging.handlers.RotatingFileHandler( fh = logging.handlers.RotatingFileHandler(
_LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8" _LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8"
) )
fh.setLevel(_LOG_LEVEL) fh.setLevel(_LOG_LEVEL)
fh.setFormatter(_FILE_FMT) fh.setFormatter(_FILE_FMT)
root.addHandler(fh) root.addHandler(fh)
root.propagate = False root.propagate = False
return root return root
main.py
+22 -13
View File
@@ -16,10 +16,15 @@ from logger import get_logger
log = get_logger(__name__) log = get_logger(__name__)
def _load_active_flow() -> dict[str, object]: def _load_flow(cfg_id: int | None = None) -> dict[str, object]:
cfg = asyncio.run(flow_config_repo.get_active()) if cfg_id is not None:
if cfg is None: cfg = asyncio.run(flow_config_repo.get(cfg_id))
log.warning("No active flow config — running without dynamic flow steps.") if cfg is None:
log.error("Flow config %d not found.", cfg_id)
else:
cfg = asyncio.run(flow_config_repo.get_first())
if cfg is None:
log.warning("No flow configs found — running without dynamic flow steps.")
return cfg or {} return cfg or {}
@@ -56,7 +61,7 @@ def _record(
log.info("%s OK (no flow config) — %d ms", identifier, total_ms) log.info("%s OK (no flow config) — %d ms", identifier, total_ms)
def run_otp(phone: str) -> None: def run_otp(phone: str, cfg_id: int | None = None) -> None:
from crawler.driver import driver_session from crawler.driver import driver_session
from crawler.dynamic_flow import DynamicFlow from crawler.dynamic_flow import DynamicFlow
from crawler.scenarios.otp_login import OTPLoginScenario from crawler.scenarios.otp_login import OTPLoginScenario
@@ -64,7 +69,7 @@ def run_otp(phone: str) -> None:
def otp_resolver(p: str) -> str: def otp_resolver(p: str) -> str:
raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway") raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
flow_cfg = _load_active_flow() flow_cfg = _load_flow(cfg_id)
target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else "" target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
t0 = time.monotonic() t0 = time.monotonic()
try: try:
@@ -77,12 +82,12 @@ def run_otp(phone: str) -> None:
_record("otp", phone, result, None, t0) _record("otp", phone, result, None, t0)
def run_fresh() -> None: def run_fresh(cfg_id: int | None = None) -> None:
from crawler.driver import driver_session from crawler.driver import driver_session
from crawler.dynamic_flow import DynamicFlow from crawler.dynamic_flow import DynamicFlow
from crawler.scenarios.fresh_session import FreshSessionScenario from crawler.scenarios.fresh_session import FreshSessionScenario
flow_cfg = _load_active_flow() flow_cfg = _load_flow(cfg_id)
target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else "" target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
t0 = time.monotonic() t0 = time.monotonic()
try: try:
@@ -95,10 +100,10 @@ def run_fresh() -> None:
_record("fresh", "fresh", result, None, t0) _record("fresh", "fresh", result, None, t0)
def run_batch(workers: int, total_runs: int, stagger_ms: int, headless: bool) -> None: def run_batch(workers: int, total_runs: int, stagger_ms: int, headless: bool, cfg_id: int | None = None) -> None:
from crawler.batch import run_batch as _run_batch from crawler.batch import run_batch as _run_batch
flow_cfg = _load_active_flow() flow_cfg = _load_flow(cfg_id)
if not flow_cfg: if not flow_cfg:
log.error("No active flow config — activate one in the admin panel first.") log.error("No active flow config — activate one in the admin panel first.")
sys.exit(1) sys.exit(1)
@@ -137,10 +142,13 @@ def main() -> None:
otp_p = sub.add_parser("otp", help="Run Scenario 1 (SMS-OTP login)") otp_p = sub.add_parser("otp", help="Run Scenario 1 (SMS-OTP login)")
otp_p.add_argument("--phone", help="Single phone number (default: all from config)") otp_p.add_argument("--phone", help="Single phone number (default: all from config)")
otp_p.add_argument("--config-id", type=int, default=None, help="Flow config ID (default: first config)")
sub.add_parser("fresh", help="Run Scenario 2 (fresh session)") fresh_p = sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
fresh_p.add_argument("--config-id", type=int, default=None, help="Flow config ID (default: first config)")
batch_p = sub.add_parser("batch", help="Run many parallel fresh-session runners") batch_p = sub.add_parser("batch", help="Run many parallel fresh-session runners")
batch_p.add_argument("--config-id", type=int, default=None, help="Flow config ID (default: first config)")
batch_p.add_argument("--workers", type=int, default=3, help="Parallel Chrome instances (default: 3)") batch_p.add_argument("--workers", type=int, default=3, help="Parallel Chrome instances (default: 3)")
batch_p.add_argument("--runs", type=int, default=10, help="Total runs to complete (default: 10)") batch_p.add_argument("--runs", type=int, default=10, help="Total runs to complete (default: 10)")
batch_p.add_argument("--stagger", type=int, default=500, help="Ms delay between worker launches (default: 500)") batch_p.add_argument("--stagger", type=int, default=500, help="Ms delay between worker launches (default: 500)")
@@ -156,15 +164,16 @@ def main() -> None:
log.error("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone") log.error("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone")
sys.exit(1) sys.exit(1)
for phone in phones: for phone in phones:
run_otp(phone) run_otp(phone, cfg_id=args.config_id)
elif args.cmd == "fresh": elif args.cmd == "fresh":
run_fresh() run_fresh(cfg_id=args.config_id)
elif args.cmd == "batch": elif args.cmd == "batch":
run_batch( run_batch(
workers=args.workers, workers=args.workers,
total_runs=args.runs, total_runs=args.runs,
stagger_ms=args.stagger, stagger_ms=args.stagger,
headless=args.headless, headless=args.headless,
cfg_id=args.config_id,
) )
pyproject.toml
+2 -1
View File
@@ -14,7 +14,8 @@ dependencies = [
"pydantic>=2.7.0", "pydantic>=2.7.0",
"python-dotenv>=1.0.1", "python-dotenv>=1.0.1",
"setuptools>=70.0.0", "setuptools>=70.0.0",
"huey>=2.5.0", "rq>=1.16.0",
"redis>=5.0.0",
] ]
[tool.pyright] [tool.pyright]
uv.lock
Generated
+72 -11
View File
@@ -42,6 +42,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" }, { url = "https://files.pythonhosted.org/packages/da/42/e921fccf5015463e32a3cf6ee7f980a6ed0f395ceeaa45060b61d86486c2/anyio-4.13.0-py3-none-any.whl", hash = "sha256:08b310f9e24a9594186fd75b4f73f4a4152069e3853f1ed8bfbf58369f4ad708", size = 114353, upload-time = "2026-03-24T12:59:08.246Z" },
] ]
[[package]]
name = "async-timeout"
version = "5.0.1"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/a5/ae/136395dfbfe00dfc94da3f3e136d0b13f394cba8f4841120e34226265780/async_timeout-5.0.1.tar.gz", hash = "sha256:d9321a7a3d5a6a5e187e824d2fa0793ce379a202935782d555d6e9d2735677d3", size = 9274, upload-time = "2024-11-06T16:41:39.6Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/fe/ba/e2081de779ca30d473f21f5b30e0e737c438205440784c7dfc81efc2b029/async_timeout-5.0.1-py3-none-any.whl", hash = "sha256:39e3809566ff85354557ec2398b55e096c8364bacac9405a7a1fa429e77fe76c", size = 6233, upload-time = "2024-11-06T16:41:37.9Z" },
]
[[package]] [[package]]
name = "attrs" name = "attrs"
version = "26.1.0" version = "26.1.0"
@@ -196,6 +205,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" }, { url = "https://files.pythonhosted.org/packages/d1/d6/3965ed04c63042e047cb6a3e6ed1a63a35087b6a609aa3a15ed8ac56c221/colorama-0.4.6-py2.py3-none-any.whl", hash = "sha256:4f1d9991f5acc0ca119f9d443620b77f9d6b33703e51011c16baf57afb285fc6", size = 25335, upload-time = "2022-10-25T02:36:20.889Z" },
] ]
[[package]]
name = "croniter"
version = "6.2.2"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "python-dateutil" },
]
sdist = { url = "https://files.pythonhosted.org/packages/df/de/5832661ed55107b8a09af3f0a2e71e0957226a59eb1dcf0a445cce6daf20/croniter-6.2.2.tar.gz", hash = "sha256:ba60832a5ec8e12e51b8691c3309a113d1cf6526bdf1a48150ce8ec7a532d0ab", size = 113762, upload-time = "2026-03-15T08:43:48.112Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/d0/39/783980e78cb92c2d7bdb1fc7dbc86e94ccc6d58224d76a7f1f51b6c51e30/croniter-6.2.2-py3-none-any.whl", hash = "sha256:a5d17b1060974d36251ea4faf388233eca8acf0d09cbd92d35f4c4ac8f279960", size = 45422, upload-time = "2026-03-15T08:43:46.626Z" },
]
[[package]] [[package]]
name = "fastapi" name = "fastapi"
version = "0.137.1" version = "0.137.1"
@@ -292,15 +313,6 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" }, { url = "https://files.pythonhosted.org/packages/2a/39/e50c7c3a983047577ee07d2a9e53faf5a69493943ec3f6a384bdc792deb2/httpx-0.28.1-py3-none-any.whl", hash = "sha256:d909fcccc110f8c7faf814ca82a9a4d816bc5a6dbfea25d6591d6985b8ba59ad", size = 73517, upload-time = "2024-12-06T15:37:21.509Z" },
] ]
[[package]]
name = "huey"
version = "3.0.3"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/d1/cb/58f229149944602917a976d533d3fe7d54770d6ea18df5931e3f4f313fa0/huey-3.0.3.tar.gz", hash = "sha256:1a17fef95fc8432f75413f1b77439cef5f3493c1ddbfba9151756b31a1b2dad3", size = 263604, upload-time = "2026-06-12T01:53:55.49Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/e8/82/f85d8918949786420716a5e421525ab12aa084bcbe32d86c9743e50bcf3d/huey-3.0.3-py3-none-any.whl", hash = "sha256:d1c687734778b8282c035a943eead8368c1736bb28abc006596fbbc01bdc96dc", size = 94945, upload-time = "2026-06-12T01:53:53.981Z" },
]
[[package]] [[package]]
name = "idna" name = "idna"
version = "3.18" version = "3.18"
@@ -479,6 +491,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" }, { url = "https://files.pythonhosted.org/packages/8d/59/b4572118e098ac8e46e399a1dd0f2d85403ce8bbaad9ec79373ed6badaf9/PySocks-1.7.1-py3-none-any.whl", hash = "sha256:2725bd0a9925919b9b51739eea5f9e2bae91e83288108a9ad338b2e3a4435ee5", size = 16725, upload-time = "2019-09-20T02:06:22.938Z" },
] ]
[[package]]
name = "python-dateutil"
version = "2.9.0.post0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "six" },
]
sdist = { url = "https://files.pythonhosted.org/packages/66/c0/0c8b6ad9f17a802ee498c46e004a0eb49bc148f2fd230864601a86dcf6db/python-dateutil-2.9.0.post0.tar.gz", hash = "sha256:37dd54208da7e1cd875388217d5e00ebd4179249f90fb72437e91a35459a0ad3", size = 342432, upload-time = "2024-03-01T18:36:20.211Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/ec/57/56b9bcc3c9c6a792fcbaf139543cee77261f3651ca9da0c93f5c1221264b/python_dateutil-2.9.0.post0-py2.py3-none-any.whl", hash = "sha256:a8b2bc7bffae282281c8140a97d3aa9c14da0b136dfe83f850eea9a5f7470427", size = 229892, upload-time = "2024-03-01T18:36:18.57Z" },
]
[[package]] [[package]]
name = "python-dotenv" name = "python-dotenv"
version = "1.2.2" version = "1.2.2"
@@ -552,6 +576,18 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" }, { url = "https://files.pythonhosted.org/packages/f1/12/de94a39c2ef588c7e6455cfbe7343d3b2dc9d6b6b2f40c4c6565744c873d/pyyaml-6.0.3-cp314-cp314t-win_arm64.whl", hash = "sha256:ebc55a14a21cb14062aa4162f906cd962b28e2e9ea38f9b4391244cd8de4ae0b", size = 149341, upload-time = "2025-09-25T21:32:56.828Z" },
] ]
[[package]]
name = "redis"
version = "8.0.0"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "async-timeout", marker = "python_full_version < '3.11.3'" },
]
sdist = { url = "https://files.pythonhosted.org/packages/53/ae/ed461cca5780b5fc8b9fe8ca0ed98d89508645fb9d880c24cc42c087678f/redis-8.0.0.tar.gz", hash = "sha256:a00c5355432051ac14e593b8b197fc76c887ee12d55a0984f69328a1115fdc49", size = 5101591, upload-time = "2026-05-28T12:45:13.5Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/27/e3/b519734372d305bd547534a9f32e4ce9f98552af753dce72cf3483a0ff0b/redis-8.0.0-py3-none-any.whl", hash = "sha256:c938c18338585009f0bc310f4c7e4e4b4d37639356c4ac072cedf3af570c8dc7", size = 499870, upload-time = "2026-05-28T12:45:11.697Z" },
]
[[package]] [[package]]
name = "requests" name = "requests"
version = "2.34.2" version = "2.34.2"
@@ -567,6 +603,20 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" }, { url = "https://files.pythonhosted.org/packages/a0/f4/c67b0b3f1b9245e8d266f0f112c500d50e5b4e83cb6f3b71b6528104182a/requests-2.34.2-py3-none-any.whl", hash = "sha256:2a0d60c172f83ac6ab31e4554906c0f3b3588d37b5cb939b1c061f4907e278e0", size = 73075, upload-time = "2026-05-14T19:25:26.443Z" },
] ]
[[package]]
name = "rq"
version = "2.9.1"
source = { registry = "https://pypi.org/simple" }
dependencies = [
{ name = "click" },
{ name = "croniter" },
{ name = "redis" },
]
sdist = { url = "https://files.pythonhosted.org/packages/ca/87/d8ce4fb6b969d11454cf6f5c6e10735ffe096e4a9e208b4cf4d501bec41d/rq-2.9.1.tar.gz", hash = "sha256:2ae30c452bd1e8e26b4a213340d4b6e3957cae23176a92f9132c661d73d0c179", size = 745898, upload-time = "2026-06-06T02:48:36.949Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/77/ae/99335807dde9ce8402d31666d385122a20ecd710a1fd62e5d7014265b783/rq-2.9.1-py3-none-any.whl", hash = "sha256:092624e1bc3b3ca9d2ce7ff4049bd7667f772f84695c9dc66a7a662f67f1939a", size = 120816, upload-time = "2026-06-06T02:48:38.587Z" },
]
[[package]] [[package]]
name = "seed" name = "seed"
version = "0.1.0" version = "0.1.0"
@@ -575,10 +625,11 @@ dependencies = [
{ name = "aiosqlite" }, { name = "aiosqlite" },
{ name = "fastapi" }, { name = "fastapi" },
{ name = "httpx" }, { name = "httpx" },
{ name = "huey" },
{ name = "pydantic" }, { name = "pydantic" },
{ name = "python-dotenv" }, { name = "python-dotenv" },
{ name = "python-multipart" }, { name = "python-multipart" },
{ name = "redis" },
{ name = "rq" },
{ name = "selenium" }, { name = "selenium" },
{ name = "selenium-stealth" }, { name = "selenium-stealth" },
{ name = "setuptools" }, { name = "setuptools" },
@@ -596,10 +647,11 @@ requires-dist = [
{ name = "aiosqlite", specifier = ">=0.20.0" }, { name = "aiosqlite", specifier = ">=0.20.0" },
{ name = "fastapi", specifier = ">=0.111.0" }, { name = "fastapi", specifier = ">=0.111.0" },
{ name = "httpx", specifier = ">=0.27.0" }, { name = "httpx", specifier = ">=0.27.0" },
{ name = "huey", specifier = ">=2.5.0" },
{ name = "pydantic", specifier = ">=2.7.0" }, { name = "pydantic", specifier = ">=2.7.0" },
{ name = "python-dotenv", specifier = ">=1.0.1" }, { name = "python-dotenv", specifier = ">=1.0.1" },
{ name = "python-multipart", specifier = ">=0.0.9" }, { name = "python-multipart", specifier = ">=0.0.9" },
{ name = "redis", specifier = ">=5.0.0" },
{ name = "rq", specifier = ">=1.16.0" },
{ name = "selenium", specifier = ">=4.18.0" }, { name = "selenium", specifier = ">=4.18.0" },
{ name = "selenium-stealth", specifier = ">=1.0.6" }, { name = "selenium-stealth", specifier = ">=1.0.6" },
{ name = "setuptools", specifier = ">=70.0.0" }, { name = "setuptools", specifier = ">=70.0.0" },
@@ -647,6 +699,15 @@ wheels = [
{ url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" }, { url = "https://files.pythonhosted.org/packages/9d/76/f789f7a86709c6b087c5a2f52f911838cad707cc613162401badc665acfe/setuptools-82.0.1-py3-none-any.whl", hash = "sha256:a59e362652f08dcd477c78bb6e7bd9d80a7995bc73ce773050228a348ce2e5bb", size = 1006223, upload-time = "2026-03-09T12:47:15.026Z" },
] ]
[[package]]
name = "six"
version = "1.17.0"
source = { registry = "https://pypi.org/simple" }
sdist = { url = "https://files.pythonhosted.org/packages/94/e7/b2c673351809dca68a0e064b6af791aa332cf192da575fd474ed7d6f16a2/six-1.17.0.tar.gz", hash = "sha256:ff70335d468e7eb6ec65b95b99d3a2836546063f63acc5171de367e834932a81", size = 34031, upload-time = "2024-12-04T17:35:28.174Z" }
wheels = [
{ url = "https://files.pythonhosted.org/packages/b7/ce/149a00dd41f10bc29e5921b496af8b574d8413afcd5e30dfa0ed46c2cc5e/six-1.17.0-py2.py3-none-any.whl", hash = "sha256:4721f391ed90541fddacab5acf947aa0d3dc7d27b2e1e8eda2be8970586c3274", size = 11050, upload-time = "2024-12-04T17:35:26.475Z" },
]
[[package]] [[package]]
name = "sniffio" name = "sniffio"
version = "1.3.1" version = "1.3.1"