nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Digipay and speedtest added

Browse Source
This commit is contained in:
Navid Filsaraee
2026-06-27 16:18:33 +03:30
parent 64886b2b10
commit 744550deaa
8 changed files with 421 additions and 16 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/routes.py
+66 -6
View File
@@ -1,9 +1,11 @@
"""FastAPI routes — dashboard API + auth."""
from __future__ import annotations
import secrets
from typing import Annotated, Any
import speedtest # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel
@@ -24,9 +26,15 @@ router = APIRouter()
security = HTTPBasic()
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
def require_auth(
credentials: Annotated[HTTPBasicCredentials, Depends(security)],
) -> str:
ok_user = secrets.compare_digest(
credentials.username.encode(), config.admin_username.encode()
)
ok_pass = secrets.compare_digest(
credentials.password.encode(), config.admin_password.encode()
)
if not (ok_user and ok_pass):
raise HTTPException(
status_code=status.HTTP_401_UNAUTHORIZED,
@@ -43,6 +51,7 @@ Auth = Annotated[str, Depends(require_auth)]
# Stats
# ---------------------------------------------------------------------------
@router.get("/api/stats")
async def stats(_: Auth) -> dict[str, object]:
return await get_stats()
@@ -52,6 +61,7 @@ async def stats(_: Auth) -> dict[str, object]:
# Flow configs
# ---------------------------------------------------------------------------
class FlowConfigIn(BaseModel):
name: str
search_texts: list[str] = []
@@ -75,6 +85,7 @@ class FlowConfigIn(BaseModel):
item_url_template: str = ""
target_item_ids: list[str] = []
target_url: str = ""
scenario: str = "dynamic"
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -101,6 +112,7 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
"item_url_template": body.item_url_template,
"target_item_ids_json": body.target_item_ids,
"target_url": body.target_url,
"scenario": body.scenario,
}
@@ -146,6 +158,7 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
# Batch runs (managed via Huey task queue)
# ---------------------------------------------------------------------------
class BatchTaskIn(BaseModel):
config_id: int
workers: int = 3
@@ -160,17 +173,25 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
cfg = await get_flow_config(body.config_id)
if not cfg:
raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found")
raise HTTPException(
status_code=404, detail=f"Flow config {body.config_id} not found"
)
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
log.info("Task enqueued — config=%d runs=%d workers=%d status=%s",
body.config_id, body.total_runs, body.workers, result["status"])
log.info(
"Task enqueued — config=%d runs=%d workers=%d status=%s",
body.config_id,
body.total_runs,
body.workers,
result["status"],
)
return result
@router.delete("/api/batch-queue/{queue_id}", status_code=200)
async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
from crawler.tasks import dequeue
removed = dequeue(queue_id)
if not removed:
raise HTTPException(status_code=404, detail="Queue item not found")
@@ -180,6 +201,7 @@ async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
@router.post("/api/batch-queue/clear", status_code=200)
async def clear_batch_queue(_: Auth) -> dict[str, object]:
from crawler.tasks import clear_queue
count = clear_queue()
return {"cleared": count}
@@ -187,6 +209,7 @@ async def clear_batch_queue(_: Auth) -> dict[str, object]:
@router.post("/api/stop-batch", status_code=200)
async def stop_batch(_: Auth) -> dict[str, object]:
from crawler.tasks import stop_current
stopped = stop_current()
if not stopped:
raise HTTPException(status_code=404, detail="No active batch to stop")
@@ -196,12 +219,14 @@ async def stop_batch(_: Auth) -> dict[str, object]:
@router.get("/api/batch-status")
async def batch_status(_: Auth) -> dict[str, object]:
from crawler.tasks import batch_status as _status
return _status() # type: ignore[return-value]
@router.get("/api/batch-runs")
async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
from db.repositories.batch_runs import batch_runs_repo
return await batch_runs_repo.list() # type: ignore[return-value]
@@ -209,6 +234,7 @@ async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
# Log viewer
# ---------------------------------------------------------------------------
@router.get("/api/logs")
def get_logs(
_: Auth,
@@ -217,6 +243,7 @@ def get_logs(
q: str = "",
) -> dict[str, object]:
import os
from logger import _LOG_FILE
path = _LOG_FILE
@@ -245,3 +272,36 @@ def get_logs(
break
return {"lines": results, "file": path}
# ---------------------------------------------------------------------------
# Speed test
# ---------------------------------------------------------------------------
@router.post("/api/speedtest")
async def run_speedtest(_: Auth) -> dict[str, object]:
import asyncio
def _run() -> dict[str, object]:
try:
log.info("running speedtest")
st = speedtest.Speedtest(secure=True)
st.get_best_server()
download_bps: float = st.download()
upload_bps: float = st.upload()
r = st.results.dict()
server: dict[str, object] = r.get("server") or {}
client: dict[str, object] = r.get("client") or {}
return {
"download_mbps": round(download_bps / 1_000_000, 2),
"upload_mbps": round(upload_bps / 1_000_000, 2),
"ping_ms": round(float(r.get("ping") or 0), 1),
"server": f"{server.get('name', '')} ({server.get('country', '')})",
"isp": str(client.get("isp", "")),
}
except Exception as e:
log.error(e)
loop = asyncio.get_event_loop()
return await loop.run_in_executor(None, _run)