From 744550deaac0a51874fc8c8ba5429bcc2bbf6103 Mon Sep 17 00:00:00 2001 From: Navid Filsaraee Date: Sat, 27 Jun 2026 16:18:33 +0330 Subject: [PATCH] Digipay and speedtest added --- admin/routes.py | 72 ++++++++++- admin/static/dashboard.html | 207 ++++++++++++++++++++++++++++++-- crawler/scenarios/digipay.py | 133 ++++++++++++++++++++ crawler/tasks.py | 10 +- db/connection.py | 1 + db/repositories/flow_configs.py | 2 +- pyproject.toml | 1 + uv.lock | 11 ++ 8 files changed, 421 insertions(+), 16 deletions(-) create mode 100644 crawler/scenarios/digipay.py diff --git a/admin/routes.py b/admin/routes.py index 5754e9d..902adfa 100644 --- a/admin/routes.py +++ b/admin/routes.py @@ -1,9 +1,11 @@ """FastAPI routes — dashboard API + auth.""" + from __future__ import annotations import secrets from typing import Annotated, Any +import speedtest # type: ignore[import-untyped] from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel @@ -24,9 +26,15 @@ router = APIRouter() security = HTTPBasic() -def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str: - ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode()) - ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode()) +def require_auth( + credentials: Annotated[HTTPBasicCredentials, Depends(security)], +) -> str: + ok_user = secrets.compare_digest( + credentials.username.encode(), config.admin_username.encode() + ) + ok_pass = secrets.compare_digest( + credentials.password.encode(), config.admin_password.encode() + ) if not (ok_user and ok_pass): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, @@ -43,6 +51,7 @@ Auth = Annotated[str, Depends(require_auth)] # Stats # --------------------------------------------------------------------------- + @router.get("/api/stats") async def stats(_: Auth) -> dict[str, object]: return await get_stats() @@ -52,6 +61,7 @@ async def stats(_: Auth) -> dict[str, object]: # Flow configs # --------------------------------------------------------------------------- + class FlowConfigIn(BaseModel): name: str search_texts: list[str] = [] @@ -75,6 +85,7 @@ class FlowConfigIn(BaseModel): item_url_template: str = "" target_item_ids: list[str] = [] target_url: str = "" + scenario: str = "dynamic" def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: @@ -101,6 +112,7 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: "item_url_template": body.item_url_template, "target_item_ids_json": body.target_item_ids, "target_url": body.target_url, + "scenario": body.scenario, } @@ -146,6 +158,7 @@ async def delete_config(cfg_id: int, _: Auth) -> None: # Batch runs (managed via Huey task queue) # --------------------------------------------------------------------------- + class BatchTaskIn(BaseModel): config_id: int workers: int = 3 @@ -160,17 +173,25 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]: cfg = await get_flow_config(body.config_id) if not cfg: - raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found") + raise HTTPException( + status_code=404, detail=f"Flow config {body.config_id} not found" + ) result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless) - log.info("Task enqueued — config=%d runs=%d workers=%d status=%s", - body.config_id, body.total_runs, body.workers, result["status"]) + log.info( + "Task enqueued — config=%d runs=%d workers=%d status=%s", + body.config_id, + body.total_runs, + body.workers, + result["status"], + ) return result @router.delete("/api/batch-queue/{queue_id}", status_code=200) async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]: from crawler.tasks import dequeue + removed = dequeue(queue_id) if not removed: raise HTTPException(status_code=404, detail="Queue item not found") @@ -180,6 +201,7 @@ async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]: @router.post("/api/batch-queue/clear", status_code=200) async def clear_batch_queue(_: Auth) -> dict[str, object]: from crawler.tasks import clear_queue + count = clear_queue() return {"cleared": count} @@ -187,6 +209,7 @@ async def clear_batch_queue(_: Auth) -> dict[str, object]: @router.post("/api/stop-batch", status_code=200) async def stop_batch(_: Auth) -> dict[str, object]: from crawler.tasks import stop_current + stopped = stop_current() if not stopped: raise HTTPException(status_code=404, detail="No active batch to stop") @@ -196,12 +219,14 @@ async def stop_batch(_: Auth) -> dict[str, object]: @router.get("/api/batch-status") async def batch_status(_: Auth) -> dict[str, object]: from crawler.tasks import batch_status as _status + return _status() # type: ignore[return-value] @router.get("/api/batch-runs") async def list_batch_runs(_: Auth) -> list[dict[str, object]]: from db.repositories.batch_runs import batch_runs_repo + return await batch_runs_repo.list() # type: ignore[return-value] @@ -209,6 +234,7 @@ async def list_batch_runs(_: Auth) -> list[dict[str, object]]: # Log viewer # --------------------------------------------------------------------------- + @router.get("/api/logs") def get_logs( _: Auth, @@ -217,6 +243,7 @@ def get_logs( q: str = "", ) -> dict[str, object]: import os + from logger import _LOG_FILE path = _LOG_FILE @@ -245,3 +272,36 @@ def get_logs( break return {"lines": results, "file": path} + + +# --------------------------------------------------------------------------- +# Speed test +# --------------------------------------------------------------------------- + + +@router.post("/api/speedtest") +async def run_speedtest(_: Auth) -> dict[str, object]: + import asyncio + + def _run() -> dict[str, object]: + try: + log.info("running speedtest") + st = speedtest.Speedtest(secure=True) + st.get_best_server() + download_bps: float = st.download() + upload_bps: float = st.upload() + r = st.results.dict() + server: dict[str, object] = r.get("server") or {} + client: dict[str, object] = r.get("client") or {} + return { + "download_mbps": round(download_bps / 1_000_000, 2), + "upload_mbps": round(upload_bps / 1_000_000, 2), + "ping_ms": round(float(r.get("ping") or 0), 1), + "server": f"{server.get('name', '')} ({server.get('country', '')})", + "isp": str(client.get("isp", "")), + } + except Exception as e: + log.error(e) + + loop = asyncio.get_event_loop() + return await loop.run_in_executor(None, _run) diff --git a/admin/static/dashboard.html b/admin/static/dashboard.html index 86a5a53..1ac44bf 100644 --- a/admin/static/dashboard.html +++ b/admin/static/dashboard.html @@ -4,6 +4,9 @@ Seed — Admin + + +