"""FastAPI routes — dashboard API + auth.""" from __future__ import annotations import secrets from datetime import UTC, datetime from pathlib import Path from typing import Annotated, Any import speedtest # type: ignore[import-untyped] from fastapi import APIRouter, Depends, HTTPException, status from fastapi.responses import FileResponse from fastapi.security import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel from admin.models import ( delete_flow_config, get_flow_config, get_stats, list_flow_configs, upsert_flow_config, ) from config import config from logger import get_logger log = get_logger(__name__) router = APIRouter() security = HTTPBasic() def require_auth( credentials: Annotated[HTTPBasicCredentials, Depends(security)], ) -> str: ok_user = secrets.compare_digest( credentials.username.encode(), config.admin_username.encode() ) ok_pass = secrets.compare_digest( credentials.password.encode(), config.admin_password.encode() ) if not (ok_user and ok_pass): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials", headers={"WWW-Authenticate": "Basic"}, ) return credentials.username Auth = Annotated[str, Depends(require_auth)] # --------------------------------------------------------------------------- # Stats # --------------------------------------------------------------------------- @router.get("/api/stats") async def stats(_: Auth) -> dict[str, object]: return await get_stats() # --------------------------------------------------------------------------- # Flow configs # --------------------------------------------------------------------------- class FlowConfigIn(BaseModel): name: str search_texts: list[str] = [] search_box_selector: str = "" search_box_selector_type: str = "css" search_submit_selector: str = "" search_submit_selector_type: str = "css" search_results_selector: str = "" search_results_selector_type: str = "css" scroll_container_selector: str = "" scroll_container_selector_type: str = "css" max_scrolls: int = 30 scroll_pause_ms: int = 1200 no_new_content_timeout_ms: int = 3000 pagination_selector: str = "" pagination_selector_type: str = "css" max_pages: int = 10 pagination_wait_ms: int = 1500 item_selector_template: str = "" item_selector_type: str = "css" item_url_template: str = "" target_item_ids: list[str] = [] target_url: str = "" scenario: str = "dynamic" stop_on_first_click: bool = False def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: return { "name": body.name, "search_texts_json": body.search_texts, "search_box_selector": body.search_box_selector, "search_box_selector_type": body.search_box_selector_type, "search_submit_selector": body.search_submit_selector, "search_submit_selector_type": body.search_submit_selector_type, "search_results_selector": body.search_results_selector, "search_results_selector_type": body.search_results_selector_type, "scroll_container_selector": body.scroll_container_selector, "scroll_container_selector_type": body.scroll_container_selector_type, "max_scrolls": body.max_scrolls, "scroll_pause_ms": body.scroll_pause_ms, "no_new_content_timeout_ms": body.no_new_content_timeout_ms, "pagination_selector": body.pagination_selector, "pagination_selector_type": body.pagination_selector_type, "max_pages": body.max_pages, "pagination_wait_ms": body.pagination_wait_ms, "item_selector_template": body.item_selector_template, "item_selector_type": body.item_selector_type, "item_url_template": body.item_url_template, "target_item_ids_json": body.target_item_ids, "target_url": body.target_url, "scenario": body.scenario, "stop_on_first_click": int(body.stop_on_first_click), } @router.get("/api/flow-configs") async def list_configs(_: Auth) -> list[dict[str, object]]: return await list_flow_configs() # type: ignore[return-value] @router.get("/api/flow-configs/{cfg_id}") async def get_config(cfg_id: int, _: Auth) -> dict[str, object]: row = await get_flow_config(cfg_id) if not row: raise HTTPException(status_code=404, detail="Not found") return row # type: ignore[return-value] @router.post("/api/flow-configs", status_code=201) async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]: new_id = await upsert_flow_config(_to_db_dict(body)) row = await get_flow_config(new_id) return row or {} # type: ignore[return-value] @router.put("/api/flow-configs/{cfg_id}") async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]: existing = await get_flow_config(cfg_id) if not existing: raise HTTPException(status_code=404, detail="Not found") await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id) row = await get_flow_config(cfg_id) return row or {} # type: ignore[return-value] @router.delete("/api/flow-configs/{cfg_id}", status_code=204) async def delete_config(cfg_id: int, _: Auth) -> None: existing = await get_flow_config(cfg_id) if not existing: raise HTTPException(status_code=404, detail="Not found") await delete_flow_config(cfg_id) # --------------------------------------------------------------------------- # Batch runs (managed via Huey task queue) # --------------------------------------------------------------------------- class BatchTaskIn(BaseModel): config_id: int workers: int = 3 total_runs: int = 10 stagger_ms: int = 500 headless: bool = True @router.post("/api/batch-queue", status_code=202) async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]: from crawler.tasks import enqueue cfg = await get_flow_config(body.config_id) if not cfg: raise HTTPException( status_code=404, detail=f"Flow config {body.config_id} not found" ) result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless) log.info( "Task enqueued — config=%d runs=%d workers=%d status=%s", body.config_id, body.total_runs, body.workers, result["status"], ) return result @router.delete("/api/batch-queue/{queue_id}", status_code=200) async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]: from crawler.tasks import dequeue removed = dequeue(queue_id) if not removed: raise HTTPException(status_code=404, detail="Queue item not found") return {"removed": queue_id} @router.post("/api/batch-queue/clear", status_code=200) async def clear_batch_queue(_: Auth) -> dict[str, object]: from crawler.tasks import clear_queue count = clear_queue() return {"cleared": count} @router.post("/api/stop-batch", status_code=200) async def stop_batch(_: Auth) -> dict[str, object]: from crawler.tasks import stop_current stopped = stop_current() if not stopped: raise HTTPException(status_code=404, detail="No active batch to stop") return {"status": "stopping"} @router.get("/api/batch-status") async def batch_status(_: Auth) -> dict[str, object]: from crawler.tasks import batch_status as _status return _status() # type: ignore[return-value] @router.get("/api/batch-runs") async def list_batch_runs(_: Auth) -> list[dict[str, object]]: from db.repositories.batch_runs import batch_runs_repo return await batch_runs_repo.list() # type: ignore[return-value] # --------------------------------------------------------------------------- # Browser-session recordings # --------------------------------------------------------------------------- def _recording_path(filename: str) -> Path: if filename != Path(filename).name or Path(filename).suffix.lower() != ".mp4": raise HTTPException(status_code=404, detail="Recording not found") directory = Path(config.recordings_dir).resolve() path = directory / filename if path.is_symlink() or not path.is_file(): raise HTTPException(status_code=404, detail="Recording not found") return path @router.get("/api/recordings") def list_recordings(_: Auth) -> list[dict[str, object]]: directory = Path(config.recordings_dir) if not directory.is_dir(): return [] rows: list[dict[str, object]] = [] for path in directory.glob("*.mp4"): if path.is_symlink(): continue try: stat_result = path.stat() except FileNotFoundError: continue rows.append( { "name": path.name, "size_bytes": stat_result.st_size, "created_at": datetime.fromtimestamp( stat_result.st_mtime, tz=UTC ).isoformat(), } ) rows.sort(key=lambda row: str(row["created_at"]), reverse=True) return rows @router.get("/api/recordings/{filename}") def download_recording(filename: str, _: Auth) -> FileResponse: path = _recording_path(filename) return FileResponse(path, media_type="video/mp4", filename=path.name) @router.delete("/api/recordings/{filename}", status_code=204) def delete_recording(filename: str, _: Auth) -> None: path = _recording_path(filename) try: path.unlink() except FileNotFoundError: raise HTTPException(status_code=404, detail="Recording not found") from None # --------------------------------------------------------------------------- # Log viewer # --------------------------------------------------------------------------- @router.get("/api/logs") def get_logs( _: Auth, n: int = 200, level: str = "", q: str = "", ) -> dict[str, object]: import os from logger import _LOG_FILE path = _LOG_FILE if not os.path.exists(path): return {"lines": [], "file": path} with open(path, encoding="utf-8", errors="replace") as f: all_lines = f.readlines() all_lines.reverse() # newest first level_upper = level.upper() q_lower = q.lower() results: list[str] = [] for raw in all_lines: line = raw.rstrip() if not line: continue if level_upper and level_upper not in line: continue if q_lower and q_lower not in line.lower(): continue results.append(line) if len(results) >= n: break return {"lines": results, "file": path} # --------------------------------------------------------------------------- # Speed test # --------------------------------------------------------------------------- @router.post("/api/speedtest") async def run_speedtest(_: Auth) -> dict[str, object]: import asyncio def _run() -> dict[str, object]: try: log.info("running speedtest") st = speedtest.Speedtest(secure=True) st.get_best_server() download_bps: float = st.download() upload_bps: float = st.upload() r = st.results.dict() server: dict[str, object] = r.get("server") or {} client: dict[str, object] = r.get("client") or {} return { "download_mbps": round(download_bps / 1_000_000, 2), "upload_mbps": round(upload_bps / 1_000_000, 2), "ping_ms": round(float(r.get("ping") or 0), 1), "server": f"{server.get('name', '')} ({server.get('country', '')})", "isp": str(client.get("isp", "")), } except Exception as e: log.error(e) loop = asyncio.get_event_loop() return await loop.run_in_executor(None, _run)