"""FastAPI routes — dashboard API + auth.""" from __future__ import annotations import secrets from typing import Annotated, Any from fastapi import APIRouter, Depends, HTTPException, status from fastapi.security import HTTPBasic, HTTPBasicCredentials from pydantic import BaseModel from admin.models import ( delete_flow_config, get_flow_config, get_stats, list_flow_configs, upsert_flow_config, ) from config import config from logger import get_logger log = get_logger(__name__) router = APIRouter() security = HTTPBasic() def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str: ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode()) ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode()) if not (ok_user and ok_pass): raise HTTPException( status_code=status.HTTP_401_UNAUTHORIZED, detail="Invalid credentials", headers={"WWW-Authenticate": "Basic"}, ) return credentials.username Auth = Annotated[str, Depends(require_auth)] # --------------------------------------------------------------------------- # Stats # --------------------------------------------------------------------------- @router.get("/api/stats") async def stats(_: Auth) -> dict[str, object]: return await get_stats() # --------------------------------------------------------------------------- # Flow configs # --------------------------------------------------------------------------- class FlowConfigIn(BaseModel): name: str search_texts: list[str] = [] search_box_selector: str = "" search_box_selector_type: str = "css" search_submit_selector: str = "" search_submit_selector_type: str = "css" search_results_selector: str = "" search_results_selector_type: str = "css" scroll_container_selector: str = "" scroll_container_selector_type: str = "css" max_scrolls: int = 30 scroll_pause_ms: int = 1200 no_new_content_timeout_ms: int = 3000 pagination_selector: str = "" pagination_selector_type: str = "css" max_pages: int = 10 pagination_wait_ms: int = 1500 item_selector_template: str = "" item_selector_type: str = "css" item_url_template: str = "" target_item_ids: list[str] = [] target_url: str = "" def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: return { "name": body.name, "search_texts_json": body.search_texts, "search_box_selector": body.search_box_selector, "search_box_selector_type": body.search_box_selector_type, "search_submit_selector": body.search_submit_selector, "search_submit_selector_type": body.search_submit_selector_type, "search_results_selector": body.search_results_selector, "search_results_selector_type": body.search_results_selector_type, "scroll_container_selector": body.scroll_container_selector, "scroll_container_selector_type": body.scroll_container_selector_type, "max_scrolls": body.max_scrolls, "scroll_pause_ms": body.scroll_pause_ms, "no_new_content_timeout_ms": body.no_new_content_timeout_ms, "pagination_selector": body.pagination_selector, "pagination_selector_type": body.pagination_selector_type, "max_pages": body.max_pages, "pagination_wait_ms": body.pagination_wait_ms, "item_selector_template": body.item_selector_template, "item_selector_type": body.item_selector_type, "item_url_template": body.item_url_template, "target_item_ids_json": body.target_item_ids, "target_url": body.target_url, } @router.get("/api/flow-configs") async def list_configs(_: Auth) -> list[dict[str, object]]: return await list_flow_configs() # type: ignore[return-value] @router.get("/api/flow-configs/{cfg_id}") async def get_config(cfg_id: int, _: Auth) -> dict[str, object]: row = await get_flow_config(cfg_id) if not row: raise HTTPException(status_code=404, detail="Not found") return row # type: ignore[return-value] @router.post("/api/flow-configs", status_code=201) async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]: new_id = await upsert_flow_config(_to_db_dict(body)) row = await get_flow_config(new_id) return row or {} # type: ignore[return-value] @router.put("/api/flow-configs/{cfg_id}") async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]: existing = await get_flow_config(cfg_id) if not existing: raise HTTPException(status_code=404, detail="Not found") await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id) row = await get_flow_config(cfg_id) return row or {} # type: ignore[return-value] @router.delete("/api/flow-configs/{cfg_id}", status_code=204) async def delete_config(cfg_id: int, _: Auth) -> None: existing = await get_flow_config(cfg_id) if not existing: raise HTTPException(status_code=404, detail="Not found") await delete_flow_config(cfg_id) # --------------------------------------------------------------------------- # Batch runs (managed via Huey task queue) # --------------------------------------------------------------------------- class BatchTaskIn(BaseModel): config_id: int workers: int = 3 total_runs: int = 10 stagger_ms: int = 500 headless: bool = True @router.post("/api/batch-queue", status_code=202) async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]: from crawler.tasks import enqueue cfg = await get_flow_config(body.config_id) if not cfg: raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found") result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless) log.info("Task enqueued — config=%d runs=%d workers=%d status=%s", body.config_id, body.total_runs, body.workers, result["status"]) return result @router.delete("/api/batch-queue/{queue_id}", status_code=200) async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]: from crawler.tasks import dequeue removed = dequeue(queue_id) if not removed: raise HTTPException(status_code=404, detail="Queue item not found") return {"removed": queue_id} @router.post("/api/batch-queue/clear", status_code=200) async def clear_batch_queue(_: Auth) -> dict[str, object]: from crawler.tasks import clear_queue count = clear_queue() return {"cleared": count} @router.post("/api/stop-batch", status_code=200) async def stop_batch(_: Auth) -> dict[str, object]: from crawler.tasks import stop_current stopped = stop_current() if not stopped: raise HTTPException(status_code=404, detail="No active batch to stop") return {"status": "stopping"} @router.get("/api/batch-status") async def batch_status(_: Auth) -> dict[str, object]: from crawler.tasks import batch_status as _status return _status() # type: ignore[return-value] @router.get("/api/batch-runs") async def list_batch_runs(_: Auth) -> list[dict[str, object]]: from db.repositories.batch_runs import batch_runs_repo return await batch_runs_repo.list() # type: ignore[return-value] # --------------------------------------------------------------------------- # Log viewer # --------------------------------------------------------------------------- @router.get("/api/logs") def get_logs( _: Auth, n: int = 200, level: str = "", q: str = "", ) -> dict[str, object]: import os from logger import _LOG_FILE path = _LOG_FILE if not os.path.exists(path): return {"lines": [], "file": path} with open(path, encoding="utf-8", errors="replace") as f: all_lines = f.readlines() all_lines.reverse() # newest first level_upper = level.upper() q_lower = q.lower() results: list[str] = [] for raw in all_lines: line = raw.rstrip() if not line: continue if level_upper and level_upper not in line: continue if q_lower and q_lower not in line.lower(): continue results.append(line) if len(results) >= n: break return {"lines": results, "file": path}