203 lines
7.2 KiB
Python
203 lines
7.2 KiB
Python
"""FastAPI routes — dashboard API + auth."""
|
|
from __future__ import annotations
|
|
|
|
import secrets
|
|
import uuid
|
|
from typing import Annotated, Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
|
from pydantic import BaseModel
|
|
|
|
from admin.models import (
|
|
activate_flow_config,
|
|
delete_flow_config,
|
|
get_flow_config,
|
|
get_stats,
|
|
list_flow_configs,
|
|
upsert_flow_config,
|
|
)
|
|
from config import config
|
|
from logger import get_logger
|
|
|
|
log = get_logger(__name__)
|
|
|
|
router = APIRouter()
|
|
security = HTTPBasic()
|
|
|
|
|
|
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
|
|
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
|
|
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
|
|
if not (ok_user and ok_pass):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
detail="Invalid credentials",
|
|
headers={"WWW-Authenticate": "Basic"},
|
|
)
|
|
return credentials.username
|
|
|
|
|
|
Auth = Annotated[str, Depends(require_auth)]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stats
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/api/stats")
|
|
async def stats(_: Auth) -> dict[str, object]:
|
|
return await get_stats()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Flow configs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FlowConfigIn(BaseModel):
|
|
name: str
|
|
is_active: bool = False
|
|
search_texts: list[str] = []
|
|
search_box_selector: str = ""
|
|
search_box_selector_type: str = "css"
|
|
search_submit_selector: str = ""
|
|
search_submit_selector_type: str = "css"
|
|
search_results_selector: str = ""
|
|
search_results_selector_type: str = "css"
|
|
scroll_container_selector: str = ""
|
|
scroll_container_selector_type: str = "css"
|
|
max_scrolls: int = 30
|
|
scroll_pause_ms: int = 1200
|
|
no_new_content_timeout_ms: int = 3000
|
|
pagination_selector: str = ""
|
|
pagination_selector_type: str = "css"
|
|
max_pages: int = 10
|
|
pagination_wait_ms: int = 1500
|
|
item_selector_template: str = ""
|
|
item_selector_type: str = "css"
|
|
item_url_template: str = ""
|
|
target_item_ids: list[str] = []
|
|
target_url: str = ""
|
|
|
|
|
|
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
|
return {
|
|
"name": body.name,
|
|
"is_active": int(body.is_active),
|
|
"search_texts_json": body.search_texts,
|
|
"search_box_selector": body.search_box_selector,
|
|
"search_box_selector_type": body.search_box_selector_type,
|
|
"search_submit_selector": body.search_submit_selector,
|
|
"search_submit_selector_type": body.search_submit_selector_type,
|
|
"search_results_selector": body.search_results_selector,
|
|
"search_results_selector_type": body.search_results_selector_type,
|
|
"scroll_container_selector": body.scroll_container_selector,
|
|
"scroll_container_selector_type": body.scroll_container_selector_type,
|
|
"max_scrolls": body.max_scrolls,
|
|
"scroll_pause_ms": body.scroll_pause_ms,
|
|
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
|
|
"pagination_selector": body.pagination_selector,
|
|
"pagination_selector_type": body.pagination_selector_type,
|
|
"max_pages": body.max_pages,
|
|
"pagination_wait_ms": body.pagination_wait_ms,
|
|
"item_selector_template": body.item_selector_template,
|
|
"item_selector_type": body.item_selector_type,
|
|
"item_url_template": body.item_url_template,
|
|
"target_item_ids_json": body.target_item_ids,
|
|
"target_url": body.target_url,
|
|
}
|
|
|
|
|
|
@router.get("/api/flow-configs")
|
|
async def list_configs(_: Auth) -> list[dict[str, object]]:
|
|
return await list_flow_configs() # type: ignore[return-value]
|
|
|
|
|
|
@router.get("/api/flow-configs/{cfg_id}")
|
|
async def get_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
|
row = await get_flow_config(cfg_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
return row # type: ignore[return-value]
|
|
|
|
|
|
@router.post("/api/flow-configs", status_code=201)
|
|
async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
new_id = await upsert_flow_config(_to_db_dict(body))
|
|
row = await get_flow_config(new_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.put("/api/flow-configs/{cfg_id}")
|
|
async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id)
|
|
row = await get_flow_config(cfg_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.post("/api/flow-configs/{cfg_id}/activate", status_code=200)
|
|
async def activate_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await activate_flow_config(cfg_id)
|
|
return {"activated": cfg_id}
|
|
|
|
|
|
@router.delete("/api/flow-configs/{cfg_id}", status_code=204)
|
|
async def delete_config(cfg_id: int, _: Auth) -> None:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await delete_flow_config(cfg_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch runs (managed via Huey task queue)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class BatchRunIn(BaseModel):
|
|
workers: int = 3
|
|
total_runs: int = 10
|
|
stagger_ms: int = 500
|
|
headless: bool = True
|
|
|
|
|
|
@router.post("/api/run-batch", status_code=202)
|
|
async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
|
|
from crawler.tasks import batch_status as _status, run_batch_task, set_current
|
|
|
|
current = _status()
|
|
if current.get("running"):
|
|
raise HTTPException(status_code=409, detail="A batch is already running")
|
|
|
|
from admin.models import get_active_flow_config
|
|
active_cfg = await get_active_flow_config()
|
|
if not active_cfg:
|
|
raise HTTPException(status_code=422, detail="No active flow config — activate one first")
|
|
|
|
batch_id = uuid.uuid4().hex
|
|
result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
|
|
set_current(batch_id, result)
|
|
|
|
log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
|
|
return {"batch_id": batch_id, "status": "queued"}
|
|
|
|
|
|
@router.post("/api/stop-batch", status_code=200)
|
|
async def stop_batch(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import stop_current
|
|
stopped = stop_current()
|
|
if not stopped:
|
|
raise HTTPException(status_code=404, detail="No active batch to stop")
|
|
return {"status": "stopping"}
|
|
|
|
|
|
@router.get("/api/batch-status")
|
|
async def batch_status(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import batch_status as _status
|
|
return _status() # type: ignore[return-value]
|