248 lines
8.2 KiB
Python
248 lines
8.2 KiB
Python
"""FastAPI routes — dashboard API + auth."""
|
|
from __future__ import annotations
|
|
|
|
import secrets
|
|
from typing import Annotated, Any
|
|
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
|
from pydantic import BaseModel
|
|
|
|
from admin.models import (
|
|
delete_flow_config,
|
|
get_flow_config,
|
|
get_stats,
|
|
list_flow_configs,
|
|
upsert_flow_config,
|
|
)
|
|
from config import config
|
|
from logger import get_logger
|
|
|
|
log = get_logger(__name__)
|
|
|
|
router = APIRouter()
|
|
security = HTTPBasic()
|
|
|
|
|
|
def require_auth(credentials: Annotated[HTTPBasicCredentials, Depends(security)]) -> str:
|
|
ok_user = secrets.compare_digest(credentials.username.encode(), config.admin_username.encode())
|
|
ok_pass = secrets.compare_digest(credentials.password.encode(), config.admin_password.encode())
|
|
if not (ok_user and ok_pass):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
detail="Invalid credentials",
|
|
headers={"WWW-Authenticate": "Basic"},
|
|
)
|
|
return credentials.username
|
|
|
|
|
|
Auth = Annotated[str, Depends(require_auth)]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stats
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/api/stats")
|
|
async def stats(_: Auth) -> dict[str, object]:
|
|
return await get_stats()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Flow configs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class FlowConfigIn(BaseModel):
|
|
name: str
|
|
search_texts: list[str] = []
|
|
search_box_selector: str = ""
|
|
search_box_selector_type: str = "css"
|
|
search_submit_selector: str = ""
|
|
search_submit_selector_type: str = "css"
|
|
search_results_selector: str = ""
|
|
search_results_selector_type: str = "css"
|
|
scroll_container_selector: str = ""
|
|
scroll_container_selector_type: str = "css"
|
|
max_scrolls: int = 30
|
|
scroll_pause_ms: int = 1200
|
|
no_new_content_timeout_ms: int = 3000
|
|
pagination_selector: str = ""
|
|
pagination_selector_type: str = "css"
|
|
max_pages: int = 10
|
|
pagination_wait_ms: int = 1500
|
|
item_selector_template: str = ""
|
|
item_selector_type: str = "css"
|
|
item_url_template: str = ""
|
|
target_item_ids: list[str] = []
|
|
target_url: str = ""
|
|
|
|
|
|
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
|
return {
|
|
"name": body.name,
|
|
"search_texts_json": body.search_texts,
|
|
"search_box_selector": body.search_box_selector,
|
|
"search_box_selector_type": body.search_box_selector_type,
|
|
"search_submit_selector": body.search_submit_selector,
|
|
"search_submit_selector_type": body.search_submit_selector_type,
|
|
"search_results_selector": body.search_results_selector,
|
|
"search_results_selector_type": body.search_results_selector_type,
|
|
"scroll_container_selector": body.scroll_container_selector,
|
|
"scroll_container_selector_type": body.scroll_container_selector_type,
|
|
"max_scrolls": body.max_scrolls,
|
|
"scroll_pause_ms": body.scroll_pause_ms,
|
|
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
|
|
"pagination_selector": body.pagination_selector,
|
|
"pagination_selector_type": body.pagination_selector_type,
|
|
"max_pages": body.max_pages,
|
|
"pagination_wait_ms": body.pagination_wait_ms,
|
|
"item_selector_template": body.item_selector_template,
|
|
"item_selector_type": body.item_selector_type,
|
|
"item_url_template": body.item_url_template,
|
|
"target_item_ids_json": body.target_item_ids,
|
|
"target_url": body.target_url,
|
|
}
|
|
|
|
|
|
@router.get("/api/flow-configs")
|
|
async def list_configs(_: Auth) -> list[dict[str, object]]:
|
|
return await list_flow_configs() # type: ignore[return-value]
|
|
|
|
|
|
@router.get("/api/flow-configs/{cfg_id}")
|
|
async def get_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
|
row = await get_flow_config(cfg_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
return row # type: ignore[return-value]
|
|
|
|
|
|
@router.post("/api/flow-configs", status_code=201)
|
|
async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
new_id = await upsert_flow_config(_to_db_dict(body))
|
|
row = await get_flow_config(new_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.put("/api/flow-configs/{cfg_id}")
|
|
async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id)
|
|
row = await get_flow_config(cfg_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.delete("/api/flow-configs/{cfg_id}", status_code=204)
|
|
async def delete_config(cfg_id: int, _: Auth) -> None:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await delete_flow_config(cfg_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch runs (managed via Huey task queue)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
class BatchTaskIn(BaseModel):
|
|
config_id: int
|
|
workers: int = 3
|
|
total_runs: int = 10
|
|
stagger_ms: int = 500
|
|
headless: bool = True
|
|
|
|
|
|
@router.post("/api/batch-queue", status_code=202)
|
|
async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
|
|
from crawler.tasks import enqueue
|
|
|
|
cfg = await get_flow_config(body.config_id)
|
|
if not cfg:
|
|
raise HTTPException(status_code=404, detail=f"Flow config {body.config_id} not found")
|
|
|
|
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
|
|
log.info("Task enqueued — config=%d runs=%d workers=%d status=%s",
|
|
body.config_id, body.total_runs, body.workers, result["status"])
|
|
return result
|
|
|
|
|
|
@router.delete("/api/batch-queue/{queue_id}", status_code=200)
|
|
async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
|
|
from crawler.tasks import dequeue
|
|
removed = dequeue(queue_id)
|
|
if not removed:
|
|
raise HTTPException(status_code=404, detail="Queue item not found")
|
|
return {"removed": queue_id}
|
|
|
|
|
|
@router.post("/api/batch-queue/clear", status_code=200)
|
|
async def clear_batch_queue(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import clear_queue
|
|
count = clear_queue()
|
|
return {"cleared": count}
|
|
|
|
|
|
@router.post("/api/stop-batch", status_code=200)
|
|
async def stop_batch(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import stop_current
|
|
stopped = stop_current()
|
|
if not stopped:
|
|
raise HTTPException(status_code=404, detail="No active batch to stop")
|
|
return {"status": "stopping"}
|
|
|
|
|
|
@router.get("/api/batch-status")
|
|
async def batch_status(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import batch_status as _status
|
|
return _status() # type: ignore[return-value]
|
|
|
|
|
|
@router.get("/api/batch-runs")
|
|
async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
|
|
from db.repositories.batch_runs import batch_runs_repo
|
|
return await batch_runs_repo.list() # type: ignore[return-value]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Log viewer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
@router.get("/api/logs")
|
|
def get_logs(
|
|
_: Auth,
|
|
n: int = 200,
|
|
level: str = "",
|
|
q: str = "",
|
|
) -> dict[str, object]:
|
|
import os
|
|
from logger import _LOG_FILE
|
|
|
|
path = _LOG_FILE
|
|
if not os.path.exists(path):
|
|
return {"lines": [], "file": path}
|
|
|
|
with open(path, encoding="utf-8", errors="replace") as f:
|
|
all_lines = f.readlines()
|
|
|
|
all_lines.reverse() # newest first
|
|
|
|
level_upper = level.upper()
|
|
q_lower = q.lower()
|
|
|
|
results: list[str] = []
|
|
for raw in all_lines:
|
|
line = raw.rstrip()
|
|
if not line:
|
|
continue
|
|
if level_upper and level_upper not in line:
|
|
continue
|
|
if q_lower and q_lower not in line.lower():
|
|
continue
|
|
results.append(line)
|
|
if len(results) >= n:
|
|
break
|
|
|
|
return {"lines": results, "file": path}
|