387 lines
12 KiB
Python
387 lines
12 KiB
Python
"""FastAPI routes — dashboard API + auth."""
|
|
|
|
from __future__ import annotations
|
|
|
|
import secrets
|
|
from datetime import UTC, datetime
|
|
from pathlib import Path
|
|
from typing import Annotated, Any
|
|
from urllib.parse import urlsplit
|
|
|
|
import speedtest # type: ignore[import-untyped]
|
|
from fastapi import APIRouter, Depends, HTTPException, status
|
|
from fastapi.responses import FileResponse
|
|
from fastapi.security import HTTPBasic, HTTPBasicCredentials
|
|
from pydantic import BaseModel, field_validator
|
|
|
|
from admin.models import (
|
|
delete_flow_config,
|
|
get_flow_config,
|
|
get_stats,
|
|
list_flow_configs,
|
|
upsert_flow_config,
|
|
)
|
|
from config import config
|
|
from logger import get_logger
|
|
|
|
log = get_logger(__name__)
|
|
|
|
router = APIRouter()
|
|
security = HTTPBasic()
|
|
|
|
|
|
def require_auth(
|
|
credentials: Annotated[HTTPBasicCredentials, Depends(security)],
|
|
) -> str:
|
|
ok_user = secrets.compare_digest(
|
|
credentials.username.encode(), config.admin_username.encode()
|
|
)
|
|
ok_pass = secrets.compare_digest(
|
|
credentials.password.encode(), config.admin_password.encode()
|
|
)
|
|
if not (ok_user and ok_pass):
|
|
raise HTTPException(
|
|
status_code=status.HTTP_401_UNAUTHORIZED,
|
|
detail="Invalid credentials",
|
|
headers={"WWW-Authenticate": "Basic"},
|
|
)
|
|
return credentials.username
|
|
|
|
|
|
Auth = Annotated[str, Depends(require_auth)]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Stats
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@router.get("/api/stats")
|
|
async def stats(_: Auth) -> dict[str, object]:
|
|
return await get_stats()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Flow configs
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class FlowConfigIn(BaseModel):
|
|
name: str
|
|
search_texts: list[str] = []
|
|
search_box_selector: str = ""
|
|
search_box_selector_type: str = "css"
|
|
search_submit_selector: str = ""
|
|
search_submit_selector_type: str = "css"
|
|
search_results_selector: str = ""
|
|
search_results_selector_type: str = "css"
|
|
scroll_container_selector: str = ""
|
|
scroll_container_selector_type: str = "css"
|
|
max_scrolls: int = 30
|
|
scroll_pause_ms: int = 1200
|
|
no_new_content_timeout_ms: int = 3000
|
|
pagination_selector: str = ""
|
|
pagination_selector_type: str = "css"
|
|
max_pages: int = 10
|
|
pagination_wait_ms: int = 1500
|
|
item_selector_template: str = ""
|
|
item_selector_type: str = "css"
|
|
item_url_template: str = ""
|
|
target_item_ids: list[str] = []
|
|
target_url: str = ""
|
|
scenario: str = "dynamic"
|
|
stop_on_first_click: bool = False
|
|
|
|
@field_validator("target_url")
|
|
@classmethod
|
|
def validate_target_url(cls, value: str) -> str:
|
|
value = value.strip()
|
|
if value and "://" not in value:
|
|
value = f"https://{value}"
|
|
parsed = urlsplit(value)
|
|
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
|
|
raise ValueError("Target URL must be a complete HTTP(S) URL")
|
|
return value
|
|
|
|
|
|
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
|
|
return {
|
|
"name": body.name,
|
|
"search_texts_json": body.search_texts,
|
|
"search_box_selector": body.search_box_selector,
|
|
"search_box_selector_type": body.search_box_selector_type,
|
|
"search_submit_selector": body.search_submit_selector,
|
|
"search_submit_selector_type": body.search_submit_selector_type,
|
|
"search_results_selector": body.search_results_selector,
|
|
"search_results_selector_type": body.search_results_selector_type,
|
|
"scroll_container_selector": body.scroll_container_selector,
|
|
"scroll_container_selector_type": body.scroll_container_selector_type,
|
|
"max_scrolls": body.max_scrolls,
|
|
"scroll_pause_ms": body.scroll_pause_ms,
|
|
"no_new_content_timeout_ms": body.no_new_content_timeout_ms,
|
|
"pagination_selector": body.pagination_selector,
|
|
"pagination_selector_type": body.pagination_selector_type,
|
|
"max_pages": body.max_pages,
|
|
"pagination_wait_ms": body.pagination_wait_ms,
|
|
"item_selector_template": body.item_selector_template,
|
|
"item_selector_type": body.item_selector_type,
|
|
"item_url_template": body.item_url_template,
|
|
"target_item_ids_json": body.target_item_ids,
|
|
"target_url": body.target_url,
|
|
"scenario": body.scenario,
|
|
"stop_on_first_click": int(body.stop_on_first_click),
|
|
}
|
|
|
|
|
|
@router.get("/api/flow-configs")
|
|
async def list_configs(_: Auth) -> list[dict[str, object]]:
|
|
return await list_flow_configs() # type: ignore[return-value]
|
|
|
|
|
|
@router.get("/api/flow-configs/{cfg_id}")
|
|
async def get_config(cfg_id: int, _: Auth) -> dict[str, object]:
|
|
row = await get_flow_config(cfg_id)
|
|
if not row:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
return row # type: ignore[return-value]
|
|
|
|
|
|
@router.post("/api/flow-configs", status_code=201)
|
|
async def create_config(body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
new_id = await upsert_flow_config(_to_db_dict(body))
|
|
row = await get_flow_config(new_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.put("/api/flow-configs/{cfg_id}")
|
|
async def update_config(cfg_id: int, body: FlowConfigIn, _: Auth) -> dict[str, object]:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await upsert_flow_config(_to_db_dict(body), cfg_id=cfg_id)
|
|
row = await get_flow_config(cfg_id)
|
|
return row or {} # type: ignore[return-value]
|
|
|
|
|
|
@router.delete("/api/flow-configs/{cfg_id}", status_code=204)
|
|
async def delete_config(cfg_id: int, _: Auth) -> None:
|
|
existing = await get_flow_config(cfg_id)
|
|
if not existing:
|
|
raise HTTPException(status_code=404, detail="Not found")
|
|
await delete_flow_config(cfg_id)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Batch runs (managed via Huey task queue)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
class BatchTaskIn(BaseModel):
|
|
config_id: int
|
|
workers: int = 3
|
|
total_runs: int = 10
|
|
stagger_ms: int = 500
|
|
headless: bool = True
|
|
|
|
|
|
@router.post("/api/batch-queue", status_code=202)
|
|
async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
|
|
from crawler.tasks import enqueue
|
|
|
|
cfg = await get_flow_config(body.config_id)
|
|
if not cfg:
|
|
raise HTTPException(
|
|
status_code=404, detail=f"Flow config {body.config_id} not found"
|
|
)
|
|
if not str(cfg.get("target_url") or "").strip():
|
|
raise HTTPException(
|
|
status_code=422,
|
|
detail="The selected flow config has no Target URL. Edit and save it first.",
|
|
)
|
|
|
|
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
|
|
log.info(
|
|
"Task enqueued — config=%d runs=%d workers=%d status=%s",
|
|
body.config_id,
|
|
body.total_runs,
|
|
body.workers,
|
|
result["status"],
|
|
)
|
|
return result
|
|
|
|
|
|
@router.delete("/api/batch-queue/{queue_id}", status_code=200)
|
|
async def remove_from_batch_queue(queue_id: str, _: Auth) -> dict[str, object]:
|
|
from crawler.tasks import dequeue
|
|
|
|
removed = dequeue(queue_id)
|
|
if not removed:
|
|
raise HTTPException(status_code=404, detail="Queue item not found")
|
|
return {"removed": queue_id}
|
|
|
|
|
|
@router.post("/api/batch-queue/clear", status_code=200)
|
|
async def clear_batch_queue(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import clear_queue
|
|
|
|
count = clear_queue()
|
|
return {"cleared": count}
|
|
|
|
|
|
@router.post("/api/stop-batch", status_code=200)
|
|
async def stop_batch(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import stop_current
|
|
|
|
stopped = stop_current()
|
|
if not stopped:
|
|
raise HTTPException(status_code=404, detail="No active batch to stop")
|
|
return {"status": "stopping"}
|
|
|
|
|
|
@router.get("/api/batch-status")
|
|
async def batch_status(_: Auth) -> dict[str, object]:
|
|
from crawler.tasks import batch_status as _status
|
|
|
|
return _status() # type: ignore[return-value]
|
|
|
|
|
|
@router.get("/api/batch-runs")
|
|
async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
|
|
from db.repositories.batch_runs import batch_runs_repo
|
|
|
|
return await batch_runs_repo.list() # type: ignore[return-value]
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Browser-session recordings
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
def _recording_path(filename: str) -> Path:
|
|
if filename != Path(filename).name or Path(filename).suffix.lower() != ".mp4":
|
|
raise HTTPException(status_code=404, detail="Recording not found")
|
|
directory = Path(config.recordings_dir).resolve()
|
|
path = directory / filename
|
|
if path.is_symlink() or not path.is_file():
|
|
raise HTTPException(status_code=404, detail="Recording not found")
|
|
return path
|
|
|
|
|
|
@router.get("/api/recordings")
|
|
def list_recordings(_: Auth) -> list[dict[str, object]]:
|
|
directory = Path(config.recordings_dir)
|
|
if not directory.is_dir():
|
|
return []
|
|
|
|
rows: list[dict[str, object]] = []
|
|
for path in directory.glob("*.mp4"):
|
|
if path.is_symlink():
|
|
continue
|
|
try:
|
|
stat_result = path.stat()
|
|
except FileNotFoundError:
|
|
continue
|
|
rows.append(
|
|
{
|
|
"name": path.name,
|
|
"size_bytes": stat_result.st_size,
|
|
"created_at": datetime.fromtimestamp(
|
|
stat_result.st_mtime, tz=UTC
|
|
).isoformat(),
|
|
}
|
|
)
|
|
rows.sort(key=lambda row: str(row["created_at"]), reverse=True)
|
|
return rows
|
|
|
|
|
|
@router.get("/api/recordings/{filename}")
|
|
def download_recording(filename: str, _: Auth) -> FileResponse:
|
|
path = _recording_path(filename)
|
|
return FileResponse(path, media_type="video/mp4", filename=path.name)
|
|
|
|
|
|
@router.delete("/api/recordings/{filename}", status_code=204)
|
|
def delete_recording(filename: str, _: Auth) -> None:
|
|
path = _recording_path(filename)
|
|
try:
|
|
path.unlink()
|
|
except FileNotFoundError:
|
|
raise HTTPException(status_code=404, detail="Recording not found") from None
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Log viewer
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@router.get("/api/logs")
|
|
def get_logs(
|
|
_: Auth,
|
|
n: int = 200,
|
|
level: str = "",
|
|
q: str = "",
|
|
) -> dict[str, object]:
|
|
import os
|
|
|
|
from logger import _LOG_FILE
|
|
|
|
path = _LOG_FILE
|
|
if not os.path.exists(path):
|
|
return {"lines": [], "file": path}
|
|
|
|
with open(path, encoding="utf-8", errors="replace") as f:
|
|
all_lines = f.readlines()
|
|
|
|
all_lines.reverse() # newest first
|
|
|
|
level_upper = level.upper()
|
|
q_lower = q.lower()
|
|
|
|
results: list[str] = []
|
|
for raw in all_lines:
|
|
line = raw.rstrip()
|
|
if not line:
|
|
continue
|
|
if level_upper and level_upper not in line:
|
|
continue
|
|
if q_lower and q_lower not in line.lower():
|
|
continue
|
|
results.append(line)
|
|
if len(results) >= n:
|
|
break
|
|
|
|
return {"lines": results, "file": path}
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Speed test
|
|
# ---------------------------------------------------------------------------
|
|
|
|
|
|
@router.post("/api/speedtest")
|
|
async def run_speedtest(_: Auth) -> dict[str, object]:
|
|
import asyncio
|
|
|
|
def _run() -> dict[str, object]:
|
|
try:
|
|
log.info("running speedtest")
|
|
st = speedtest.Speedtest(secure=True)
|
|
st.get_best_server()
|
|
download_bps: float = st.download()
|
|
upload_bps: float = st.upload()
|
|
r = st.results.dict()
|
|
server: dict[str, object] = r.get("server") or {}
|
|
client: dict[str, object] = r.get("client") or {}
|
|
return {
|
|
"download_mbps": round(download_bps / 1_000_000, 2),
|
|
"upload_mbps": round(upload_bps / 1_000_000, 2),
|
|
"ping_ms": round(float(r.get("ping") or 0), 1),
|
|
"server": f"{server.get('name', '')} ({server.get('country', '')})",
|
|
"isp": str(client.get("isp", "")),
|
|
}
|
|
except Exception as e:
|
|
log.error(e)
|
|
|
|
loop = asyncio.get_event_loop()
|
|
return await loop.run_in_executor(None, _run)
|