wip

admin/routes.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import secrets
+import uuid
 from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, HTTPException, status
@@ -17,6 +18,9 @@ from admin.models import (
     upsert_flow_config,
 )
 from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
 
 router = APIRouter()
 security = HTTPBasic()
@@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel):
     search_submit_selector: str = ""
     search_submit_selector_type: str = "css"
     search_results_selector: str = ""
+    search_results_selector_type: str = "css"
     scroll_container_selector: str = ""
+    scroll_container_selector_type: str = "css"
     max_scrolls: int = 30
     scroll_pause_ms: int = 1200
     no_new_content_timeout_ms: int = 3000
+    pagination_selector: str = ""
+    pagination_selector_type: str = "css"
+    max_pages: int = 10
+    pagination_wait_ms: int = 1500
     item_selector_template: str = ""
     item_selector_type: str = "css"
+    item_url_template: str = ""
     target_item_ids: list[str] = []
+    target_url: str = ""
 
 
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
         "search_submit_selector": body.search_submit_selector,
         "search_submit_selector_type": body.search_submit_selector_type,
         "search_results_selector": body.search_results_selector,
+        "search_results_selector_type": body.search_results_selector_type,
         "scroll_container_selector": body.scroll_container_selector,
+        "scroll_container_selector_type": body.scroll_container_selector_type,
         "max_scrolls": body.max_scrolls,
         "scroll_pause_ms": body.scroll_pause_ms,
         "no_new_content_timeout_ms": body.no_new_content_timeout_ms,
+        "pagination_selector": body.pagination_selector,
+        "pagination_selector_type": body.pagination_selector_type,
+        "max_pages": body.max_pages,
+        "pagination_wait_ms": body.pagination_wait_ms,
         "item_selector_template": body.item_selector_template,
         "item_selector_type": body.item_selector_type,
+        "item_url_template": body.item_url_template,
         "target_item_ids_json": body.target_item_ids,
+        "target_url": body.target_url,
     }
 
 
@@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
     if not existing:
         raise HTTPException(status_code=404, detail="Not found")
     await delete_flow_config(cfg_id)
+
+
+# ---------------------------------------------------------------------------
+# Batch runs (managed via Huey task queue)
+# ---------------------------------------------------------------------------
+
+class BatchRunIn(BaseModel):
+    workers: int = 3
+    total_runs: int = 10
+    stagger_ms: int = 500
+    headless: bool = True
+
+
+@router.post("/api/run-batch", status_code=202)
+async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status, run_batch_task, set_current
+
+    current = _status()
+    if current.get("running"):
+        raise HTTPException(status_code=409, detail="A batch is already running")
+
+    from admin.models import get_active_flow_config
+    active_cfg = await get_active_flow_config()
+    if not active_cfg:
+        raise HTTPException(status_code=422, detail="No active flow config — activate one first")
+
+    batch_id = uuid.uuid4().hex
+    result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
+    set_current(batch_id, result)
+
+    log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
+    return {"batch_id": batch_id, "status": "queued"}
+
+
+@router.post("/api/stop-batch", status_code=200)
+async def stop_batch(_: Auth) -> dict[str, object]:
+    from crawler.tasks import stop_current
+    stopped = stop_current()
+    if not stopped:
+        raise HTTPException(status_code=404, detail="No active batch to stop")
+    return {"status": "stopping"}
+
+
+@router.get("/api/batch-status")
+async def batch_status(_: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status
+    return _status()  # type: ignore[return-value]