wip

crawler/batch.py

@@ -0,0 +1,117 @@
+"""Batch runner — spawn N parallel Chrome workers to execute the active flow config."""
+from __future__ import annotations
+
+import asyncio
+import threading
+import time
+import uuid
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Any, Callable
+
+from logger import get_logger
+
+log = get_logger(__name__)
+
+
+def _single_run(
+    run_index: int,
+    batch_id: str,
+    cfg: dict[str, Any],
+    headless: bool,
+) -> tuple[bool, str | None]:
+    """One isolated Chrome session. Returns (success, failure_reason)."""
+    from crawler.driver import driver_session
+    from crawler.dynamic_flow import DynamicFlow
+    from crawler.scenarios.fresh_session import FreshSessionScenario
+    from db.repositories.runs import runs_repo
+
+    identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
+    t0 = time.monotonic()
+    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+    try:
+        target_url: str = cfg.get("target_url") or ""
+        with driver_session(headless=headless) as driver:
+            FreshSessionScenario(driver, target_url).run()
+            result = DynamicFlow(driver, cfg).run() if cfg else None
+
+        total_ms = int((time.monotonic() - t0) * 1000)
+        if result:
+            steps_data = [
+                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
+                for s in result.steps
+            ]
+            asyncio.run(runs_repo.insert(
+                "batch", identifier, result.success,
+                result.failure_reason, steps_data, total_ms,
+            ))
+            log.info("[batch:%s] Run %d done — success=%s (%d ms)", batch_id[:8], run_index, result.success, total_ms)
+            return result.success, result.failure_reason
+        else:
+            asyncio.run(runs_repo.insert("batch", identifier, True, None, [], total_ms))
+            log.info("[batch:%s] Run %d done (no flow config) — %d ms", batch_id[:8], run_index, total_ms)
+            return True, None
+
+    except Exception as exc:
+        total_ms = int((time.monotonic() - t0) * 1000)
+        asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
+        log.error("[batch:%s] Run %d failed — %s (%d ms)", batch_id[:8], run_index, exc, total_ms)
+        return False, str(exc)
+
+
+def run_batch(
+    cfg: dict[str, Any],
+    workers: int,
+    total_runs: int,
+    headless: bool = True,
+    stagger_ms: int = 0,
+    on_progress: Callable[[int, int, int], None] | None = None,
+) -> dict[str, Any]:
+    """
+    Run `total_runs` crawler sessions with up to `workers` parallel Chrome processes.
+
+    `stagger_ms` delays each successive submission to the pool, spreading
+    Chrome startup load and reducing bot-detection fingerprint clustering.
+
+    `on_progress(completed, succeeded, failed)` is called after every finished run.
+
+    Returns {"batch_id", "total", "succeeded", "failed"}.
+    """
+    batch_id = uuid.uuid4().hex
+    log.info(
+        "Batch %s — %d run(s), %d worker(s), stagger=%d ms, headless=%s",
+        batch_id[:8], total_runs, workers, stagger_ms, headless,
+    )
+
+    succeeded = 0
+    failed = 0
+    lock = threading.Lock()
+
+    with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=f"batch-{batch_id[:8]}") as pool:
+        futures: list[Future[tuple[bool, str | None]]] = []
+        for i in range(total_runs):
+            futures.append(pool.submit(_single_run, i, batch_id, cfg, headless))
+            if stagger_ms > 0 and i < total_runs - 1:
+                time.sleep(stagger_ms / 1000)
+
+        for fut in as_completed(futures):
+            try:
+                success, _ = fut.result()
+            except Exception as exc:
+                log.error("[batch:%s] Unhandled thread exception: %s", batch_id[:8], exc)
+                success = False
+
+            with lock:
+                if success:
+                    succeeded += 1
+                else:
+                    failed += 1
+                completed = succeeded + failed
+
+            if on_progress:
+                on_progress(completed, succeeded, failed)
+
+    log.info(
+        "Batch %s finished — %d/%d succeeded, %d failed",
+        batch_id[:8], succeeded, total_runs, failed,
+    )
+    return {"batch_id": batch_id, "total": total_runs, "succeeded": succeeded, "failed": failed}