wip

.env.example

@@ -1,6 +1,8 @@
 # Copy to .env and fill in
 
-TARGET_URL=https://example.com
+# Logging
+LOG_LEVEL=INFO        # DEBUG | INFO | WARNING | ERROR | CRITICAL
+# LOG_FILE=logs/seed.log   # optional rotating file log
 
 # Admin panel credentials
 ADMIN_USERNAME=admin
@@ -13,7 +15,8 @@ DB_PATH=data/tracker.db
 
 # Chrome: set to false to watch the browser window
 HEADLESS=true
-# CHROME_BINARY=/usr/bin/chromium  # optional: explicit Chrome path
+# CHROME_BINARY=/usr/bin/google-chrome-stable  # optional: explicit Chrome binary path
+# CHROMEDRIVER_PATH=drivers/chromedriver       # pre-patched driver (run: make patch-driver)
 
 # Scenario 1 — comma-separated phone numbers
 PHONE_NUMBERS=+989100000001,+989100000002,+989100000003

.gitignore

@@ -5,3 +5,4 @@ __pycache__/
 data/
 *.db
 *.swp
+drivers/

Makefile

@@ -11,7 +11,7 @@ GREEN := $(shell tput setaf 2 2>/dev/null)
 CYAN  := $(shell tput setaf 6 2>/dev/null)
 
 .PHONY: help install sync lock lint typecheck \
-        admin otp fresh \
+        admin otp fresh batch \
         build up down logs shell \
         clean
 
@@ -46,6 +46,10 @@ typecheck: ## Run Pyright type-checker
 
 # ── Crawler ───────────────────────────────────────────────────────────────────
 
+patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chrome updates)
+	@mkdir -p drivers
+	PYTHONPATH=. $(PYTHON) scripts/patch_driver.py
+
 admin: ## Start the admin panel (http://localhost:8000)
 	@mkdir -p $(DATA_DIR)
 	$(PYTHON) main.py admin
@@ -62,6 +66,14 @@ fresh: ## Run Scenario 2 — fresh cookie session
 	@mkdir -p $(DATA_DIR)
 	$(PYTHON) main.py fresh
 
+WORKERS ?= 3
+RUNS    ?= 10
+STAGGER ?= 500
+
+batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500)
+	@mkdir -p $(DATA_DIR)
+	$(PYTHON) main.py batch --workers $(WORKERS) --runs $(RUNS) --stagger $(STAGGER)
+
 # ── Docker ────────────────────────────────────────────────────────────────────
 
 build: ## Build the Docker image

admin/main.py

@@ -1,6 +1,7 @@
 """FastAPI admin application."""
 from __future__ import annotations
 
+import threading
 from pathlib import Path
 
 from fastapi import FastAPI
@@ -21,6 +22,20 @@ app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
 @app.on_event("startup")
 def on_startup() -> None:
     init_db()
+    _start_huey_consumer()
+
+
+def _start_huey_consumer() -> None:
+    from huey.consumer import Consumer
+    from crawler.tasks import huey
+
+    class _ThreadConsumer(Consumer):
+        def _set_signal_handlers(self) -> None:
+            pass  # signal.signal() only works on the main thread
+
+    consumer = _ThreadConsumer(huey, workers=1, periodic=False)
+    t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
+    t.start()
 
 
 @app.get("/")

admin/routes.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import secrets
+import uuid
 from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, HTTPException, status
@@ -17,6 +18,9 @@ from admin.models import (
     upsert_flow_config,
 )
 from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
 
 router = APIRouter()
 security = HTTPBasic()
@@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel):
     search_submit_selector: str = ""
     search_submit_selector_type: str = "css"
     search_results_selector: str = ""
+    search_results_selector_type: str = "css"
     scroll_container_selector: str = ""
+    scroll_container_selector_type: str = "css"
     max_scrolls: int = 30
     scroll_pause_ms: int = 1200
     no_new_content_timeout_ms: int = 3000
+    pagination_selector: str = ""
+    pagination_selector_type: str = "css"
+    max_pages: int = 10
+    pagination_wait_ms: int = 1500
     item_selector_template: str = ""
     item_selector_type: str = "css"
+    item_url_template: str = ""
     target_item_ids: list[str] = []
+    target_url: str = ""
 
 
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
         "search_submit_selector": body.search_submit_selector,
         "search_submit_selector_type": body.search_submit_selector_type,
         "search_results_selector": body.search_results_selector,
+        "search_results_selector_type": body.search_results_selector_type,
         "scroll_container_selector": body.scroll_container_selector,
+        "scroll_container_selector_type": body.scroll_container_selector_type,
         "max_scrolls": body.max_scrolls,
         "scroll_pause_ms": body.scroll_pause_ms,
         "no_new_content_timeout_ms": body.no_new_content_timeout_ms,
+        "pagination_selector": body.pagination_selector,
+        "pagination_selector_type": body.pagination_selector_type,
+        "max_pages": body.max_pages,
+        "pagination_wait_ms": body.pagination_wait_ms,
         "item_selector_template": body.item_selector_template,
         "item_selector_type": body.item_selector_type,
+        "item_url_template": body.item_url_template,
         "target_item_ids_json": body.target_item_ids,
+        "target_url": body.target_url,
     }
 
 
@@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
     if not existing:
         raise HTTPException(status_code=404, detail="Not found")
     await delete_flow_config(cfg_id)
+
+
+# ---------------------------------------------------------------------------
+# Batch runs (managed via Huey task queue)
+# ---------------------------------------------------------------------------
+
+class BatchRunIn(BaseModel):
+    workers: int = 3
+    total_runs: int = 10
+    stagger_ms: int = 500
+    headless: bool = True
+
+
+@router.post("/api/run-batch", status_code=202)
+async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status, run_batch_task, set_current
+
+    current = _status()
+    if current.get("running"):
+        raise HTTPException(status_code=409, detail="A batch is already running")
+
+    from admin.models import get_active_flow_config
+    active_cfg = await get_active_flow_config()
+    if not active_cfg:
+        raise HTTPException(status_code=422, detail="No active flow config — activate one first")
+
+    batch_id = uuid.uuid4().hex
+    result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
+    set_current(batch_id, result)
+
+    log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
+    return {"batch_id": batch_id, "status": "queued"}
+
+
+@router.post("/api/stop-batch", status_code=200)
+async def stop_batch(_: Auth) -> dict[str, object]:
+    from crawler.tasks import stop_current
+    stopped = stop_current()
+    if not stopped:
+        raise HTTPException(status_code=404, detail="No active batch to stop")
+    return {"status": "stopping"}
+
+
+@router.get("/api/batch-status")
+async def batch_status(_: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status
+    return _status()  # type: ignore[return-value]

admin/static/dashboard.html

@@ -102,6 +102,17 @@
     }
     .form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
     .form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
+    details.adv { margin-top: 20px; border: 1px solid var(--border); border-radius: 9px; }
+    details.adv > summary {
+      list-style: none; cursor: pointer; padding: 11px 14px;
+      font-size: 12px; font-weight: 600; color: var(--muted);
+      display: flex; align-items: center; gap: 7px; user-select: none;
+    }
+    details.adv > summary::-webkit-details-marker { display: none; }
+    details.adv > summary::before { content: '▶'; font-size: 9px; transition: transform .15s; }
+    details.adv[open] > summary::before { transform: rotate(90deg); }
+    details.adv > summary:hover { color: var(--text); }
+    .adv-body { padding: 4px 14px 14px; }
     .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
     .fg { margin-bottom: 14px; }
     .fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
@@ -124,6 +135,9 @@
     .modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
     .error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
     .error-msg.hidden { display: none; }
+    .remember-row { display: flex; align-items: center; gap: 8px; margin-top: 12px; }
+    .remember-row input[type=checkbox] { width: 15px; height: 15px; accent-color: var(--accent); cursor: pointer; }
+    .remember-row label { color: var(--muted); font-size: 13px; cursor: pointer; user-select: none; }
   </style>
 </head>
 <body>
@@ -134,7 +148,11 @@
     <h2>Seed Admin</h2>
     <div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
     <div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
-    <button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
+    <div class="remember-row">
+      <input type="checkbox" id="inp-remember" />
+      <label for="inp-remember">Remember me</label>
+    </div>
+    <button class="btn btn-primary" style="width:100%;margin-top:14px" id="login-btn">Sign in</button>
     <p class="error-msg hidden" id="auth-error">Invalid credentials</p>
   </div>
 </div>
@@ -149,18 +167,34 @@
       <label>Config Name</label>
       <input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
     </div>
+
+    <div class="fg">
+      <label>Target URL</label>
+      <input id="cfg-target-url" type="url" placeholder="https://example.com" />
+    </div>
+
+    <div class="fg" style="margin-top:6px">
+      <label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
+      <textarea id="cfg-search-texts" placeholder="shoes&#10;blue sneakers&#10;running shoes"></textarea>
+    </div>
+
+    <div class="fg">
+      <label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
+      <textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
+    </div>
+
+    <!-- Advanced -->
+    <details class="adv" id="adv-details">
+      <summary>Advanced Settings</summary>
+      <div class="adv-body">
+
         <div class="fg">
           <label style="display:flex;align-items:center;gap:8px;cursor:pointer">
             <input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
           </label>
         </div>
 
-    <!-- Search -->
         <div class="form-section">Search Step</div>
-    <div class="fg">
-      <label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
-      <textarea id="cfg-search-texts" placeholder="shoes&#10;blue sneakers&#10;running shoes"></textarea>
-    </div>
         <div class="form-row">
           <div class="fg">
             <label>Search Box Selector</label>
@@ -173,7 +207,7 @@
         </div>
         <div class="form-row">
           <div class="fg">
-        <label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
+            <label>Submit Button Selector <span style="color:var(--muted)">(blank → Enter key)</span></label>
             <input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
           </div>
           <div class="fg">
@@ -181,16 +215,24 @@
             <select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
           </div>
         </div>
+        <div class="form-row">
           <div class="fg">
             <label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
             <input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
           </div>
+          <div class="fg">
+            <label>Type</label>
+            <select id="cfg-results-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
 
-    <!-- Scroll -->
+        <div class="form-section">Scroll</div>
-    <div class="form-section">Scroll Step (Infinite Scroll)</div>
         <div class="fg">
           <label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
-      <input id="cfg-scroll-container" type="text" placeholder=".product-list" />
+          <div style="display:flex;gap:8px;align-items:center">
+            <input id="cfg-scroll-container" type="text" placeholder=".product-list" style="flex:1" />
+            <select id="cfg-scroll-container-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
         </div>
         <div class="form-row">
           <div class="fg">
@@ -203,25 +245,51 @@
           </div>
         </div>
         <div class="fg">
-      <label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
+          <label>No New Content Timeout (ms)</label>
           <input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
         </div>
 
-    <!-- Item -->
+        <div class="form-section">Pagination <span style="font-weight:400;color:var(--muted)">(optional)</span></div>
+        <div class="fg">
+          <label>Next Page Button Selector <span style="color:var(--muted)">(blank = disabled)</span></label>
+          <div style="display:flex;gap:8px;align-items:center">
+            <input id="cfg-pagination-sel" type="text" placeholder=".pagination .next, a[rel='next']" style="flex:1" />
+            <select id="cfg-pagination-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Max Pages</label>
+            <input id="cfg-max-pages" type="number" min="1" max="200" value="10" />
+          </div>
+          <div class="fg">
+            <label>Wait After Page Turn (ms)</label>
+            <input id="cfg-pagination-wait" type="number" min="200" max="15000" value="1500" />
+          </div>
+        </div>
+
         <div class="form-section">Item Selection</div>
         <div class="fg">
-      <label>Item Selector Template</label>
+          <label>Item URL Template <span style="color:var(--muted)">(href pattern, highest priority)</span></label>
-      <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
+          <input id="cfg-item-url-tpl" type="text" placeholder="/products/{item_id}" />
-      <p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
+          <p class="hint">Finds <code>&lt;a href&gt;</code> whose href contains this pattern. Use <code>{item_id}</code> as placeholder.</p>
+        </div>
+        <div class="fg">
+          <label>Item Selector Template <span style="color:var(--muted)">(fallback)</span></label>
+          <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or /products/{item_id}" />
         </div>
         <div class="fg">
           <label>Selector Type</label>
-      <select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          <select id="cfg-item-sel-type" style="width:auto">
+            <option value="css">CSS</option>
+            <option value="xpath">XPath</option>
+            <option value="a-link">A-Link (href contains)</option>
+          </select>
+          <p class="hint" id="alink-hint" style="display:none">Template matched against <code>&lt;a href&gt;</code> — e.g. <code>/products/{item_id}</code>.</p>
         </div>
-    <div class="fg">
+
-      <label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
-      <textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
       </div>
+    </details>
 
     <div class="form-footer">
       <button class="btn btn-ghost" id="form-cancel">Cancel</button>
@@ -244,6 +312,7 @@
 <div class="tabs">
   <div class="tab active" data-tab="dashboard">Dashboard</div>
   <div class="tab" data-tab="flows">Flow Configs</div>
+  <div class="tab" data-tab="batch">Batch Run</div>
 </div>
 
 <!-- ── Dashboard Tab ──────────────────────────────────────────── -->
@@ -288,8 +357,12 @@
       <div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
       <div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
     </div>
+    <div style="display:flex;gap:8px">
+      <button class="btn btn-ghost" id="btn-import-config">↑ Import</button>
       <button class="btn btn-primary" id="btn-new-config">+ New Config</button>
     </div>
+  </div>
+  <input type="file" id="import-file-input" accept=".json" style="display:none">
 
   <div class="config-list" id="config-list">
     <p class="empty">No configs yet. Create one to get started.</p>
@@ -297,6 +370,96 @@
 </main>
 </div>
 
+<!-- ── Batch Run Tab ─────────────────────────────────────────── -->
+<div class="tab-panel" id="tab-batch">
+<main>
+  <div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div>
+  <div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using the active flow config.</div>
+
+  <div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px">
+
+    <!-- Controls -->
+    <div class="panel" style="align-self:start">
+      <div class="section-title">Configuration</div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label>
+        <input id="batch-workers" type="number" min="1" max="50" value="3" />
+      </div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Total Runs</label>
+        <input id="batch-runs" type="number" min="1" max="10000" value="10" />
+      </div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Stagger Between Starts (ms)</label>
+        <input id="batch-stagger" type="number" min="0" max="30000" value="500" />
+      </div>
+      <div style="display:flex;align-items:center;gap:8px;margin-bottom:20px">
+        <input id="batch-headless" type="checkbox" checked style="width:16px;height:16px;cursor:pointer" />
+        <label for="batch-headless" style="cursor:pointer;margin:0">Headless</label>
+      </div>
+      <div style="display:flex;gap:10px">
+        <button class="btn btn-primary" id="batch-start-btn" style="flex:1">▶ Start</button>
+        <button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button>
+      </div>
+      <div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div>
+    </div>
+
+    <!-- Status -->
+    <div class="panel">
+      <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
+        <div class="section-title" style="margin:0">Live Status</div>
+        <span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span>
+      </div>
+      <div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div>
+      <div id="batch-status-body" style="display:none">
+        <div style="margin-bottom:16px">
+          <div style="display:flex;justify-content:space-between;margin-bottom:6px">
+            <span style="color:var(--muted);font-size:12px">Progress</span>
+            <span id="batch-progress-label" style="font-size:12px">0 / 0</span>
+          </div>
+          <div style="background:var(--surface2);border-radius:4px;height:8px;overflow:hidden">
+            <div id="batch-progress-bar" style="height:100%;background:var(--accent);width:0%;transition:width .3s"></div>
+          </div>
+        </div>
+        <div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:10px;margin-bottom:16px">
+          <div class="card" style="padding:12px">
+            <div class="label">Completed</div>
+            <div class="value accent" id="bs-completed">0</div>
+          </div>
+          <div class="card" style="padding:12px">
+            <div class="label">Succeeded</div>
+            <div class="value green" id="bs-succeeded">0</div>
+          </div>
+          <div class="card" style="padding:12px">
+            <div class="label">Failed</div>
+            <div class="value red" id="bs-failed">0</div>
+          </div>
+        </div>
+        <div style="font-size:12px;color:var(--muted);margin-bottom:14px">
+          Workers: <strong id="bs-workers">–</strong> &nbsp;·&nbsp;
+          Started: <strong id="bs-started">–</strong>
+          <span id="bs-finished-row" style="display:none"> &nbsp;·&nbsp; Finished: <strong id="bs-finished">–</strong></span>
+        </div>
+
+        <!-- Per-worker table -->
+        <div class="section-title">Workers</div>
+        <table id="bs-worker-table" style="font-size:12px">
+          <thead>
+            <tr>
+              <th>#</th>
+              <th>Status</th>
+              <th>Duration</th>
+              <th>Error</th>
+            </tr>
+          </thead>
+          <tbody id="bs-worker-tbody"></tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+</main>
+</div>
+
 <script>
 /* ─────────────────────────── Auth ─────────────────────────── */
 let _creds = null;
@@ -309,24 +472,73 @@ async function apiFetch(path, opts = {}) {
   return res;
 }
 
-function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
+const _STORAGE_KEY = 'seed_admin_creds';
-function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
+
+function showAuth() {
+  document.getElementById('auth-modal').classList.remove('hidden');
+}
+function hideAuth() {
+  document.getElementById('auth-modal').classList.add('hidden');
+}
+
+function _loadSaved() {
+  try {
+    const raw = localStorage.getItem(_STORAGE_KEY);
+    if (!raw) return null;
+    return JSON.parse(raw);
+  } catch { return null; }
+}
+
+function _saveCreds(u, p) {
+  localStorage.setItem(_STORAGE_KEY, JSON.stringify({ u, p }));
+}
+
+function _clearCreds() {
+  localStorage.removeItem(_STORAGE_KEY);
+}
+
+async function _attemptLogin(u, p) {
+  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
+  return res;
+}
 
 document.getElementById('login-btn').addEventListener('click', async () => {
   const u = document.getElementById('inp-user').value.trim();
   const p = document.getElementById('inp-pass').value;
-  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
+  const remember = document.getElementById('inp-remember').checked;
+  const res = await _attemptLogin(u, p);
   if (res.ok) {
     _creds = { u, p };
+    remember ? _saveCreds(u, p) : _clearCreds();
     document.getElementById('auth-error').classList.add('hidden');
     hideAuth();
     await Promise.all([loadStats(), loadConfigs()]);
     startAutoRefresh();
   } else {
+    _clearCreds();
     document.getElementById('auth-error').classList.remove('hidden');
   }
 });
-document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
+document.getElementById('inp-pass').addEventListener('keydown', e => {
+  if (e.key === 'Enter') document.getElementById('login-btn').click();
+});
+
+// Auto-login from saved credentials on page load
+(async () => {
+  const saved = _loadSaved();
+  if (!saved) return;
+  const res = await _attemptLogin(saved.u, saved.p);
+  if (res.ok) {
+    _creds = saved;
+    document.getElementById('inp-user').value = saved.u;
+    document.getElementById('inp-remember').checked = true;
+    hideAuth();
+    await Promise.all([loadStats(), loadConfigs()]);
+    startAutoRefresh();
+  } else {
+    _clearCreds(); // saved creds are stale (password changed)
+  }
+})();
 
 /* ─────────────────────────── Tabs ─────────────────────────── */
 document.querySelectorAll('.tab').forEach(tab => {
@@ -336,6 +548,7 @@ document.querySelectorAll('.tab').forEach(tab => {
     tab.classList.add('active');
     document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
     if (tab.dataset.tab === 'flows') loadConfigs();
+    if (tab.dataset.tab === 'batch') pollBatchStatus();
   });
 });
 
@@ -426,11 +639,15 @@ function renderConfigs() {
         </div>
         <div class="config-meta" style="margin-top:4px">
           Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
+          ${c.item_url_template ? `&nbsp;·&nbsp; href pattern: <code style="font-size:11px">${esc(c.item_url_template)}</code>` : ''}
+          ${c.pagination_selector ? `&nbsp;·&nbsp; Pagination: <code style="font-size:11px">${esc(c.pagination_selector)}</code> (max ${c.max_pages} pages)` : ''}
         </div>
       </div>
       <div class="config-actions">
         ${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
         <button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
+        <button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button>
+        <button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button>
         <button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
       </div>
     </div>
@@ -449,25 +666,143 @@ async function deleteConfig(id) {
   if (res && (res.ok || res.status === 204)) await loadConfigs();
 }
 
+function exportConfig(id) {
+  const c = _configs.find(x => x.id === id);
+  if (!c) return;
+  const out = {
+    name: c.name,
+    search_texts: c.search_texts_json || [],
+    search_box_selector: c.search_box_selector,
+    search_box_selector_type: c.search_box_selector_type,
+    search_submit_selector: c.search_submit_selector,
+    search_submit_selector_type: c.search_submit_selector_type,
+    search_results_selector: c.search_results_selector,
+    search_results_selector_type: c.search_results_selector_type,
+    scroll_container_selector: c.scroll_container_selector,
+    scroll_container_selector_type: c.scroll_container_selector_type,
+    max_scrolls: c.max_scrolls,
+    scroll_pause_ms: c.scroll_pause_ms,
+    no_new_content_timeout_ms: c.no_new_content_timeout_ms,
+    pagination_selector: c.pagination_selector,
+    pagination_selector_type: c.pagination_selector_type,
+    max_pages: c.max_pages,
+    pagination_wait_ms: c.pagination_wait_ms,
+    item_selector_template: c.item_selector_template,
+    item_selector_type: c.item_selector_type,
+    item_url_template: c.item_url_template,
+    target_item_ids: c.target_item_ids_json || [],
+    target_url: c.target_url || '',
+  };
+  const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
+  const url = URL.createObjectURL(blob);
+  const a = document.createElement('a');
+  a.href = url; a.download = `${c.name.replace(/[^a-z0-9_-]/gi, '_')}.json`;
+  a.click(); URL.revokeObjectURL(url);
+}
+
+async function duplicateConfig(id) {
+  const c = _configs.find(x => x.id === id);
+  if (!c) return;
+  const body = {
+    name: c.name + ' (copy)',
+    is_active: false,
+    search_texts: c.search_texts_json || [],
+    search_box_selector: c.search_box_selector || '',
+    search_box_selector_type: c.search_box_selector_type || 'css',
+    search_submit_selector: c.search_submit_selector || '',
+    search_submit_selector_type: c.search_submit_selector_type || 'css',
+    search_results_selector: c.search_results_selector || '',
+    search_results_selector_type: c.search_results_selector_type || 'css',
+    scroll_container_selector: c.scroll_container_selector || '',
+    scroll_container_selector_type: c.scroll_container_selector_type || 'css',
+    max_scrolls: c.max_scrolls ?? 30,
+    scroll_pause_ms: c.scroll_pause_ms ?? 1200,
+    no_new_content_timeout_ms: c.no_new_content_timeout_ms ?? 3000,
+    pagination_selector: c.pagination_selector || '',
+    pagination_selector_type: c.pagination_selector_type || 'css',
+    max_pages: c.max_pages ?? 10,
+    pagination_wait_ms: c.pagination_wait_ms ?? 1500,
+    item_selector_template: c.item_selector_template || '',
+    item_selector_type: c.item_selector_type || 'css',
+    item_url_template: c.item_url_template || '',
+    target_item_ids: c.target_item_ids_json || [],
+    target_url: c.target_url || '',
+  };
+  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+  if (res && res.ok) await loadConfigs();
+}
+
+document.getElementById('btn-import-config').addEventListener('click', () => {
+  document.getElementById('import-file-input').click();
+});
+document.getElementById('import-file-input').addEventListener('change', async (e) => {
+  const file = e.target.files[0];
+  if (!file) return;
+  e.target.value = '';
+  let data;
+  try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; }
+  const body = {
+    name: data.name || file.name.replace(/\.json$/, ''),
+    is_active: false,
+    search_texts: data.search_texts || [],
+    search_box_selector: data.search_box_selector || '',
+    search_box_selector_type: data.search_box_selector_type || 'css',
+    search_submit_selector: data.search_submit_selector || '',
+    search_submit_selector_type: data.search_submit_selector_type || 'css',
+    search_results_selector: data.search_results_selector || '',
+    search_results_selector_type: data.search_results_selector_type || 'css',
+    scroll_container_selector: data.scroll_container_selector || '',
+    scroll_container_selector_type: data.scroll_container_selector_type || 'css',
+    max_scrolls: data.max_scrolls ?? 30,
+    scroll_pause_ms: data.scroll_pause_ms ?? 1200,
+    no_new_content_timeout_ms: data.no_new_content_timeout_ms ?? 3000,
+    pagination_selector: data.pagination_selector || '',
+    pagination_selector_type: data.pagination_selector_type || 'css',
+    max_pages: data.max_pages ?? 10,
+    pagination_wait_ms: data.pagination_wait_ms ?? 1500,
+    item_selector_template: data.item_selector_template || '',
+    item_selector_type: data.item_selector_type || 'css',
+    item_url_template: data.item_url_template || '',
+    target_item_ids: data.target_item_ids || [],
+    target_url: data.target_url || '',
+  };
+  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+  if (res && res.ok) await loadConfigs();
+});
+
 /* ─── Form ─── */
-function openNewForm() {
+function _resetAdvanced() {
-  document.getElementById('form-title').textContent = 'New Flow Config';
-  document.getElementById('cfg-id').value = '';
-  document.getElementById('cfg-name').value = '';
   document.getElementById('cfg-active').checked = false;
-  document.getElementById('cfg-search-texts').value = '';
   document.getElementById('cfg-search-sel').value = '';
   document.getElementById('cfg-search-sel-type').value = 'css';
   document.getElementById('cfg-submit-sel').value = '';
   document.getElementById('cfg-submit-sel-type').value = 'css';
   document.getElementById('cfg-results-sel').value = '';
+  document.getElementById('cfg-results-sel-type').value = 'css';
   document.getElementById('cfg-scroll-container').value = '';
+  document.getElementById('cfg-scroll-container-type').value = 'css';
   document.getElementById('cfg-max-scrolls').value = '30';
   document.getElementById('cfg-scroll-pause').value = '1200';
   document.getElementById('cfg-no-content-timeout').value = '3000';
+  document.getElementById('cfg-pagination-sel').value = '';
+  document.getElementById('cfg-pagination-sel-type').value = 'css';
+  document.getElementById('cfg-max-pages').value = '10';
+  document.getElementById('cfg-pagination-wait').value = '1500';
   document.getElementById('cfg-item-sel').value = '';
   document.getElementById('cfg-item-sel-type').value = 'css';
+  document.getElementById('alink-hint').style.display = 'none';
+  document.getElementById('cfg-item-url-tpl').value = '';
+}
+
+function openNewForm() {
+  document.getElementById('form-title').textContent = 'New Flow Config';
+  document.getElementById('cfg-id').value = '';
+  document.getElementById('cfg-name').value = '';
+  document.getElementById('cfg-target-url').value = '';
+  document.getElementById('cfg-search-texts').value = '';
   document.getElementById('cfg-item-ids').value = '';
+  _resetAdvanced();
+  document.getElementById('adv-details').removeAttribute('open');
   document.getElementById('config-overlay').classList.remove('hidden');
 }
 
@@ -477,20 +812,31 @@ function openEditForm(id) {
   document.getElementById('form-title').textContent = 'Edit Flow Config';
   document.getElementById('cfg-id').value = id;
   document.getElementById('cfg-name').value = c.name || '';
-  document.getElementById('cfg-active').checked = !!c.is_active;
+  document.getElementById('cfg-target-url').value = c.target_url || '';
   document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
+  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
+  document.getElementById('cfg-active').checked = !!c.is_active;
   document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
   document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
   document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
   document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
   document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
+  document.getElementById('cfg-results-sel-type').value = c.search_results_selector_type || 'css';
   document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
+  document.getElementById('cfg-scroll-container-type').value = c.scroll_container_selector_type || 'css';
   document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
   document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
   document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
+  document.getElementById('cfg-pagination-sel').value = c.pagination_selector || '';
+  document.getElementById('cfg-pagination-sel-type').value = c.pagination_selector_type || 'css';
+  document.getElementById('cfg-max-pages').value = c.max_pages ?? 10;
+  document.getElementById('cfg-pagination-wait').value = c.pagination_wait_ms ?? 1500;
   document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
-  document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
+  const _ist = c.item_selector_type || 'css';
-  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
+  document.getElementById('cfg-item-sel-type').value = _ist;
+  document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
+  document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
+  document.getElementById('adv-details').setAttribute('open', '');
   document.getElementById('config-overlay').classList.remove('hidden');
 }
 
@@ -508,19 +854,30 @@ function collectForm() {
     search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
     search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
     search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
+    search_results_selector_type: document.getElementById('cfg-results-sel-type').value,
     scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
+    scroll_container_selector_type: document.getElementById('cfg-scroll-container-type').value,
     max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
     scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
     no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
+    pagination_selector: document.getElementById('cfg-pagination-sel').value.trim(),
+    pagination_selector_type: document.getElementById('cfg-pagination-sel-type').value,
+    max_pages: parseInt(document.getElementById('cfg-max-pages').value) || 10,
+    pagination_wait_ms: parseInt(document.getElementById('cfg-pagination-wait').value) || 1500,
     item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
     item_selector_type: document.getElementById('cfg-item-sel-type').value,
+    item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
     target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
+    target_url: document.getElementById('cfg-target-url').value.trim(),
   };
 }
 
 document.getElementById('btn-new-config').addEventListener('click', openNewForm);
 document.getElementById('form-cancel').addEventListener('click', closeForm);
 document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
+document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
+  document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
+});
 
 document.getElementById('form-save').addEventListener('click', async () => {
   const body = collectForm();
@@ -540,8 +897,9 @@ document.getElementById('form-save').addEventListener('click', async () => {
     closeForm();
     await loadConfigs();
   } else {
-    alert('Failed to save config. Check console.');
+    const detail = res ? await res.text().catch(() => `HTTP ${res.status}`) : 'No response (network error)';
-    console.error(await res?.text());
+    console.error('Save failed:', detail);
+    alert(`Save failed (${res?.status ?? '?'}):\n${detail}\n\nIf columns are missing, restart the admin panel to apply DB migrations.`);
   }
 });
 
@@ -553,6 +911,162 @@ function fmtTime(iso) {
   if (!iso) return '–';
   return new Date(iso).toLocaleString();
 }
+
+/* ─────────────────────────── Batch Run ─────────────────────── */
+let _batchPollTimer = null;
+
+const _STATUS_COLOR = {
+  running: 'var(--accent)',
+  ok:      'var(--green)',
+  failed:  'var(--red)',
+  error:   'var(--red)',
+  skipped: 'var(--muted)',
+  pending: 'var(--muted)',
+};
+
+function fmtDuration(ms) {
+  if (ms == null) return '–';
+  if (ms < 1000) return ms + ' ms';
+  return (ms / 1000).toFixed(1) + ' s';
+}
+
+function fmtTs(ts) {
+  if (!ts) return '–';
+  return new Date(ts * 1000).toLocaleTimeString();
+}
+
+function renderBatchStatus(s) {
+  const idle     = document.getElementById('batch-idle-msg');
+  const body     = document.getElementById('batch-status-body');
+  const errEl    = document.getElementById('batch-error');
+  const startBtn = document.getElementById('batch-start-btn');
+  const stopBtn  = document.getElementById('batch-stop-btn');
+  const badge    = document.getElementById('bs-active-badge');
+
+  if (!s.batch_id) {
+    idle.style.display = '';
+    body.style.display = 'none';
+    startBtn.style.display = '';
+    stopBtn.style.display = 'none';
+    badge.style.display = 'none';
+    return;
+  }
+
+  idle.style.display = 'none';
+  body.style.display = '';
+
+  const pct = s.total ? Math.round(s.completed / s.total * 100) : 0;
+  document.getElementById('batch-progress-bar').style.width = pct + '%';
+  document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`;
+  document.getElementById('bs-completed').textContent = s.completed ?? 0;
+  document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0;
+  document.getElementById('bs-failed').textContent = s.failed ?? 0;
+  document.getElementById('bs-workers').textContent = s.configured_workers ?? '–';
+  document.getElementById('bs-started').textContent = fmtTs(s.started_at);
+
+  const finRow = document.getElementById('bs-finished-row');
+  if (s.finished_at) {
+    finRow.style.display = '';
+    document.getElementById('bs-finished').textContent = fmtTs(s.finished_at);
+  } else {
+    finRow.style.display = 'none';
+  }
+
+  // Active worker badge
+  const active = s.active_workers ?? 0;
+  if (s.running && active > 0) {
+    badge.textContent = `${active} / ${s.configured_workers} active`;
+    badge.style.display = '';
+  } else {
+    badge.style.display = 'none';
+  }
+
+  // Per-worker table
+  const tbody = document.getElementById('bs-worker-tbody');
+  const workers = s.workers_detail || [];
+  tbody.innerHTML = workers.map(w => {
+    const color = _STATUS_COLOR[w.status] || 'var(--muted)';
+    const label = w.status === 'running' ? '⟳ running' : w.status;
+    const err = w.error && w.status !== 'skipped' ? `<span style="color:var(--red)">${esc(w.error)}</span>` : '–';
+    return `<tr>
+      <td style="font-family:monospace">${w.slot}</td>
+      <td style="color:${color};font-weight:600">${label}</td>
+      <td>${fmtDuration(w.duration_ms)}</td>
+      <td style="max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td>
+    </tr>`;
+  }).join('');
+
+  if (s.error) {
+    errEl.textContent = 'Error: ' + s.error;
+    errEl.style.display = '';
+  } else {
+    errEl.style.display = 'none';
+  }
+
+  if (s.running) {
+    startBtn.style.display = 'none';
+    stopBtn.style.display = '';
+    stopBtn.disabled = s.stopping || false;
+    stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop';
+  } else {
+    startBtn.style.display = '';
+    stopBtn.style.display = 'none';
+  }
+}
+
+async function pollBatchStatus() {
+  const res = await apiFetch('/api/batch-status');
+  if (!res || !res.ok) return;
+  const s = await res.json();
+  renderBatchStatus(s);
+  if (s.running && !_batchPollTimer) startBatchPoll();
+}
+
+function startBatchPoll() {
+  if (_batchPollTimer) return;
+  _batchPollTimer = setInterval(async () => {
+    const r = await apiFetch('/api/batch-status');
+    if (!r || !r.ok) return;
+    const st = await r.json();
+    renderBatchStatus(st);
+    if (!st.running) {
+      clearInterval(_batchPollTimer);
+      _batchPollTimer = null;
+    }
+  }, 1500);
+}
+
+document.getElementById('batch-start-btn').addEventListener('click', async () => {
+  const workers  = parseInt(document.getElementById('batch-workers').value) || 3;
+  const runs     = parseInt(document.getElementById('batch-runs').value) || 10;
+  const stagger  = parseInt(document.getElementById('batch-stagger').value) || 0;
+  const headless = document.getElementById('batch-headless').checked;
+
+  const res = await apiFetch('/api/run-batch', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ workers, total_runs: runs, stagger_ms: stagger, headless }),
+  });
+
+  if (!res) return;
+  if (res.status === 409) { alert('A batch is already running.'); return; }
+  if (res.status === 422) { const d = await res.json(); alert(d.detail || 'No active flow config.'); return; }
+  if (!res.ok) { alert('Failed to start batch.'); return; }
+
+  await pollBatchStatus();
+  startBatchPoll();
+});
+
+document.getElementById('batch-stop-btn').addEventListener('click', async () => {
+  const btn = document.getElementById('batch-stop-btn');
+  btn.disabled = true;
+  btn.textContent = '■ Stopping…';
+  const res = await apiFetch('/api/stop-batch', { method: 'POST' });
+  if (!res || !res.ok) {
+    btn.disabled = false;
+    btn.textContent = '■ Stop';
+  }
+});
 </script>
 </body>
 </html>

config.py

@@ -9,7 +9,6 @@ load_dotenv()
 
 @dataclass
 class Config:
-    target_url: str = os.getenv("TARGET_URL", "https://example.com")
     admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
     admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
     admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
@@ -17,6 +16,7 @@ class Config:
     db_path: str = os.getenv("DB_PATH", "data/tracker.db")
     headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
     chrome_binary: str | None = os.getenv("CHROME_BINARY")
+    chromedriver_path: str = os.getenv("CHROMEDRIVER_PATH", "drivers/chromedriver")
 
     # Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
     phone_numbers: list[str] = field(default_factory=list)

crawler/batch.py

@@ -0,0 +1,117 @@
+"""Batch runner — spawn N parallel Chrome workers to execute the active flow config."""
+from __future__ import annotations
+
+import asyncio
+import threading
+import time
+import uuid
+from concurrent.futures import Future, ThreadPoolExecutor, as_completed
+from typing import Any, Callable
+
+from logger import get_logger
+
+log = get_logger(__name__)
+
+
+def _single_run(
+    run_index: int,
+    batch_id: str,
+    cfg: dict[str, Any],
+    headless: bool,
+) -> tuple[bool, str | None]:
+    """One isolated Chrome session. Returns (success, failure_reason)."""
+    from crawler.driver import driver_session
+    from crawler.dynamic_flow import DynamicFlow
+    from crawler.scenarios.fresh_session import FreshSessionScenario
+    from db.repositories.runs import runs_repo
+
+    identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
+    t0 = time.monotonic()
+    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+    try:
+        target_url: str = cfg.get("target_url") or ""
+        with driver_session(headless=headless) as driver:
+            FreshSessionScenario(driver, target_url).run()
+            result = DynamicFlow(driver, cfg).run() if cfg else None
+
+        total_ms = int((time.monotonic() - t0) * 1000)
+        if result:
+            steps_data = [
+                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
+                for s in result.steps
+            ]
+            asyncio.run(runs_repo.insert(
+                "batch", identifier, result.success,
+                result.failure_reason, steps_data, total_ms,
+            ))
+            log.info("[batch:%s] Run %d done — success=%s (%d ms)", batch_id[:8], run_index, result.success, total_ms)
+            return result.success, result.failure_reason
+        else:
+            asyncio.run(runs_repo.insert("batch", identifier, True, None, [], total_ms))
+            log.info("[batch:%s] Run %d done (no flow config) — %d ms", batch_id[:8], run_index, total_ms)
+            return True, None
+
+    except Exception as exc:
+        total_ms = int((time.monotonic() - t0) * 1000)
+        asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
+        log.error("[batch:%s] Run %d failed — %s (%d ms)", batch_id[:8], run_index, exc, total_ms)
+        return False, str(exc)
+
+
+def run_batch(
+    cfg: dict[str, Any],
+    workers: int,
+    total_runs: int,
+    headless: bool = True,
+    stagger_ms: int = 0,
+    on_progress: Callable[[int, int, int], None] | None = None,
+) -> dict[str, Any]:
+    """
+    Run `total_runs` crawler sessions with up to `workers` parallel Chrome processes.
+
+    `stagger_ms` delays each successive submission to the pool, spreading
+    Chrome startup load and reducing bot-detection fingerprint clustering.
+
+    `on_progress(completed, succeeded, failed)` is called after every finished run.
+
+    Returns {"batch_id", "total", "succeeded", "failed"}.
+    """
+    batch_id = uuid.uuid4().hex
+    log.info(
+        "Batch %s — %d run(s), %d worker(s), stagger=%d ms, headless=%s",
+        batch_id[:8], total_runs, workers, stagger_ms, headless,
+    )
+
+    succeeded = 0
+    failed = 0
+    lock = threading.Lock()
+
+    with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=f"batch-{batch_id[:8]}") as pool:
+        futures: list[Future[tuple[bool, str | None]]] = []
+        for i in range(total_runs):
+            futures.append(pool.submit(_single_run, i, batch_id, cfg, headless))
+            if stagger_ms > 0 and i < total_runs - 1:
+                time.sleep(stagger_ms / 1000)
+
+        for fut in as_completed(futures):
+            try:
+                success, _ = fut.result()
+            except Exception as exc:
+                log.error("[batch:%s] Unhandled thread exception: %s", batch_id[:8], exc)
+                success = False
+
+            with lock:
+                if success:
+                    succeeded += 1
+                else:
+                    failed += 1
+                completed = succeeded + failed
+
+            if on_progress:
+                on_progress(completed, succeeded, failed)
+
+    log.info(
+        "Batch %s finished — %d/%d succeeded, %d failed",
+        batch_id[:8], succeeded, total_runs, failed,
+    )
+    return {"batch_id": batch_id, "total": total_runs, "succeeded": succeeded, "failed": failed}

crawler/driver.py

@@ -1,9 +1,14 @@
 """Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
 from __future__ import annotations
 
+import os
 import random
+import shutil
+import subprocess
+import sys
 import time
 from contextlib import contextmanager
+from pathlib import Path
 from typing import Generator
 
 import undetected_chromedriver as uc
@@ -11,6 +16,16 @@ from selenium.webdriver.chrome.options import Options
 from selenium_stealth import stealth
 
 from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
+
+# Selenium talks to ChromeDriver over localhost — must not go through any system proxy.
+for _var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
+    os.environ.pop(_var, None)
+_no_proxy = "localhost,127.0.0.1,::1"
+os.environ.setdefault("no_proxy", _no_proxy)
+os.environ.setdefault("NO_PROXY", _no_proxy)
 
 _USER_AGENTS = [
     "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@@ -18,31 +33,79 @@ _USER_AGENTS = [
     "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
 ]
 
+_CHROME_CANDIDATES = [
+    "google-chrome-stable",
+    "google-chrome",
+    "chromium",
+    "chromium-browser",
+]
 
-def _build_options(user_agent: str, headless: bool) -> Options:
+
+def _detect_chrome() -> tuple[str, int]:
+    """Return (binary_path, major_version) for the first Chrome found."""
+    candidates = list(_CHROME_CANDIDATES)
+    if config.chrome_binary:
+        candidates.insert(0, config.chrome_binary)
+
+    for binary in candidates:
+        resolved = shutil.which(binary) or (binary if Path(binary).exists() else None)
+        if not resolved:
+            continue
+        try:
+            out = subprocess.check_output(
+                [resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
+            ).strip()
+            major = int(out.split()[-1].split(".")[0])
+            log.debug("Detected Chrome %d via '%s'", major, resolved)
+            return resolved, major
+        except Exception:
+            continue
+
+    log.critical("No Chrome binary found. Install Chrome or set CHROME_BINARY in .env")
+    sys.exit(1)
+
+
+def _local_driver_path() -> str | None:
+    path = Path(config.chromedriver_path)
+    if path.exists():
+        return str(path)
+    log.warning(
+        "No patched ChromeDriver at '%s'. Run 'make patch-driver' once. Falling back to auto-download.",
+        path,
+    )
+    return None
+
+
+def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
     opts = Options()
+    opts.binary_location = chrome_binary
     if headless:
         opts.add_argument("--headless=new")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
     opts.add_argument("--disable-dev-shm-usage")
+    opts.add_argument("--disable-gpu")
     opts.add_argument("--disable-blink-features=AutomationControlled")
     opts.add_argument("--disable-infobars")
     opts.add_argument("--window-size=1920,1080")
-    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
-    opts.add_experimental_option("useAutomationExtension", False)
-    if config.chrome_binary:
-        opts.binary_location = config.chrome_binary
     return opts
 
 
 def make_driver(headless: bool | None = None) -> uc.Chrome:
     """Return a stealthed undetected Chrome instance."""
     use_headless = config.headless if headless is None else headless
+    chrome_binary, major = _detect_chrome()
     ua = random.choice(_USER_AGENTS)
-    opts = _build_options(ua, use_headless)
+    opts = _build_options(ua, use_headless, chrome_binary)
 
-    driver = uc.Chrome(options=opts, use_subprocess=True)
+    log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
+
+    driver = uc.Chrome(
+        options=opts,
+        driver_executable_path=_local_driver_path(),
+        version_main=major,
+        use_subprocess=True,
+    )
 
     stealth(
         driver,
@@ -54,7 +117,6 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
         fix_hairline=True,
     )
 
-    # Mask navigator.webdriver via CDP
     driver.execute_cdp_cmd(
         "Page.addScriptToEvaluateOnNewDocument",
         {
@@ -65,19 +127,19 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
             """
         },
     )
+    log.info("Driver ready (Chrome %d)", major)
     return driver
 
 
 @contextmanager
 def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
-    """Context manager that guarantees driver.quit() on exit."""
     driver = make_driver(headless)
     try:
         yield driver
     finally:
         driver.quit()
+        log.debug("Driver session closed")
 
 
 def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
-    """Sleep for a random human-like interval."""
     time.sleep(random.uniform(lo, hi))

crawler/dynamic_flow.py

@@ -4,15 +4,17 @@ Dynamic flow executor — reads the active flow config from DB at runtime and ru
   2. scroll_step  : infinite-scroll until target item is visible or max_scrolls reached
   3. click_step   : click each target item id
 """
+
 from __future__ import annotations
 
-import asyncio
+import math
+import random
 import time
 from dataclasses import dataclass, field
 from typing import Any
 
 import undetected_chromedriver as uc
-from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -21,6 +23,9 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 class DynamicFlowError(Exception):
@@ -71,10 +76,30 @@ def _wait_for(
     selector: str,
     selector_type: str,
     timeout: float = 15.0,
+    label: str = "",
 ) -> WebElement:
-    return WebDriverWait(driver, timeout).until(
+    log.debug(
+        "Waiting for %s [%s:%s] (timeout=%ss)",
+        label or "element",
+        selector_type,
+        selector,
+        timeout,
+    )
+    try:
+        el = WebDriverWait(driver, timeout).until(
             EC.presence_of_element_located((_by(selector_type), selector))
         )
+        log.debug("Found %s [%s:%s]", label or "element", selector_type, selector)
+        return el
+    except Exception as exc:
+        log.warning(
+            "Selector not found — %s [%s:%s]: %s",
+            label or "element",
+            selector_type,
+            selector,
+            exc,
+        )
+        raise
 
 
 class DynamicFlow:
@@ -87,6 +112,7 @@ class DynamicFlow:
             config_id=self.cfg["id"],
             config_name=self.cfg["name"],
         )
+        log.info("Starting flow '%s' (id=%s)", self.cfg["name"], self.cfg["id"])
 
         search_texts: list[str] = self.cfg.get("search_texts_json") or []
         item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
@@ -95,6 +121,7 @@ class DynamicFlow:
             step = self._run_search(text)
             result.steps.append(step)
             if not step.success:
+                log.error("Search step failed for '%s' — aborting flow", text)
                 return result
             human_delay(0.8, 1.5)
 
@@ -102,12 +129,21 @@ class DynamicFlow:
                 scroll_step = self._run_scroll_and_find(item_id)
                 result.steps.append(scroll_step)
                 if not scroll_step.success:
-                    continue  # try next item_id, don't abort the whole flow
+                    log.warning(
+                        "Scroll/find failed for item '%s' — skipping click", item_id
+                    )
+                    continue
 
                 click_step = self._run_click(item_id, scroll_step.data.get("element"))
                 result.steps.append(click_step)
                 human_delay(1.0, 2.5)
 
+        log.info(
+            "Flow '%s' finished — success=%s, steps=%d",
+            self.cfg["name"],
+            result.success,
+            len(result.steps),
+        )
         return result
 
     # ------------------------------------------------------------------
@@ -116,44 +152,70 @@ class DynamicFlow:
     def _run_search(self, text: str) -> StepResult:
         t0 = time.monotonic()
         name = f"search:{text}"
+        log.info("[%s] Searching for '%s'", name, text)
         try:
             sel = self.cfg["search_box_selector"]
             sel_type = self.cfg["search_box_selector_type"]
             if not sel:
                 raise DynamicFlowError("search_box_selector is not configured")
 
-            box = _wait_for(self.driver, sel, sel_type)
+            box = _wait_for(self.driver, sel, sel_type, label="search box")
             box.clear()
             human_delay(0.3, 0.6)
 
-            # Type character by character for a human feel
             for ch in text:
                 box.send_keys(ch)
                 time.sleep(0.04)
+            log.debug("[%s] Typed search text", name)
 
             human_delay(0.4, 0.8)
 
-            # Submit: explicit button or Enter
             submit_sel = self.cfg.get("search_submit_selector", "")
             if submit_sel:
                 submit_type = self.cfg.get("search_submit_selector_type", "css")
-                btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
+                log.debug(
+                    "[%s] Clicking submit button [%s:%s]", name, submit_type, submit_sel
+                )
+                btn = _wait_for(
+                    self.driver,
+                    submit_sel,
+                    submit_type,
+                    timeout=5.0,
+                    label="submit button",
+                )
                 btn.click()
             else:
+                log.debug("[%s] Submitting via Enter key", name)
                 box.send_keys(Keys.RETURN)
 
-            # Wait for results container if configured
             results_sel = self.cfg.get("search_results_selector", "")
+            results_sel_type = self.cfg.get("search_results_selector_type", "css")
             if results_sel:
-                _wait_for(self.driver, results_sel, "css", timeout=15.0)
+                log.debug(
+                    "[%s] Waiting for results container [%s:%s]",
+                    name,
+                    results_sel_type,
+                    results_sel,
+                )
+                _wait_for(
+                    self.driver,
+                    results_sel,
+                    results_sel_type,
+                    timeout=15.0,
+                    label="results container",
+                )
 
             human_delay(1.0, 2.0)
-            return StepResult(name=name, success=True, data={"text": text},
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.info("[%s] Search OK — %d ms", name, ms)
+            return StepResult(
+                name=name, success=True, data={"text": text}, duration_ms=ms
+            )
 
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
 
     # ------------------------------------------------------------------
     # Step: scroll (infinite-scroll) until item selector matches
@@ -161,39 +223,131 @@ class DynamicFlow:
     def _run_scroll_and_find(self, item_id: str) -> StepResult:
         t0 = time.monotonic()
         name = f"scroll_find:{item_id}"
+        log.info("[%s] Scrolling to find item '%s'", name, item_id)
         try:
-            item_sel = self._build_item_selector(item_id)
+            item_sel, item_sel_type = self._build_item_selector(item_id)
-            item_sel_type = self.cfg.get("item_selector_type", "css")
             scroll_container = self.cfg.get("scroll_container_selector", "").strip()
+            scroll_container_type = self.cfg.get(
+                "scroll_container_selector_type", "css"
+            )
             max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
             pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
             no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
+            pagination_sel: str = self.cfg.get("pagination_selector", "").strip()
+            pagination_sel_type: str = self.cfg.get("pagination_selector_type", "css")
+            max_pages: int = int(self.cfg.get("max_pages", 10))
+            pagination_wait_ms: int = int(self.cfg.get("pagination_wait_ms", 1500))
+
+            log.debug(
+                "[%s] Selector [%s:%s], container=[%s:'%s'], max_scrolls=%d, pagination=%s",
+                name,
+                item_sel_type,
+                item_sel,
+                scroll_container_type,
+                scroll_container or "window",
+                max_scrolls,
+                (
+                    f"[{pagination_sel_type}:{pagination_sel}] max_pages={max_pages}"
+                    if pagination_sel
+                    else "off"
+                ),
+            )
 
-            # Check if already visible before scrolling
             found = self._find_item_visible(item_sel, item_sel_type)
             if found:
-                return StepResult(name=name, success=True,
+                log.info("[%s] Item visible immediately (no scroll needed)", name)
-                                  data={"item_id": item_id, "scrolls": 0, "element": found},
+                return StepResult(
-                                  duration_ms=int((time.monotonic() - t0) * 1000))
+                    name=name,
+                    success=True,
+                    data={
+                        "item_id": item_id,
+                        "scrolls": 0,
+                        "pages": 0,
+                        "element": found,
+                    },
+                    duration_ms=int((time.monotonic() - t0) * 1000),
+                )
 
-            last_height = self._get_scroll_height(scroll_container)
+            last_height = self._get_scroll_height(
+                scroll_container, scroll_container_type
+            )
             stale_since: float | None = None
+            total_scrolls = 0
+            page_num = 0
+            scroll_on_page = 0
 
-            for scroll_num in range(1, max_scrolls + 1):
+            while scroll_on_page < max_scrolls:
-                self._scroll_down(scroll_container)
+                # Check for the item BEFORE scrolling so items near the top of a
+                # freshly-paginated page are seen before we jump to the bottom.
+                found = self._find_item_visible(item_sel, item_sel_type)
+                if found:
+                    ms = int((time.monotonic() - t0) * 1000)
+                    log.info(
+                        "[%s] Found after %d scroll(s) on page %d — %d ms",
+                        name,
+                        total_scrolls,
+                        page_num + 1,
+                        ms,
+                    )
+                    return StepResult(
+                        name=name,
+                        success=True,
+                        data={
+                            "item_id": item_id,
+                            "scrolls": total_scrolls,
+                            "pages": page_num,
+                            "element": found,
+                        },
+                        duration_ms=ms,
+                    )
+
+                scroll_on_page += 1
+                total_scrolls += 1
+                self._scroll_down(scroll_container, scroll_container_type)
                 time.sleep(pause_ms / 1000)
 
-                found = self._find_item_visible(item_sel, item_sel_type)
+                new_height = self._get_scroll_height(
-                if found:
+                    scroll_container, scroll_container_type
-                    return StepResult(name=name, success=True,
+                )
-                                      data={"item_id": item_id, "scrolls": scroll_num, "element": found},
-                                      duration_ms=int((time.monotonic() - t0) * 1000))
-
-                new_height = self._get_scroll_height(scroll_container)
                 if new_height == last_height:
                     if stale_since is None:
                         stale_since = time.monotonic()
+                        log.debug(
+                            "[%s] Page height unchanged at scroll %d — starting stale timer",
+                            name,
+                            scroll_on_page,
+                        )
                     elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
+                        # Infinite scroll exhausted — try pagination if configured
+                        if pagination_sel and page_num < max_pages:
+                            if self._try_paginate(
+                                name,
+                                pagination_sel,
+                                pagination_sel_type,
+                                pagination_wait_ms,
+                            ):
+                                page_num += 1
+                                scroll_on_page = 0
+                                stale_since = None
+                                last_height = self._get_scroll_height(
+                                    scroll_container, scroll_container_type
+                                )
+                                log.info(
+                                    "[%s] Advanced to pagination page %d",
+                                    name,
+                                    page_num + 1,
+                                )
+                                # Loop continues — item check happens at top of next iteration
+                            else:
+                                raise DynamicFlowError(
+                                    f"Pagination button [{pagination_sel_type}:{pagination_sel}] "
+                                    f"not visible on page {page_num + 1}"
+                                )
+                        elif pagination_sel and page_num >= max_pages:
+                            raise DynamicFlowError(
+                                f"Item '{item_id}' not found after {page_num} pagination page(s)"
+                            )
+                        else:
                             raise DynamicFlowError(
                                 f"No new content after {no_new_ms} ms — reached end of page "
                                 f"without finding item '{item_id}'"
@@ -201,16 +355,138 @@ class DynamicFlow:
                 else:
                     stale_since = None
                     last_height = new_height
-
+                    log.debug(
-            raise DynamicFlowError(
+                        "[%s] Scroll %d (page %d) — new height %d px",
-                f"Item '{item_id}' not found after {max_scrolls} scrolls"
+                        name,
+                        scroll_on_page,
+                        page_num + 1,
+                        new_height,
                     )
 
-        except DynamicFlowError:
+            raise DynamicFlowError(
-            raise
+                f"Item '{item_id}' not found after {total_scrolls} scrolls"
+            )
+
+        except DynamicFlowError as exc:
+            ms = int((time.monotonic() - t0) * 1000)
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Unexpected error — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
+
+    # ------------------------------------------------------------------
+    # Human mouse simulation
+    # ------------------------------------------------------------------
+    def _human_click(self, el: WebElement, name: str = "") -> None:
+        """
+        Simulate a human-like mouse approach and click:
+          1. Smooth scroll element into viewport
+          2. Dispatch JS mousemove events along a curved path
+          3. Trigger mouseenter / mouseover on the element
+          4. Click via ActionChains with random sub-element offset
+          5. Fallback: synthetic JS mouse event sequence
+        """
+        # Step 1 — bring element into view smoothly
+        self.driver.execute_script(
+            "arguments[0].scrollIntoView({block:'center', behavior:'smooth'});", el
+        )
+        time.sleep(random.uniform(0.35, 0.7))
+
+        # Step 2 — get element centre in viewport coordinates
+        rect: dict[str, float] = self.driver.execute_script(
+            """
+            const r = arguments[0].getBoundingClientRect();
+            return {x: r.left, y: r.top, w: r.width, h: r.height,
+                    vw: window.innerWidth, vh: window.innerHeight};
+            """,
+            el,
+        )
+        cx = rect["x"] + rect["w"] / 2
+        cy = rect["y"] + rect["h"] / 2
+        vw = rect["vw"]
+        vh = rect["vh"]
+
+        # Step 3 — build a curved path from a random off-element start
+        sx = random.uniform(vw * 0.1, vw * 0.5)
+        sy = random.uniform(vh * 0.1, vh * 0.5)
+        # Control point for quadratic Bézier — pulled perpendicular to the straight line
+        dx, dy = cx - sx, cy - sy
+        perp_x = -dy * random.uniform(0.15, 0.4)
+        perp_y = dx * random.uniform(0.15, 0.4)
+        qx = sx + dx / 2 + perp_x
+        qy = sy + dy / 2 + perp_y
+
+        steps = random.randint(10, 18)
+        prev_mx, prev_my = sx, sy
+        for i in range(1, steps + 1):
+            t = i / steps
+            # Quadratic Bézier point
+            bx = (1 - t) ** 2 * sx + 2 * (1 - t) * t * qx + t**2 * cx
+            by = (1 - t) ** 2 * sy + 2 * (1 - t) * t * qy + t**2 * cy
+            # Add small Gaussian jitter that fades near the target
+            jitter = max(0.0, 1.0 - t) * 4.0
+            mx = bx + random.gauss(0, jitter)
+            my = by + random.gauss(0, jitter)
+
+            self.driver.execute_script(
+                """
+                document.dispatchEvent(new MouseEvent('mousemove', {
+                    bubbles: true, cancelable: true,
+                    clientX: arguments[0], clientY: arguments[1]
+                }));
+                """,
+                int(mx),
+                int(my),
+            )
+            # Variable inter-step pause — faster in the middle, slower near target
+            speed = 0.5 + 0.5 * math.sin(math.pi * t)
+            time.sleep(random.uniform(0.008, 0.025) / max(speed, 0.3))
+            prev_mx, prev_my = mx, my
+
+        # Step 4 — hover events on the element itself
+        self.driver.execute_script(
+            """
+            arguments[0].dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
+            arguments[0].dispatchEvent(new MouseEvent('mouseover',  {bubbles: true}));
+            """,
+            el,
+        )
+        time.sleep(random.uniform(0.1, 0.28))
+
+        # Step 5 — ActionChains click with random offset from centre
+        off_x = int(random.uniform(-rect["w"] * 0.28, rect["w"] * 0.28))
+        off_y = int(random.uniform(-rect["h"] * 0.28, rect["h"] * 0.28))
+        log.debug(
+            "[%s] Human click — target=(%.0f,%.0f) offset=(%d,%d)",
+            name,
+            cx,
+            cy,
+            off_x,
+            off_y,
+        )
+        try:
+            ActionChains(self.driver).move_to_element_with_offset(
+                el, off_x, off_y
+            ).pause(random.uniform(0.05, 0.14)).click().perform()
+            log.debug("[%s] ActionChains click OK", name)
+        except Exception as ac_exc:
+            log.warning(
+                "[%s] ActionChains failed (%s) — synthetic JS click", name, ac_exc
+            )
+            self.driver.execute_script(
+                """
+                const opts = {bubbles: true, cancelable: true,
+                              clientX: arguments[1], clientY: arguments[2]};
+                arguments[0].dispatchEvent(new MouseEvent('mousedown', opts));
+                arguments[0].dispatchEvent(new MouseEvent('mouseup',   opts));
+                arguments[0].dispatchEvent(new MouseEvent('click',     opts));
+                """,
+                el,
+                int(cx + off_x),
+                int(cy + off_y),
+            )
 
     # ------------------------------------------------------------------
     # Step: click the found element
@@ -218,65 +494,205 @@ class DynamicFlow:
     def _run_click(self, item_id: str, element: Any) -> StepResult:
         t0 = time.monotonic()
         name = f"click:{item_id}"
+        log.info("[%s] Clicking item '%s'", name, item_id)
         try:
             if element is None:
                 raise DynamicFlowError("No element reference from scroll step")
 
-            el: WebElement = element
+            self._human_click(element, name)
-            # Scroll element into view and click
-            self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
-            human_delay(0.3, 0.7)
-            try:
-                el.click()
-            except Exception:
-                # Fallback: JS click
-                self.driver.execute_script("arguments[0].click();", el)
 
             human_delay(0.8, 1.5)
-            return StepResult(name=name, success=True,
+            ms = int((time.monotonic() - t0) * 1000)
+            log.info(
+                "[%s] Click OK — landed on %s — %d ms",
+                name,
+                self.driver.current_url,
+                ms,
+            )
+            return StepResult(
+                name=name,
+                success=True,
                 data={"item_id": item_id, "url_after": self.driver.current_url},
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+                duration_ms=ms,
+            )
 
         except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Failed — %s", name, exc)
+            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
+
+    # ------------------------------------------------------------------
+    # Pagination helper
+    # ------------------------------------------------------------------
+    def _try_paginate(
+        self, step_name: str, sel: str, sel_type: str, wait_ms: int
+    ) -> bool:
+        """
+        Find the next-page button and click it reliably.
+
+        Uses a layered approach rather than full human-mouse simulation because
+        pagination buttons are small and offset-based clicking is fragile:
+          1. Native Selenium click (triggers real browser events)
+          2. JS click() on the element
+          3. Synthetic pointer + mouse + click event dispatch
+        """
+        log.debug("[%s] Looking for pagination button [%s:%s]", step_name, sel_type, sel)
+
+        # Search all matching elements, not just the first visible one,
+        # in case the button is present but momentarily off-screen.
+        try:
+            candidates = _find_all(self.driver, sel, sel_type)
+        except Exception as exc:
+            log.warning("[%s] Pagination selector failed — %s", step_name, exc)
+            return False
+
+        btn: WebElement | None = None
+        for el in candidates:
+            try:
+                if el.is_displayed() and el.is_enabled():
+                    btn = el
+                    break
+            except Exception:
+                continue
+
+        if btn is None:
+            log.debug("[%s] Pagination button not found / not interactable", step_name)
+            return False
+
+        log.debug("[%s] Pagination button found — attempting click", step_name)
+        try:
+            self.driver.execute_script(
+                "arguments[0].scrollIntoView({block:'nearest', behavior:'instant'});", btn
+            )
+            time.sleep(random.uniform(0.15, 0.35))
+
+            # Layer 1: native Selenium click
+            try:
+                btn.click()
+                log.debug("[%s] Pagination: native click OK", step_name)
+            except Exception as e1:
+                log.debug("[%s] Pagination: native click failed (%s) — trying JS", step_name, e1)
+                # Layer 2: JS .click()
+                try:
+                    self.driver.execute_script("arguments[0].click();", btn)
+                    log.debug("[%s] Pagination: JS click OK", step_name)
+                except Exception as e2:
+                    log.debug("[%s] Pagination: JS click failed (%s) — dispatching events", step_name, e2)
+                    # Layer 3: synthetic pointer + mouse + click events
+                    self.driver.execute_script(
+                        """
+                        const el = arguments[0];
+                        const r = el.getBoundingClientRect();
+                        const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
+                        const opts = {bubbles: true, cancelable: true, clientX: cx, clientY: cy};
+                        ['pointerdown','mousedown','pointerup','mouseup','click']
+                            .forEach(t => el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, opts)));
+                        """,
+                        btn,
+                    )
+                    log.debug("[%s] Pagination: synthetic events dispatched", step_name)
+
+            log.info("[%s] Pagination button clicked — waiting %d ms", step_name, wait_ms)
+            time.sleep(wait_ms / 1000)
+            return True
+
+        except Exception as exc:
+            log.warning("[%s] Pagination click failed — %s", step_name, exc)
+            return False
 
     # ------------------------------------------------------------------
     # Helpers
     # ------------------------------------------------------------------
-    def _build_item_selector(self, item_id: str) -> str:
+    def _build_item_selector(self, item_id: str) -> tuple[str, str]:
+        """
+        Return (selector, effective_type).
+
+        Priority:
+          1. item_url_template — renders the URL pattern and uses XPath
+             contains(@href, ...) to find the matching <a> element.
+          2. item_selector_type == "a-link" — same href-contains approach but
+             driven by item_selector_template instead.
+          3. CSS / XPath — item_selector_template used verbatim.
+
+        Href-based searches always use XPath so that arbitrary URL characters
+        (colons, brackets, dots, slashes) never break the selector.
+        """
+        url_tpl: str = self.cfg.get("item_url_template", "").strip()
+        if url_tpl:
+            href = url_tpl.replace("{item_id}", item_id)
+            log.debug("item_url_template resolved to href pattern '%s'", href)
+            return f"//a[contains(@href, '{href}')]", "xpath"
+
+        sel_type: str = self.cfg.get("item_selector_type", "css")
         template: str = self.cfg.get("item_selector_template", "")
+
+        if sel_type == "a-link":
+            href_fragment = (
+                template.replace("{item_id}", item_id) if template else item_id
+            )
+            return f"//a[contains(@href, '{href_fragment}')]", "xpath"
+
         if not template:
             raise DynamicFlowError("item_selector_template is not configured")
-        return template.replace("{item_id}", item_id)
+        return template.replace("{item_id}", item_id), sel_type
 
-    def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
+    def _find_item_visible(
+        self, selector: str, selector_type: str
+    ) -> WebElement | None:
         try:
             els = _find_all(self.driver, selector, selector_type)
+        except Exception as exc:
+            log.debug("Selector error [%s:%s]: %s", selector_type, selector, exc)
+            return None
         for el in els:
+            try:
                 if el.is_displayed():
                     return el
-        except NoSuchElementException:
+            except Exception:
-            pass
+                continue
         return None
 
-    def _get_scroll_height(self, container_sel: str) -> int:
+    def _get_scroll_height(
+        self, container_sel: str, container_type: str = "css"
+    ) -> int:
+        body_h = int(self.driver.execute_script(
+            "return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)"
+        ))
         if container_sel:
             try:
-                el = _find(self.driver, container_sel, "css")
+                el = _find(self.driver, container_sel, container_type)
-                return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
+                container_h = int(self.driver.execute_script("return arguments[0].scrollHeight", el))
-            except Exception:
+                return max(body_h, container_h)
-                pass
+            except Exception as exc:
-        return int(self.driver.execute_script("return document.body.scrollHeight"))
+                log.warning(
-
+                    "Could not read scrollHeight from container [%s:'%s']: %s — using body",
-    def _scroll_down(self, container_sel: str) -> None:
+                    container_type, container_sel, exc,
-        if container_sel:
+                )
-            try:
+        return body_h
-                el = _find(self.driver, container_sel, "css")
+
-                self.driver.execute_script(
+    def _scroll_down(self, container_sel: str, container_type: str = "css") -> None:
-                    "arguments[0].scrollTop = arguments[0].scrollHeight", el
+        if container_sel:
+            try:
+                el = _find(self.driver, container_sel, container_type)
+                self.driver.execute_script(
+                    """
+                    const el = arguments[0];
+                    el.scrollTop = el.scrollHeight;
+                    el.dispatchEvent(new Event('scroll', {bubbles: true}));
+                    """,
+                    el,
+                )
+            except Exception as exc:
+                log.warning(
+                    "Could not scroll container [%s:'%s']: %s — falling through to window",
+                    container_type, container_sel, exc,
+                )
+        # Always advance the window too — most infinite-scroll triggers
+        # listen to window scroll events, not container scroll events.
+        self.driver.execute_script(
+            """
+            window.scrollTo(0, document.body.scrollHeight);
+            window.dispatchEvent(new Event('scroll', {bubbles: true}));
+            document.dispatchEvent(new Event('scroll', {bubbles: true}));
+            """
         )
-                return
-            except Exception:
-                pass
-        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")

crawler/scenarios/fresh_session.py

@@ -2,11 +2,9 @@
 from __future__ import annotations
 
 import undetected_chromedriver as uc
-from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
 
-from config import config
+from crawler.driver import human_delay
-from crawler.driver import human_delay, make_driver
 
 
 class FreshSessionScenario:
@@ -19,8 +17,9 @@ class FreshSessionScenario:
     explicitly delete all cookies after loading to be safe.
     """
 
-    def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
+    def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
         self.driver = driver
+        self.url = url
         self.wait = WebDriverWait(driver, timeout)
 
     def run(self) -> dict[str, object]:
@@ -32,7 +31,7 @@ class FreshSessionScenario:
             "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
         )
 
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
         human_delay(2.0, 4.0)
 
         # Wait for the page to reach a ready state

crawler/scenarios/otp_login.py

@@ -10,7 +10,6 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 
-from config import config
 from crawler.driver import human_delay
 
 
@@ -32,11 +31,13 @@ class OTPLoginScenario:
         driver: uc.Chrome,
         phone: str,
         otp_resolver: Callable[[str], str],
+        url: str,
         timeout: int = 30,
     ) -> None:
         self.driver = driver
         self.phone = phone
         self.otp_resolver = otp_resolver
+        self.url = url
         self.wait = WebDriverWait(driver, timeout)
 
     # ------------------------------------------------------------------
@@ -50,7 +51,7 @@ class OTPLoginScenario:
 
     def run(self) -> dict[str, object]:
         """Execute the OTP login. Returns a result dict."""
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
         human_delay(1.5, 3.0)
 
         try:

crawler/tasks.py

@@ -0,0 +1,214 @@
+"""Huey task definitions and shared batch state for the admin panel."""
+from __future__ import annotations
+
+import threading
+import time
+from concurrent.futures import ThreadPoolExecutor, as_completed
+from typing import Any
+
+from huey import MemoryHuey
+
+huey = MemoryHuey("seed", immediate=False)
+
+# ── Shared state (all accessed under _lock) ───────────────────────────────────
+_lock = threading.Lock()
+
+_current_batch_id: str | None = None
+_current_result: Any = None          # Huey Result — kept for revocation
+
+_batch_meta: dict[str, Any] = {}     # batch_id → aggregate metadata
+_worker_slots: dict[str, dict] = {}  # f"{batch_id}:{slot:04d}" → worker info
+_stop_events: dict[str, threading.Event] = {}
+
+
+# ── Public helpers ─────────────────────────────────────────────────────────────
+
+def get_current_batch_id() -> str | None:
+    with _lock:
+        return _current_batch_id
+
+
+def set_current(batch_id: str, result: Any) -> None:
+    global _current_batch_id, _current_result
+    with _lock:
+        _current_batch_id = batch_id
+        _current_result = result
+
+
+def stop_current() -> bool:
+    """Signal the running batch to stop. Returns True if there was something to stop."""
+    with _lock:
+        bid = _current_batch_id
+        result = _current_result
+
+    if bid is None:
+        return False
+
+    # Revoke task if it hasn't started yet (still pending in Huey queue)
+    if result is not None:
+        try:
+            result.revoke(revoke_once=True)
+        except Exception:
+            pass
+
+    # Signal running workers to stop after their current step
+    ev = _stop_events.get(bid)
+    if ev:
+        ev.set()
+
+    with _lock:
+        if bid in _batch_meta:
+            _batch_meta[bid]["stopping"] = True
+
+    return True
+
+
+def batch_status() -> dict[str, Any]:
+    with _lock:
+        bid = _current_batch_id
+        if bid is None:
+            return {"batch_id": None, "running": False}
+
+        meta = dict(_batch_meta.get(bid, {}))
+        prefix = f"{bid}:"
+        workers = [dict(v) for k, v in _worker_slots.items() if k.startswith(prefix)]
+
+    meta["batch_id"] = bid
+    meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0))
+    meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running")
+    return meta
+
+
+# ── Huey task ─────────────────────────────────────────────────────────────────
+
+@huey.task(name="run_batch")
+def run_batch_task(
+    batch_id: str,
+    cfg: dict[str, Any],
+    workers: int,
+    total_runs: int,
+    headless: bool,
+    stagger_ms: int,
+) -> dict[str, Any]:
+    import asyncio
+
+    from crawler.driver import driver_session
+    from crawler.dynamic_flow import DynamicFlow
+    from crawler.scenarios.fresh_session import FreshSessionScenario
+    from db.repositories.runs import runs_repo
+    from logger import get_logger
+
+    log = get_logger(__name__)
+
+    stop_ev = threading.Event()
+    with _lock:
+        _stop_events[batch_id] = stop_ev
+        _batch_meta[batch_id] = {
+            "running": True,
+            "stopping": False,
+            "total": total_runs,
+            "completed": 0,
+            "succeeded": 0,
+            "failed": 0,
+            "configured_workers": workers,
+            "started_at": time.time(),
+            "finished_at": None,
+            "error": None,
+        }
+
+    def _run_one(slot: int) -> bool:
+        slot_key = f"{batch_id}:{slot:04d}"
+
+        if stop_ev.is_set():
+            with _lock:
+                _worker_slots[slot_key] = {
+                    "slot": slot, "status": "skipped",
+                    "started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped",
+                }
+            return False
+
+        with _lock:
+            _worker_slots[slot_key] = {
+                "slot": slot, "status": "running",
+                "started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None,
+            }
+
+        t0 = time.monotonic()
+        identifier = f"batch-{batch_id[:8]}-{slot:04d}"
+        log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
+
+        try:
+            target_url: str = cfg.get("target_url") or ""
+            with driver_session(headless=headless) as driver:
+                FreshSessionScenario(driver, target_url).run()
+                result = DynamicFlow(driver, cfg).run() if cfg else None
+
+            total_ms = int((time.monotonic() - t0) * 1000)
+            success = result.success if result else True
+            reason = result.failure_reason if result else None
+            steps = [
+                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
+                for s in (result.steps if result else [])
+            ]
+            asyncio.run(runs_repo.insert("batch", identifier, success, reason, steps, total_ms))
+
+            status = "ok" if success else "failed"
+            with _lock:
+                _worker_slots[slot_key].update({
+                    "status": status, "finished_at": time.time(), "duration_ms": total_ms,
+                })
+            log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms)
+            return success
+
+        except Exception as exc:
+            total_ms = int((time.monotonic() - t0) * 1000)
+            asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
+            with _lock:
+                _worker_slots[slot_key].update({
+                    "status": "error", "error": str(exc),
+                    "finished_at": time.time(), "duration_ms": total_ms,
+                })
+            log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc)
+            return False
+
+    succeeded = 0
+    failed = 0
+
+    try:
+        with ThreadPoolExecutor(
+            max_workers=workers,
+            thread_name_prefix=f"batch-{batch_id[:8]}",
+        ) as pool:
+            futures = []
+            for i in range(total_runs):
+                if stop_ev.is_set():
+                    break
+                futures.append(pool.submit(_run_one, i))
+                if stagger_ms > 0 and i < total_runs - 1:
+                    if stop_ev.wait(timeout=stagger_ms / 1000):
+                        break  # stop was signalled during the stagger delay
+
+            for fut in as_completed(futures):
+                try:
+                    ok = fut.result()
+                except Exception:
+                    ok = False
+                if ok:
+                    succeeded += 1
+                else:
+                    failed += 1
+                with _lock:
+                    _batch_meta[batch_id]["completed"] = succeeded + failed
+                    _batch_meta[batch_id]["succeeded"] = succeeded
+                    _batch_meta[batch_id]["failed"] = failed
+
+    except Exception as exc:
+        with _lock:
+            _batch_meta[batch_id]["error"] = str(exc)
+
+    finally:
+        with _lock:
+            _batch_meta[batch_id]["running"] = False
+            _batch_meta[batch_id]["finished_at"] = time.time()
+
+    return {"batch_id": batch_id, "succeeded": succeeded, "failed": failed}

db/connection.py

@@ -38,13 +38,21 @@ CREATE TABLE IF NOT EXISTS flow_configs (
     search_submit_selector      TEXT    NOT NULL DEFAULT '',
     search_submit_selector_type TEXT    NOT NULL DEFAULT 'css',
     search_results_selector      TEXT    NOT NULL DEFAULT '',
+    search_results_selector_type TEXT    NOT NULL DEFAULT 'css',
     scroll_container_selector      TEXT    NOT NULL DEFAULT '',
+    scroll_container_selector_type TEXT    NOT NULL DEFAULT 'css',
     max_scrolls                 INTEGER NOT NULL DEFAULT 30,
     scroll_pause_ms             INTEGER NOT NULL DEFAULT 1200,
     no_new_content_timeout_ms   INTEGER NOT NULL DEFAULT 3000,
+    pagination_selector         TEXT    NOT NULL DEFAULT '',
+    pagination_selector_type    TEXT    NOT NULL DEFAULT 'css',
+    max_pages                   INTEGER NOT NULL DEFAULT 10,
+    pagination_wait_ms          INTEGER NOT NULL DEFAULT 1500,
     item_selector_template      TEXT    NOT NULL DEFAULT '',
     item_selector_type          TEXT    NOT NULL DEFAULT 'css',
-    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]'
+    item_url_template           TEXT    NOT NULL DEFAULT '',
+    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]',
+    target_url                  TEXT    NOT NULL DEFAULT ''
 );
 """
 
@@ -55,10 +63,29 @@ def init_db() -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     conn = sqlite3.connect(str(path))
     conn.executescript(_DDL)
+    # Additive migrations for columns added after initial release
+    _migrate(conn)
     conn.commit()
     conn.close()
 
 
+def _migrate(conn: sqlite3.Connection) -> None:
+    existing = {row[1] for row in conn.execute("PRAGMA table_info(flow_configs)")}
+    additions = {
+        "search_results_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "item_url_template": "TEXT NOT NULL DEFAULT ''",
+        "scroll_container_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "pagination_selector": "TEXT NOT NULL DEFAULT ''",
+        "pagination_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "max_pages": "INTEGER NOT NULL DEFAULT 10",
+        "pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
+        "target_url": "TEXT NOT NULL DEFAULT ''",
+    }
+    for col, definition in additions.items():
+        if col not in existing:
+            conn.execute(f"ALTER TABLE flow_configs ADD COLUMN {col} {definition}")
+
+
 @asynccontextmanager
 async def get_db() -> AsyncGenerator[aiosqlite.Connection, None]:
     """Async context manager — yields an open, Row-factory-enabled connection."""

db/repositories/flow_configs.py

@@ -11,9 +11,13 @@ FlowConfigRow = dict[str, Any]
 _FIELDS = [
     "name", "is_active",
     "search_texts_json", "search_box_selector", "search_box_selector_type",
-    "search_submit_selector", "search_submit_selector_type", "search_results_selector",
+    "search_submit_selector", "search_submit_selector_type",
-    "scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
+    "search_results_selector", "search_results_selector_type",
-    "item_selector_template", "item_selector_type", "target_item_ids_json",
+    "scroll_container_selector", "scroll_container_selector_type",
+    "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
+    "pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms",
+    "item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json",
+    "target_url",
 ]
 
 

logger.py

@@ -0,0 +1,109 @@
+"""Central logging configuration for the Seed project.
+
+Usage anywhere in the codebase:
+    from logger import get_logger
+    log = get_logger(__name__)
+    log.info("message")
+    log.warning("watch out")
+    log.error("something broke")
+
+Environment variables:
+    LOG_LEVEL   — DEBUG | INFO | WARNING | ERROR | CRITICAL  (default: INFO)
+    LOG_FILE    — path to write a rotating log file (optional)
+"""
+from __future__ import annotations
+
+import logging
+import logging.handlers
+import os
+import sys
+from typing import Final
+
+_LEVEL_MAP: Final = {
+    "DEBUG": logging.DEBUG,
+    "INFO": logging.INFO,
+    "WARNING": logging.WARNING,
+    "ERROR": logging.ERROR,
+    "CRITICAL": logging.CRITICAL,
+}
+
+_LOG_LEVEL: int = _LEVEL_MAP.get(os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
+_LOG_FILE: str | None = os.getenv("LOG_FILE")
+
+# ── ANSI colour codes (console only) ─────────────────────────────────────────
+_RESET  = "\033[0m"
+_BOLD   = "\033[1m"
+_GREY   = "\033[90m"
+_CYAN   = "\033[96m"
+_YELLOW = "\033[93m"
+_RED    = "\033[91m"
+_BRED   = "\033[1;91m"
+
+_LEVEL_COLOURS: Final[dict[int, str]] = {
+    logging.DEBUG:    _GREY,
+    logging.INFO:     _CYAN,
+    logging.WARNING:  _YELLOW,
+    logging.ERROR:    _RED,
+    logging.CRITICAL: _BRED,
+}
+
+
+class _ColourFormatter(logging.Formatter):
+    _FMT = "{colour}{level:<8}{reset} {grey}{asctime}  {name}{reset}  {msg}"
+
+    def format(self, record: logging.LogRecord) -> str:
+        colour = _LEVEL_COLOURS.get(record.levelno, "")
+        level  = record.levelname
+        grey   = _GREY if sys.stderr.isatty() else ""
+        reset  = _RESET if sys.stderr.isatty() else ""
+        col    = colour if sys.stderr.isatty() else ""
+
+        self.datefmt = "%H:%M:%S"
+        base = super().format(record)
+        return self._FMT.format(
+            colour=col, level=level, reset=reset,
+            grey=grey, asctime=self.formatTime(record, self.datefmt),
+            name=record.name, msg=record.getMessage(),
+        ) + (f"\n{record.exc_text}" if record.exc_info and self.formatException(record.exc_info) else "")
+
+
+_FILE_FMT = logging.Formatter(
+    fmt="%(asctime)s  %(levelname)-8s  %(name)s  %(message)s",
+    datefmt="%Y-%m-%dT%H:%M:%S",
+)
+
+
+def _build_root_logger() -> logging.Logger:
+    root = logging.getLogger("seed")
+    root.setLevel(_LOG_LEVEL)
+
+    if root.handlers:
+        return root  # already configured (e.g. imported twice)
+
+    # Console handler
+    console = logging.StreamHandler(sys.stderr)
+    console.setLevel(_LOG_LEVEL)
+    console.setFormatter(_ColourFormatter())
+    root.addHandler(console)
+
+    # Optional rotating file handler
+    if _LOG_FILE:
+        fh = logging.handlers.RotatingFileHandler(
+            _LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8"
+        )
+        fh.setLevel(_LOG_LEVEL)
+        fh.setFormatter(_FILE_FMT)
+        root.addHandler(fh)
+
+    root.propagate = False
+    return root
+
+
+_root = _build_root_logger()
+
+
+def get_logger(name: str) -> logging.Logger:
+    """Return a child logger namespaced under 'seed'."""
+    if name.startswith("seed.") or name == "seed":
+        return logging.getLogger(name)
+    return logging.getLogger(f"seed.{name}")

main.py

@@ -8,15 +8,18 @@ import time
 
 import uvicorn
 
+from config import config
 from db import flow_config_repo, init_db
 from db.repositories.runs import runs_repo
-from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
 
 
 def _load_active_flow() -> dict[str, object]:
     cfg = asyncio.run(flow_config_repo.get_active())
     if cfg is None:
-        print("[WARN] No active flow config — running without dynamic flow steps.", file=sys.stderr)
+        log.warning("No active flow config — running without dynamic flow steps.")
     return cfg or {}
 
 
@@ -32,7 +35,7 @@ def _record(
     total_ms = int((time.monotonic() - t0) * 1000)
     if exc is not None:
         asyncio.run(runs_repo.insert(scenario, identifier, False, str(exc), [], total_ms))
-        print(f"[FAIL] {identifier}: {exc}", file=sys.stderr)
+        log.error("%s failed: %s", identifier, exc)
         return
 
     if isinstance(flow_result, DynamicFlowResult):
@@ -44,12 +47,13 @@ def _record(
             scenario, identifier, flow_result.success,
             flow_result.failure_reason, steps_data, total_ms,
         ))
-        status = "OK" if flow_result.success else f"FAIL ({flow_result.failure_reason})"
+        if flow_result.success:
+            log.info("%s OK — %d ms", identifier, total_ms)
+        else:
+            log.error("%s FAIL (%s) — %d ms", identifier, flow_result.failure_reason, total_ms)
     else:
         asyncio.run(runs_repo.insert(scenario, identifier, True, None, [], total_ms))
-        status = "OK (no flow config)"
+        log.info("%s OK (no flow config) — %d ms", identifier, total_ms)
-
-    print(f"[{status}] {identifier} — {total_ms} ms")
 
 
 def run_otp(phone: str) -> None:
@@ -58,14 +62,14 @@ def run_otp(phone: str) -> None:
     from crawler.scenarios.otp_login import OTPLoginScenario
 
     def otp_resolver(p: str) -> str:
-        # TODO: wire up your SMS gateway / webhook here.
         raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
 
     flow_cfg = _load_active_flow()
+    target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
     t0 = time.monotonic()
     try:
         with driver_session() as driver:
-            OTPLoginScenario(driver, phone, otp_resolver).run()
+            OTPLoginScenario(driver, phone, otp_resolver, target_url).run()
             result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
     except Exception as exc:
         _record("otp", phone, None, exc, t0)
@@ -79,10 +83,11 @@ def run_fresh() -> None:
     from crawler.scenarios.fresh_session import FreshSessionScenario
 
     flow_cfg = _load_active_flow()
+    target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
     t0 = time.monotonic()
     try:
         with driver_session() as driver:
-            FreshSessionScenario(driver).run()
+            FreshSessionScenario(driver, target_url).run()
             result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
     except Exception as exc:
         _record("fresh", "fresh", None, exc, t0)
@@ -90,6 +95,34 @@ def run_fresh() -> None:
     _record("fresh", "fresh", result, None, t0)
 
 
+def run_batch(workers: int, total_runs: int, stagger_ms: int, headless: bool) -> None:
+    from crawler.batch import run_batch as _run_batch
+
+    flow_cfg = _load_active_flow()
+    if not flow_cfg:
+        log.error("No active flow config — activate one in the admin panel first.")
+        sys.exit(1)
+
+    completed_count = [0]
+
+    def _progress(completed: int, succeeded: int, failed: int) -> None:
+        completed_count[0] = completed
+        log.info("Progress: %d/%d  (ok=%d  fail=%d)", completed, total_runs, succeeded, failed)
+
+    result = _run_batch(
+        cfg=flow_cfg,
+        workers=workers,
+        total_runs=total_runs,
+        headless=headless,
+        stagger_ms=stagger_ms,
+        on_progress=_progress,
+    )
+    log.info(
+        "Batch complete — succeeded=%d/%d  failed=%d",
+        result["succeeded"], result["total"], result["failed"],
+    )
+
+
 def run_admin() -> None:
     init_db()
     from admin.main import app
@@ -107,6 +140,12 @@ def main() -> None:
 
     sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
 
+    batch_p = sub.add_parser("batch", help="Run many parallel fresh-session runners")
+    batch_p.add_argument("--workers", type=int, default=3, help="Parallel Chrome instances (default: 3)")
+    batch_p.add_argument("--runs", type=int, default=10, help="Total runs to complete (default: 10)")
+    batch_p.add_argument("--stagger", type=int, default=500, help="Ms delay between worker launches (default: 500)")
+    batch_p.add_argument("--no-headless", dest="headless", action="store_false", help="Show browser windows")
+
     args = parser.parse_args()
 
     if args.cmd == "admin":
@@ -114,12 +153,19 @@ def main() -> None:
     elif args.cmd == "otp":
         phones = [args.phone] if args.phone else config.phone_numbers
         if not phones:
-            print("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone", file=sys.stderr)
+            log.error("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone")
             sys.exit(1)
         for phone in phones:
             run_otp(phone)
     elif args.cmd == "fresh":
         run_fresh()
+    elif args.cmd == "batch":
+        run_batch(
+            workers=args.workers,
+            total_runs=args.runs,
+            stagger_ms=args.stagger,
+            headless=args.headless,
+        )
 
 
 if __name__ == "__main__":

pyproject.toml

@@ -14,6 +14,7 @@ dependencies = [
     "pydantic>=2.7.0",
     "python-dotenv>=1.0.1",
     "setuptools>=70.0.0",
+    "huey>=2.5.0",
 ]
 
 [tool.pyright]

scripts/patch_driver.py

@@ -0,0 +1,106 @@
+"""
+Download and patch ChromeDriver once.  Run via: make patch-driver
+
+Uses the official Chrome for Testing API to fetch the exact ChromeDriver
+version matching the installed Chrome, then patches it with undetected_chromedriver.
+"""
+from __future__ import annotations
+
+import io
+import shutil
+import stat
+import subprocess
+import sys
+import urllib.request
+import zipfile
+from pathlib import Path
+
+import undetected_chromedriver as uc
+
+from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
+
+
+def _get_chrome_full_version() -> tuple[int, str]:
+    candidates = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
+    if config.chrome_binary:
+        candidates.insert(0, config.chrome_binary)
+
+    for binary in candidates:
+        if not shutil.which(binary) and not Path(binary).exists():
+            continue
+        try:
+            out = subprocess.check_output(
+                [binary, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
+            ).strip()
+            full = out.split()[-1]
+            major = int(full.split(".")[0])
+            log.info("Detected Chrome %s via '%s'", full, binary)
+            return major, full
+        except Exception:
+            continue
+
+    log.critical("Could not detect Chrome version. Set CHROME_BINARY in .env")
+    sys.exit(1)
+
+
+def _latest_chromedriver_version(major: int) -> str:
+    url = f"https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_{major}"
+    try:
+        with urllib.request.urlopen(url, timeout=15) as resp:
+            version = resp.read().decode().strip()
+            log.info("Latest ChromeDriver for Chrome %d: %s", major, version)
+            return version
+    except Exception as exc:
+        log.critical("Could not fetch ChromeDriver version for Chrome %d: %s", major, exc)
+        sys.exit(1)
+
+
+def _download_chromedriver(version: str, dest: Path) -> None:
+    zip_url = (
+        f"https://storage.googleapis.com/chrome-for-testing-public"
+        f"/{version}/linux64/chromedriver-linux64.zip"
+    )
+    log.info("Downloading ChromeDriver %s …", version)
+    try:
+        with urllib.request.urlopen(zip_url, timeout=60) as resp:
+            data = resp.read()
+    except Exception as exc:
+        log.critical("Download failed: %s", exc)
+        sys.exit(1)
+
+    with zipfile.ZipFile(io.BytesIO(data)) as zf:
+        binary_name = next(
+            n for n in zf.namelist() if n.endswith("/chromedriver") or n == "chromedriver"
+        )
+        dest.write_bytes(zf.read(binary_name))
+
+    dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
+    log.info("Extracted to %s", dest)
+
+
+def _patch(dest: Path) -> None:
+    log.info("Patching binary …")
+    patcher = uc.Patcher(executable_path=str(dest))
+    patcher.patch_exe()
+    log.info("Patch applied successfully")
+
+
+def main() -> None:
+    dest = Path(config.chromedriver_path)
+    dest.parent.mkdir(parents=True, exist_ok=True)
+
+    major, _ = _get_chrome_full_version()
+    cd_version = _latest_chromedriver_version(major)
+
+    _download_chromedriver(cd_version, dest)
+    _patch(dest)
+
+    log.info("ChromeDriver %s ready at %s", cd_version, dest)
+    log.info("Re-run 'make patch-driver' only when Chrome updates to a new major version.")
+
+
+if __name__ == "__main__":
+    main()

uv.toml

@@ -1,2 +0,0 @@
-# Force resolution through the official PyPI index
-index-url = "https://pypi.org/simple"