wip

2026-06-16 16:56:27 +03:30
parent a5d09be46e
commit 24155ad6bf
22 changed files with 2485 additions and 762 deletions
@@ -1,6 +1,8 @@
 # Copy to .env and fill in
-TARGET_URL=https://example.com
+# Logging
 LOG_LEVEL=INFO        # DEBUG | INFO | WARNING | ERROR | CRITICAL
 # LOG_FILE=logs/seed.log   # optional rotating file log
 # Admin panel credentials
 ADMIN_USERNAME=admin
@@ -13,7 +15,8 @@ DB_PATH=data/tracker.db
 # Chrome: set to false to watch the browser window
 HEADLESS=true
-# CHROME_BINARY=/usr/bin/chromium  # optional: explicit Chrome path
+# CHROME_BINARY=/usr/bin/google-chrome-stable  # optional: explicit Chrome binary path
 # CHROMEDRIVER_PATH=drivers/chromedriver       # pre-patched driver (run: make patch-driver)
 # Scenario 1 — comma-separated phone numbers
 PHONE_NUMBERS=+989100000001,+989100000002,+989100000003
@@ -5,3 +5,4 @@ __pycache__/
 data/
 *.db
 *.swp
 drivers/
@@ -11,7 +11,7 @@ GREEN := $(shell tput setaf 2 2>/dev/null)
 CYAN  := $(shell tput setaf 6 2>/dev/null)
 .PHONY: help install sync lock lint typecheck \
-        admin otp fresh \
+        admin otp fresh batch \
        build up down logs shell \
        clean
@@ -46,6 +46,10 @@ typecheck: ## Run Pyright type-checker
 # ── Crawler ───────────────────────────────────────────────────────────────────
 patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chrome updates)
 	@mkdir -p drivers
 	PYTHONPATH=. $(PYTHON) scripts/patch_driver.py
 admin: ## Start the admin panel (http://localhost:8000)
 	@mkdir -p $(DATA_DIR)
 	$(PYTHON) main.py admin
@@ -62,6 +66,14 @@ fresh: ## Run Scenario 2 — fresh cookie session
 	@mkdir -p $(DATA_DIR)
 	$(PYTHON) main.py fresh
 WORKERS ?= 3
 RUNS    ?= 10
 STAGGER ?= 500
 batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500)
 	@mkdir -p $(DATA_DIR)
 	$(PYTHON) main.py batch --workers $(WORKERS) --runs $(RUNS) --stagger $(STAGGER)
 # ── Docker ────────────────────────────────────────────────────────────────────
 build: ## Build the Docker image
@@ -1,6 +1,7 @@
 """FastAPI admin application."""
 from __future__ import annotations
 import threading
 from pathlib import Path
 from fastapi import FastAPI
@@ -21,6 +22,20 @@ app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
@app.on_event("startup")
 def on_startup() -> None:
    init_db()
    _start_huey_consumer()
 def _start_huey_consumer() -> None:
    from huey.consumer import Consumer
    from crawler.tasks import huey
    class _ThreadConsumer(Consumer):
        def _set_signal_handlers(self) -> None:
            pass  # signal.signal() only works on the main thread
    consumer = _ThreadConsumer(huey, workers=1, periodic=False)
    t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
    t.start()
@app.get("/")
@@ -2,6 +2,7 @@
 from __future__ import annotations
 import secrets
 import uuid
 from typing import Annotated, Any
 from fastapi import APIRouter, Depends, HTTPException, status
@@ -17,6 +18,9 @@ from admin.models import (
    upsert_flow_config,
 )
 from config import config
 from logger import get_logger
 log = get_logger(__name__)
 router = APIRouter()
 security = HTTPBasic()
@@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel):
    search_submit_selector: str = ""
    search_submit_selector_type: str = "css"
    search_results_selector: str = ""
    search_results_selector_type: str = "css"
    scroll_container_selector: str = ""
    scroll_container_selector_type: str = "css"
    max_scrolls: int = 30
    scroll_pause_ms: int = 1200
    no_new_content_timeout_ms: int = 3000
    pagination_selector: str = ""
    pagination_selector_type: str = "css"
    max_pages: int = 10
    pagination_wait_ms: int = 1500
    item_selector_template: str = ""
    item_selector_type: str = "css"
    item_url_template: str = ""
    target_item_ids: list[str] = []
    target_url: str = ""
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
        "search_submit_selector": body.search_submit_selector,
        "search_submit_selector_type": body.search_submit_selector_type,
        "search_results_selector": body.search_results_selector,
        "search_results_selector_type": body.search_results_selector_type,
        "scroll_container_selector": body.scroll_container_selector,
        "scroll_container_selector_type": body.scroll_container_selector_type,
        "max_scrolls": body.max_scrolls,
        "scroll_pause_ms": body.scroll_pause_ms,
        "no_new_content_timeout_ms": body.no_new_content_timeout_ms,
        "pagination_selector": body.pagination_selector,
        "pagination_selector_type": body.pagination_selector_type,
        "max_pages": body.max_pages,
        "pagination_wait_ms": body.pagination_wait_ms,
        "item_selector_template": body.item_selector_template,
        "item_selector_type": body.item_selector_type,
        "item_url_template": body.item_url_template,
        "target_item_ids_json": body.target_item_ids,
        "target_url": body.target_url,
    }
@@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
    if not existing:
        raise HTTPException(status_code=404, detail="Not found")
    await delete_flow_config(cfg_id)
 # ---------------------------------------------------------------------------
 # Batch runs (managed via Huey task queue)
 # ---------------------------------------------------------------------------
 class BatchRunIn(BaseModel):
    workers: int = 3
    total_runs: int = 10
    stagger_ms: int = 500
    headless: bool = True
@router.post("/api/run-batch", status_code=202)
 async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
    from crawler.tasks import batch_status as _status, run_batch_task, set_current
    current = _status()
    if current.get("running"):
        raise HTTPException(status_code=409, detail="A batch is already running")
    from admin.models import get_active_flow_config
    active_cfg = await get_active_flow_config()
    if not active_cfg:
        raise HTTPException(status_code=422, detail="No active flow config — activate one first")
    batch_id = uuid.uuid4().hex
    result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
    set_current(batch_id, result)
    log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
    return {"batch_id": batch_id, "status": "queued"}
@router.post("/api/stop-batch", status_code=200)
 async def stop_batch(_: Auth) -> dict[str, object]:
    from crawler.tasks import stop_current
    stopped = stop_current()
    if not stopped:
        raise HTTPException(status_code=404, detail="No active batch to stop")
    return {"status": "stopping"}
@router.get("/api/batch-status")
 async def batch_status(_: Auth) -> dict[str, object]:
    from crawler.tasks import batch_status as _status
    return _status()  # type: ignore[return-value]
@@ -102,6 +102,17 @@
    }
    .form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
    .form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
    details.adv { margin-top: 20px; border: 1px solid var(--border); border-radius: 9px; }
    details.adv > summary {
      list-style: none; cursor: pointer; padding: 11px 14px;
      font-size: 12px; font-weight: 600; color: var(--muted);
      display: flex; align-items: center; gap: 7px; user-select: none;
    }
    details.adv > summary::-webkit-details-marker { display: none; }
    details.adv > summary::before { content: '▶'; font-size: 9px; transition: transform .15s; }
    details.adv[open] > summary::before { transform: rotate(90deg); }
    details.adv > summary:hover { color: var(--text); }
    .adv-body { padding: 4px 14px 14px; }
    .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
    .fg { margin-bottom: 14px; }
    .fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
@@ -124,6 +135,9 @@
    .modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
    .error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
    .error-msg.hidden { display: none; }
    .remember-row { display: flex; align-items: center; gap: 8px; margin-top: 12px; }
    .remember-row input[type=checkbox] { width: 15px; height: 15px; accent-color: var(--accent); cursor: pointer; }
    .remember-row label { color: var(--muted); font-size: 13px; cursor: pointer; user-select: none; }
  </style>
 </head>
 <body>
@@ -134,7 +148,11 @@
    <h2>Seed Admin</h2>
    <div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
    <div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
-    <button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
+    <div class="remember-row">
      <input type="checkbox" id="inp-remember" />
      <label for="inp-remember">Remember me</label>
    </div>
    <button class="btn btn-primary" style="width:100%;margin-top:14px" id="login-btn">Sign in</button>
    <p class="error-msg hidden" id="auth-error">Invalid credentials</p>
  </div>
 </div>
@@ -149,80 +167,130 @@
      <label>Config Name</label>
      <input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
    </div>
    <div class="fg">
-      <label style="display:flex;align-items:center;gap:8px;cursor:pointer">
+      <label>Target URL</label>
-        <input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
+      <input id="cfg-target-url" type="url" placeholder="https://example.com" />
      </label>
    </div>
-    <!-- Search -->
+    <div class="fg" style="margin-top:6px">
    <div class="form-section">Search Step</div>
    <div class="fg">
      <label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
      <textarea id="cfg-search-texts" placeholder="shoes&#10;blue sneakers&#10;running shoes"></textarea>
    </div>
    <div class="form-row">
      <div class="fg">
        <label>Search Box Selector</label>
        <input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
      </div>
      <div class="fg">
        <label>Type</label>
        <select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
      </div>
    </div>
    <div class="form-row">
      <div class="fg">
        <label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
        <input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
      </div>
      <div class="fg">
        <label>Type</label>
        <select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
      </div>
    </div>
    <div class="fg">
      <label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
      <input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
    </div>
    <!-- Scroll -->
    <div class="form-section">Scroll Step (Infinite Scroll)</div>
    <div class="fg">
      <label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
      <input id="cfg-scroll-container" type="text" placeholder=".product-list" />
    </div>
    <div class="form-row">
      <div class="fg">
        <label>Max Scrolls</label>
        <input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
      </div>
      <div class="fg">
        <label>Scroll Pause (ms)</label>
        <input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
      </div>
    </div>
    <div class="fg">
      <label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
      <input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
    </div>
    <!-- Item -->
    <div class="form-section">Item Selection</div>
    <div class="fg">
      <label>Item Selector Template</label>
      <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
      <p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
    </div>
    <div class="fg">
      <label>Selector Type</label>
      <select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
    </div>
    <div class="fg">
      <label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
      <textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
    </div>
    <!-- Advanced -->
    <details class="adv" id="adv-details">
      <summary>Advanced Settings</summary>
      <div class="adv-body">
        <div class="fg">
          <label style="display:flex;align-items:center;gap:8px;cursor:pointer">
            <input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
          </label>
        </div>
        <div class="form-section">Search Step</div>
        <div class="form-row">
          <div class="fg">
            <label>Search Box Selector</label>
            <input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
          </div>
          <div class="fg">
            <label>Type</label>
            <select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
          </div>
        </div>
        <div class="form-row">
          <div class="fg">
            <label>Submit Button Selector <span style="color:var(--muted)">(blank → Enter key)</span></label>
            <input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
          </div>
          <div class="fg">
            <label>Type</label>
            <select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
          </div>
        </div>
        <div class="form-row">
          <div class="fg">
            <label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
            <input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
          </div>
          <div class="fg">
            <label>Type</label>
            <select id="cfg-results-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
          </div>
        </div>
        <div class="form-section">Scroll</div>
        <div class="fg">
          <label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
          <div style="display:flex;gap:8px;align-items:center">
            <input id="cfg-scroll-container" type="text" placeholder=".product-list" style="flex:1" />
            <select id="cfg-scroll-container-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
          </div>
        </div>
        <div class="form-row">
          <div class="fg">
            <label>Max Scrolls</label>
            <input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
          </div>
          <div class="fg">
            <label>Scroll Pause (ms)</label>
            <input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
          </div>
        </div>
        <div class="fg">
          <label>No New Content Timeout (ms)</label>
          <input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
        </div>
        <div class="form-section">Pagination <span style="font-weight:400;color:var(--muted)">(optional)</span></div>
        <div class="fg">
          <label>Next Page Button Selector <span style="color:var(--muted)">(blank = disabled)</span></label>
          <div style="display:flex;gap:8px;align-items:center">
            <input id="cfg-pagination-sel" type="text" placeholder=".pagination .next, a[rel='next']" style="flex:1" />
            <select id="cfg-pagination-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
          </div>
        </div>
        <div class="form-row">
          <div class="fg">
            <label>Max Pages</label>
            <input id="cfg-max-pages" type="number" min="1" max="200" value="10" />
          </div>
          <div class="fg">
            <label>Wait After Page Turn (ms)</label>
            <input id="cfg-pagination-wait" type="number" min="200" max="15000" value="1500" />
          </div>
        </div>
        <div class="form-section">Item Selection</div>
        <div class="fg">
          <label>Item URL Template <span style="color:var(--muted)">(href pattern, highest priority)</span></label>
          <input id="cfg-item-url-tpl" type="text" placeholder="/products/{item_id}" />
          <p class="hint">Finds <code>&lt;a href&gt;</code> whose href contains this pattern. Use <code>{item_id}</code> as placeholder.</p>
        </div>
        <div class="fg">
          <label>Item Selector Template <span style="color:var(--muted)">(fallback)</span></label>
          <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or /products/{item_id}" />
        </div>
        <div class="fg">
          <label>Selector Type</label>
          <select id="cfg-item-sel-type" style="width:auto">
            <option value="css">CSS</option>
            <option value="xpath">XPath</option>
            <option value="a-link">A-Link (href contains)</option>
          </select>
          <p class="hint" id="alink-hint" style="display:none">Template matched against <code>&lt;a href&gt;</code> — e.g. <code>/products/{item_id}</code>.</p>
        </div>
      </div>
    </details>
    <div class="form-footer">
      <button class="btn btn-ghost" id="form-cancel">Cancel</button>
      <button class="btn btn-primary" id="form-save">Save Config</button>
@@ -244,6 +312,7 @@
 <div class="tabs">
  <div class="tab active" data-tab="dashboard">Dashboard</div>
  <div class="tab" data-tab="flows">Flow Configs</div>
  <div class="tab" data-tab="batch">Batch Run</div>
 </div>
 <!-- ── Dashboard Tab ──────────────────────────────────────────── -->
@@ -288,8 +357,12 @@
      <div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
      <div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
    </div>
-    <button class="btn btn-primary" id="btn-new-config">+ New Config</button>
+    <div style="display:flex;gap:8px">
      <button class="btn btn-ghost" id="btn-import-config">↑ Import</button>
      <button class="btn btn-primary" id="btn-new-config">+ New Config</button>
    </div>
  </div>
  <input type="file" id="import-file-input" accept=".json" style="display:none">
  <div class="config-list" id="config-list">
    <p class="empty">No configs yet. Create one to get started.</p>
@@ -297,6 +370,96 @@
 </main>
 </div>
 <!-- ── Batch Run Tab ─────────────────────────────────────────── -->
 <div class="tab-panel" id="tab-batch">
 <main>
  <div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div>
  <div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using the active flow config.</div>
  <div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px">
    <!-- Controls -->
    <div class="panel" style="align-self:start">
      <div class="section-title">Configuration</div>
      <div class="fg" style="margin-bottom:14px">
        <label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label>
        <input id="batch-workers" type="number" min="1" max="50" value="3" />
      </div>
      <div class="fg" style="margin-bottom:14px">
        <label>Total Runs</label>
        <input id="batch-runs" type="number" min="1" max="10000" value="10" />
      </div>
      <div class="fg" style="margin-bottom:14px">
        <label>Stagger Between Starts (ms)</label>
        <input id="batch-stagger" type="number" min="0" max="30000" value="500" />
      </div>
      <div style="display:flex;align-items:center;gap:8px;margin-bottom:20px">
        <input id="batch-headless" type="checkbox" checked style="width:16px;height:16px;cursor:pointer" />
        <label for="batch-headless" style="cursor:pointer;margin:0">Headless</label>
      </div>
      <div style="display:flex;gap:10px">
        <button class="btn btn-primary" id="batch-start-btn" style="flex:1">▶ Start</button>
        <button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button>
      </div>
      <div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div>
    </div>
    <!-- Status -->
    <div class="panel">
      <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
        <div class="section-title" style="margin:0">Live Status</div>
        <span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span>
      </div>
      <div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div>
      <div id="batch-status-body" style="display:none">
        <div style="margin-bottom:16px">
          <div style="display:flex;justify-content:space-between;margin-bottom:6px">
            <span style="color:var(--muted);font-size:12px">Progress</span>
            <span id="batch-progress-label" style="font-size:12px">0 / 0</span>
          </div>
          <div style="background:var(--surface2);border-radius:4px;height:8px;overflow:hidden">
            <div id="batch-progress-bar" style="height:100%;background:var(--accent);width:0%;transition:width .3s"></div>
          </div>
        </div>
        <div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:10px;margin-bottom:16px">
          <div class="card" style="padding:12px">
            <div class="label">Completed</div>
            <div class="value accent" id="bs-completed">0</div>
          </div>
          <div class="card" style="padding:12px">
            <div class="label">Succeeded</div>
            <div class="value green" id="bs-succeeded">0</div>
          </div>
          <div class="card" style="padding:12px">
            <div class="label">Failed</div>
            <div class="value red" id="bs-failed">0</div>
          </div>
        </div>
        <div style="font-size:12px;color:var(--muted);margin-bottom:14px">
          Workers: <strong id="bs-workers">–</strong> &nbsp;·&nbsp;
          Started: <strong id="bs-started">–</strong>
          <span id="bs-finished-row" style="display:none"> &nbsp;·&nbsp; Finished: <strong id="bs-finished">–</strong></span>
        </div>
        <!-- Per-worker table -->
        <div class="section-title">Workers</div>
        <table id="bs-worker-table" style="font-size:12px">
          <thead>
            <tr>
              <th>#</th>
              <th>Status</th>
              <th>Duration</th>
              <th>Error</th>
            </tr>
          </thead>
          <tbody id="bs-worker-tbody"></tbody>
        </table>
      </div>
    </div>
  </div>
 </main>
 </div>
 <script>
 /* ─────────────────────────── Auth ─────────────────────────── */
 let _creds = null;
@@ -309,24 +472,73 @@ async function apiFetch(path, opts = {}) {
  return res;
 }
-function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
+const _STORAGE_KEY = 'seed_admin_creds';
-function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
+
 function showAuth() {
  document.getElementById('auth-modal').classList.remove('hidden');
 }
 function hideAuth() {
  document.getElementById('auth-modal').classList.add('hidden');
 }
 function _loadSaved() {
  try {
    const raw = localStorage.getItem(_STORAGE_KEY);
    if (!raw) return null;
    return JSON.parse(raw);
  } catch { return null; }
 }
 function _saveCreds(u, p) {
  localStorage.setItem(_STORAGE_KEY, JSON.stringify({ u, p }));
 }
 function _clearCreds() {
  localStorage.removeItem(_STORAGE_KEY);
 }
 async function _attemptLogin(u, p) {
  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
  return res;
 }
 document.getElementById('login-btn').addEventListener('click', async () => {
  const u = document.getElementById('inp-user').value.trim();
  const p = document.getElementById('inp-pass').value;
-  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
+  const remember = document.getElementById('inp-remember').checked;
  const res = await _attemptLogin(u, p);
  if (res.ok) {
    _creds = { u, p };
    remember ? _saveCreds(u, p) : _clearCreds();
    document.getElementById('auth-error').classList.add('hidden');
    hideAuth();
    await Promise.all([loadStats(), loadConfigs()]);
    startAutoRefresh();
  } else {
    _clearCreds();
    document.getElementById('auth-error').classList.remove('hidden');
  }
 });
-document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
+document.getElementById('inp-pass').addEventListener('keydown', e => {
  if (e.key === 'Enter') document.getElementById('login-btn').click();
 });
 // Auto-login from saved credentials on page load
 (async () => {
  const saved = _loadSaved();
  if (!saved) return;
  const res = await _attemptLogin(saved.u, saved.p);
  if (res.ok) {
    _creds = saved;
    document.getElementById('inp-user').value = saved.u;
    document.getElementById('inp-remember').checked = true;
    hideAuth();
    await Promise.all([loadStats(), loadConfigs()]);
    startAutoRefresh();
  } else {
    _clearCreds(); // saved creds are stale (password changed)
  }
 })();
 /* ─────────────────────────── Tabs ─────────────────────────── */
 document.querySelectorAll('.tab').forEach(tab => {
@@ -336,6 +548,7 @@ document.querySelectorAll('.tab').forEach(tab => {
    tab.classList.add('active');
    document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
    if (tab.dataset.tab === 'flows') loadConfigs();
    if (tab.dataset.tab === 'batch') pollBatchStatus();
  });
 });
@@ -426,11 +639,15 @@ function renderConfigs() {
        </div>
        <div class="config-meta" style="margin-top:4px">
          Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
          ${c.item_url_template ? `&nbsp;·&nbsp; href pattern: <code style="font-size:11px">${esc(c.item_url_template)}</code>` : ''}
          ${c.pagination_selector ? `&nbsp;·&nbsp; Pagination: <code style="font-size:11px">${esc(c.pagination_selector)}</code> (max ${c.max_pages} pages)` : ''}
        </div>
      </div>
      <div class="config-actions">
        ${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
        <button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
        <button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button>
        <button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button>
        <button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
      </div>
    </div>
@@ -449,25 +666,143 @@ async function deleteConfig(id) {
  if (res && (res.ok || res.status === 204)) await loadConfigs();
 }
 function exportConfig(id) {
  const c = _configs.find(x => x.id === id);
  if (!c) return;
  const out = {
    name: c.name,
    search_texts: c.search_texts_json || [],
    search_box_selector: c.search_box_selector,
    search_box_selector_type: c.search_box_selector_type,
    search_submit_selector: c.search_submit_selector,
    search_submit_selector_type: c.search_submit_selector_type,
    search_results_selector: c.search_results_selector,
    search_results_selector_type: c.search_results_selector_type,
    scroll_container_selector: c.scroll_container_selector,
    scroll_container_selector_type: c.scroll_container_selector_type,
    max_scrolls: c.max_scrolls,
    scroll_pause_ms: c.scroll_pause_ms,
    no_new_content_timeout_ms: c.no_new_content_timeout_ms,
    pagination_selector: c.pagination_selector,
    pagination_selector_type: c.pagination_selector_type,
    max_pages: c.max_pages,
    pagination_wait_ms: c.pagination_wait_ms,
    item_selector_template: c.item_selector_template,
    item_selector_type: c.item_selector_type,
    item_url_template: c.item_url_template,
    target_item_ids: c.target_item_ids_json || [],
    target_url: c.target_url || '',
  };
  const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
  const url = URL.createObjectURL(blob);
  const a = document.createElement('a');
  a.href = url; a.download = `${c.name.replace(/[^a-z0-9_-]/gi, '_')}.json`;
  a.click(); URL.revokeObjectURL(url);
 }
 async function duplicateConfig(id) {
  const c = _configs.find(x => x.id === id);
  if (!c) return;
  const body = {
    name: c.name + ' (copy)',
    is_active: false,
    search_texts: c.search_texts_json || [],
    search_box_selector: c.search_box_selector || '',
    search_box_selector_type: c.search_box_selector_type || 'css',
    search_submit_selector: c.search_submit_selector || '',
    search_submit_selector_type: c.search_submit_selector_type || 'css',
    search_results_selector: c.search_results_selector || '',
    search_results_selector_type: c.search_results_selector_type || 'css',
    scroll_container_selector: c.scroll_container_selector || '',
    scroll_container_selector_type: c.scroll_container_selector_type || 'css',
    max_scrolls: c.max_scrolls ?? 30,
    scroll_pause_ms: c.scroll_pause_ms ?? 1200,
    no_new_content_timeout_ms: c.no_new_content_timeout_ms ?? 3000,
    pagination_selector: c.pagination_selector || '',
    pagination_selector_type: c.pagination_selector_type || 'css',
    max_pages: c.max_pages ?? 10,
    pagination_wait_ms: c.pagination_wait_ms ?? 1500,
    item_selector_template: c.item_selector_template || '',
    item_selector_type: c.item_selector_type || 'css',
    item_url_template: c.item_url_template || '',
    target_item_ids: c.target_item_ids_json || [],
    target_url: c.target_url || '',
  };
  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
  if (res && res.ok) await loadConfigs();
 }
 document.getElementById('btn-import-config').addEventListener('click', () => {
  document.getElementById('import-file-input').click();
 });
 document.getElementById('import-file-input').addEventListener('change', async (e) => {
  const file = e.target.files[0];
  if (!file) return;
  e.target.value = '';
  let data;
  try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; }
  const body = {
    name: data.name || file.name.replace(/\.json$/, ''),
    is_active: false,
    search_texts: data.search_texts || [],
    search_box_selector: data.search_box_selector || '',
    search_box_selector_type: data.search_box_selector_type || 'css',
    search_submit_selector: data.search_submit_selector || '',
    search_submit_selector_type: data.search_submit_selector_type || 'css',
    search_results_selector: data.search_results_selector || '',
    search_results_selector_type: data.search_results_selector_type || 'css',
    scroll_container_selector: data.scroll_container_selector || '',
    scroll_container_selector_type: data.scroll_container_selector_type || 'css',
    max_scrolls: data.max_scrolls ?? 30,
    scroll_pause_ms: data.scroll_pause_ms ?? 1200,
    no_new_content_timeout_ms: data.no_new_content_timeout_ms ?? 3000,
    pagination_selector: data.pagination_selector || '',
    pagination_selector_type: data.pagination_selector_type || 'css',
    max_pages: data.max_pages ?? 10,
    pagination_wait_ms: data.pagination_wait_ms ?? 1500,
    item_selector_template: data.item_selector_template || '',
    item_selector_type: data.item_selector_type || 'css',
    item_url_template: data.item_url_template || '',
    target_item_ids: data.target_item_ids || [],
    target_url: data.target_url || '',
  };
  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
  if (res && res.ok) await loadConfigs();
 });
 /* ─── Form ─── */
-function openNewForm() {
+function _resetAdvanced() {
  document.getElementById('form-title').textContent = 'New Flow Config';
  document.getElementById('cfg-id').value = '';
  document.getElementById('cfg-name').value = '';
  document.getElementById('cfg-active').checked = false;
  document.getElementById('cfg-search-texts').value = '';
  document.getElementById('cfg-search-sel').value = '';
  document.getElementById('cfg-search-sel-type').value = 'css';
  document.getElementById('cfg-submit-sel').value = '';
  document.getElementById('cfg-submit-sel-type').value = 'css';
  document.getElementById('cfg-results-sel').value = '';
  document.getElementById('cfg-results-sel-type').value = 'css';
  document.getElementById('cfg-scroll-container').value = '';
  document.getElementById('cfg-scroll-container-type').value = 'css';
  document.getElementById('cfg-max-scrolls').value = '30';
  document.getElementById('cfg-scroll-pause').value = '1200';
  document.getElementById('cfg-no-content-timeout').value = '3000';
  document.getElementById('cfg-pagination-sel').value = '';
  document.getElementById('cfg-pagination-sel-type').value = 'css';
  document.getElementById('cfg-max-pages').value = '10';
  document.getElementById('cfg-pagination-wait').value = '1500';
  document.getElementById('cfg-item-sel').value = '';
  document.getElementById('cfg-item-sel-type').value = 'css';
  document.getElementById('alink-hint').style.display = 'none';
  document.getElementById('cfg-item-url-tpl').value = '';
 }
 function openNewForm() {
  document.getElementById('form-title').textContent = 'New Flow Config';
  document.getElementById('cfg-id').value = '';
  document.getElementById('cfg-name').value = '';
  document.getElementById('cfg-target-url').value = '';
  document.getElementById('cfg-search-texts').value = '';
  document.getElementById('cfg-item-ids').value = '';
  _resetAdvanced();
  document.getElementById('adv-details').removeAttribute('open');
  document.getElementById('config-overlay').classList.remove('hidden');
 }
@@ -477,20 +812,31 @@ function openEditForm(id) {
  document.getElementById('form-title').textContent = 'Edit Flow Config';
  document.getElementById('cfg-id').value = id;
  document.getElementById('cfg-name').value = c.name || '';
-  document.getElementById('cfg-active').checked = !!c.is_active;
+  document.getElementById('cfg-target-url').value = c.target_url || '';
  document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
  document.getElementById('cfg-active').checked = !!c.is_active;
  document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
  document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
  document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
  document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
  document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
  document.getElementById('cfg-results-sel-type').value = c.search_results_selector_type || 'css';
  document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
  document.getElementById('cfg-scroll-container-type').value = c.scroll_container_selector_type || 'css';
  document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
  document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
  document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
  document.getElementById('cfg-pagination-sel').value = c.pagination_selector || '';
  document.getElementById('cfg-pagination-sel-type').value = c.pagination_selector_type || 'css';
  document.getElementById('cfg-max-pages').value = c.max_pages ?? 10;
  document.getElementById('cfg-pagination-wait').value = c.pagination_wait_ms ?? 1500;
  document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
-  document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
+  const _ist = c.item_selector_type || 'css';
-  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
+  document.getElementById('cfg-item-sel-type').value = _ist;
  document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
  document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
  document.getElementById('adv-details').setAttribute('open', '');
  document.getElementById('config-overlay').classList.remove('hidden');
 }
@@ -508,19 +854,30 @@ function collectForm() {
    search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
    search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
    search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
    search_results_selector_type: document.getElementById('cfg-results-sel-type').value,
    scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
    scroll_container_selector_type: document.getElementById('cfg-scroll-container-type').value,
    max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
    scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
    no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
    pagination_selector: document.getElementById('cfg-pagination-sel').value.trim(),
    pagination_selector_type: document.getElementById('cfg-pagination-sel-type').value,
    max_pages: parseInt(document.getElementById('cfg-max-pages').value) || 10,
    pagination_wait_ms: parseInt(document.getElementById('cfg-pagination-wait').value) || 1500,
    item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
    item_selector_type: document.getElementById('cfg-item-sel-type').value,
    item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
    target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
    target_url: document.getElementById('cfg-target-url').value.trim(),
  };
 }
 document.getElementById('btn-new-config').addEventListener('click', openNewForm);
 document.getElementById('form-cancel').addEventListener('click', closeForm);
 document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
 document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
  document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
 });
 document.getElementById('form-save').addEventListener('click', async () => {
  const body = collectForm();
@@ -540,8 +897,9 @@ document.getElementById('form-save').addEventListener('click', async () => {
    closeForm();
    await loadConfigs();
  } else {
-    alert('Failed to save config. Check console.');
+    const detail = res ? await res.text().catch(() => `HTTP ${res.status}`) : 'No response (network error)';
-    console.error(await res?.text());
+    console.error('Save failed:', detail);
    alert(`Save failed (${res?.status ?? '?'}):\n${detail}\n\nIf columns are missing, restart the admin panel to apply DB migrations.`);
  }
 });
@@ -553,6 +911,162 @@ function fmtTime(iso) {
  if (!iso) return '–';
  return new Date(iso).toLocaleString();
 }
 /* ─────────────────────────── Batch Run ─────────────────────── */
 let _batchPollTimer = null;
 const _STATUS_COLOR = {
  running: 'var(--accent)',
  ok:      'var(--green)',
  failed:  'var(--red)',
  error:   'var(--red)',
  skipped: 'var(--muted)',
  pending: 'var(--muted)',
 };
 function fmtDuration(ms) {
  if (ms == null) return '–';
  if (ms < 1000) return ms + ' ms';
  return (ms / 1000).toFixed(1) + ' s';
 }
 function fmtTs(ts) {
  if (!ts) return '–';
  return new Date(ts * 1000).toLocaleTimeString();
 }
 function renderBatchStatus(s) {
  const idle     = document.getElementById('batch-idle-msg');
  const body     = document.getElementById('batch-status-body');
  const errEl    = document.getElementById('batch-error');
  const startBtn = document.getElementById('batch-start-btn');
  const stopBtn  = document.getElementById('batch-stop-btn');
  const badge    = document.getElementById('bs-active-badge');
  if (!s.batch_id) {
    idle.style.display = '';
    body.style.display = 'none';
    startBtn.style.display = '';
    stopBtn.style.display = 'none';
    badge.style.display = 'none';
    return;
  }
  idle.style.display = 'none';
  body.style.display = '';
  const pct = s.total ? Math.round(s.completed / s.total * 100) : 0;
  document.getElementById('batch-progress-bar').style.width = pct + '%';
  document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`;
  document.getElementById('bs-completed').textContent = s.completed ?? 0;
  document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0;
  document.getElementById('bs-failed').textContent = s.failed ?? 0;
  document.getElementById('bs-workers').textContent = s.configured_workers ?? '–';
  document.getElementById('bs-started').textContent = fmtTs(s.started_at);
  const finRow = document.getElementById('bs-finished-row');
  if (s.finished_at) {
    finRow.style.display = '';
    document.getElementById('bs-finished').textContent = fmtTs(s.finished_at);
  } else {
    finRow.style.display = 'none';
  }
  // Active worker badge
  const active = s.active_workers ?? 0;
  if (s.running && active > 0) {
    badge.textContent = `${active} / ${s.configured_workers} active`;
    badge.style.display = '';
  } else {
    badge.style.display = 'none';
  }
  // Per-worker table
  const tbody = document.getElementById('bs-worker-tbody');
  const workers = s.workers_detail || [];
  tbody.innerHTML = workers.map(w => {
    const color = _STATUS_COLOR[w.status] || 'var(--muted)';
    const label = w.status === 'running' ? '⟳ running' : w.status;
    const err = w.error && w.status !== 'skipped' ? `<span style="color:var(--red)">${esc(w.error)}</span>` : '–';
    return `<tr>
      <td style="font-family:monospace">${w.slot}</td>
      <td style="color:${color};font-weight:600">${label}</td>
      <td>${fmtDuration(w.duration_ms)}</td>
      <td style="max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td>
    </tr>`;
  }).join('');
  if (s.error) {
    errEl.textContent = 'Error: ' + s.error;
    errEl.style.display = '';
  } else {
    errEl.style.display = 'none';
  }
  if (s.running) {
    startBtn.style.display = 'none';
    stopBtn.style.display = '';
    stopBtn.disabled = s.stopping || false;
    stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop';
  } else {
    startBtn.style.display = '';
    stopBtn.style.display = 'none';
  }
 }
 async function pollBatchStatus() {
  const res = await apiFetch('/api/batch-status');
  if (!res || !res.ok) return;
  const s = await res.json();
  renderBatchStatus(s);
  if (s.running && !_batchPollTimer) startBatchPoll();
 }
 function startBatchPoll() {
  if (_batchPollTimer) return;
  _batchPollTimer = setInterval(async () => {
    const r = await apiFetch('/api/batch-status');
    if (!r || !r.ok) return;
    const st = await r.json();
    renderBatchStatus(st);
    if (!st.running) {
      clearInterval(_batchPollTimer);
      _batchPollTimer = null;
    }
  }, 1500);
 }
 document.getElementById('batch-start-btn').addEventListener('click', async () => {
  const workers  = parseInt(document.getElementById('batch-workers').value) || 3;
  const runs     = parseInt(document.getElementById('batch-runs').value) || 10;
  const stagger  = parseInt(document.getElementById('batch-stagger').value) || 0;
  const headless = document.getElementById('batch-headless').checked;
  const res = await apiFetch('/api/run-batch', {
    method: 'POST',
    headers: { 'Content-Type': 'application/json' },
    body: JSON.stringify({ workers, total_runs: runs, stagger_ms: stagger, headless }),
  });
  if (!res) return;
  if (res.status === 409) { alert('A batch is already running.'); return; }
  if (res.status === 422) { const d = await res.json(); alert(d.detail || 'No active flow config.'); return; }
  if (!res.ok) { alert('Failed to start batch.'); return; }
  await pollBatchStatus();
  startBatchPoll();
 });
 document.getElementById('batch-stop-btn').addEventListener('click', async () => {
  const btn = document.getElementById('batch-stop-btn');
  btn.disabled = true;
  btn.textContent = '■ Stopping…';
  const res = await apiFetch('/api/stop-batch', { method: 'POST' });
  if (!res || !res.ok) {
    btn.disabled = false;
    btn.textContent = '■ Stop';
  }
 });
 </script>
 </body>
 </html>
@@ -9,7 +9,6 @@ load_dotenv()
@dataclass
 class Config:
    target_url: str = os.getenv("TARGET_URL", "https://example.com")
    admin_username: str = os.getenv("ADMIN_USERNAME", "admin")
    admin_password: str = os.getenv("ADMIN_PASSWORD", "changeme")
    admin_host: str = os.getenv("ADMIN_HOST", "127.0.0.1")
@@ -17,6 +16,7 @@ class Config:
    db_path: str = os.getenv("DB_PATH", "data/tracker.db")
    headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
    chrome_binary: str | None = os.getenv("CHROME_BINARY")
    chromedriver_path: str = os.getenv("CHROMEDRIVER_PATH", "drivers/chromedriver")
    # Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
    phone_numbers: list[str] = field(default_factory=list)
@@ -0,0 +1,117 @@
 """Batch runner — spawn N parallel Chrome workers to execute the active flow config."""
 from __future__ import annotations
 import asyncio
 import threading
 import time
 import uuid
 from concurrent.futures import Future, ThreadPoolExecutor, as_completed
 from typing import Any, Callable
 from logger import get_logger
 log = get_logger(__name__)
 def _single_run(
    run_index: int,
    batch_id: str,
    cfg: dict[str, Any],
    headless: bool,
 ) -> tuple[bool, str | None]:
    """One isolated Chrome session. Returns (success, failure_reason)."""
    from crawler.driver import driver_session
    from crawler.dynamic_flow import DynamicFlow
    from crawler.scenarios.fresh_session import FreshSessionScenario
    from db.repositories.runs import runs_repo
    identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
    t0 = time.monotonic()
    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
    try:
        target_url: str = cfg.get("target_url") or ""
        with driver_session(headless=headless) as driver:
            FreshSessionScenario(driver, target_url).run()
            result = DynamicFlow(driver, cfg).run() if cfg else None
        total_ms = int((time.monotonic() - t0) * 1000)
        if result:
            steps_data = [
                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
                for s in result.steps
            ]
            asyncio.run(runs_repo.insert(
                "batch", identifier, result.success,
                result.failure_reason, steps_data, total_ms,
            ))
            log.info("[batch:%s] Run %d done — success=%s (%d ms)", batch_id[:8], run_index, result.success, total_ms)
            return result.success, result.failure_reason
        else:
            asyncio.run(runs_repo.insert("batch", identifier, True, None, [], total_ms))
            log.info("[batch:%s] Run %d done (no flow config) — %d ms", batch_id[:8], run_index, total_ms)
            return True, None
    except Exception as exc:
        total_ms = int((time.monotonic() - t0) * 1000)
        asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
        log.error("[batch:%s] Run %d failed — %s (%d ms)", batch_id[:8], run_index, exc, total_ms)
        return False, str(exc)
 def run_batch(
    cfg: dict[str, Any],
    workers: int,
    total_runs: int,
    headless: bool = True,
    stagger_ms: int = 0,
    on_progress: Callable[[int, int, int], None] | None = None,
 ) -> dict[str, Any]:
    """
    Run `total_runs` crawler sessions with up to `workers` parallel Chrome processes.
    `stagger_ms` delays each successive submission to the pool, spreading
    Chrome startup load and reducing bot-detection fingerprint clustering.
    `on_progress(completed, succeeded, failed)` is called after every finished run.
    Returns {"batch_id", "total", "succeeded", "failed"}.
    """
    batch_id = uuid.uuid4().hex
    log.info(
        "Batch %s — %d run(s), %d worker(s), stagger=%d ms, headless=%s",
        batch_id[:8], total_runs, workers, stagger_ms, headless,
    )
    succeeded = 0
    failed = 0
    lock = threading.Lock()
    with ThreadPoolExecutor(max_workers=workers, thread_name_prefix=f"batch-{batch_id[:8]}") as pool:
        futures: list[Future[tuple[bool, str | None]]] = []
        for i in range(total_runs):
            futures.append(pool.submit(_single_run, i, batch_id, cfg, headless))
            if stagger_ms > 0 and i < total_runs - 1:
                time.sleep(stagger_ms / 1000)
        for fut in as_completed(futures):
            try:
                success, _ = fut.result()
            except Exception as exc:
                log.error("[batch:%s] Unhandled thread exception: %s", batch_id[:8], exc)
                success = False
            with lock:
                if success:
                    succeeded += 1
                else:
                    failed += 1
                completed = succeeded + failed
            if on_progress:
                on_progress(completed, succeeded, failed)
    log.info(
        "Batch %s finished — %d/%d succeeded, %d failed",
        batch_id[:8], succeeded, total_runs, failed,
    )
    return {"batch_id": batch_id, "total": total_runs, "succeeded": succeeded, "failed": failed}
@@ -1,9 +1,14 @@
 """Stealth browser factory — bypasses Cloudflare/ArvanCloud bot detection."""
 from __future__ import annotations
 import os
 import random
 import shutil
 import subprocess
 import sys
 import time
 from contextlib import contextmanager
 from pathlib import Path
 from typing import Generator
 import undetected_chromedriver as uc
@@ -11,6 +16,16 @@ from selenium.webdriver.chrome.options import Options
 from selenium_stealth import stealth
 from config import config
 from logger import get_logger
 log = get_logger(__name__)
 # Selenium talks to ChromeDriver over localhost — must not go through any system proxy.
 for _var in ("http_proxy", "https_proxy", "HTTP_PROXY", "HTTPS_PROXY"):
    os.environ.pop(_var, None)
 _no_proxy = "localhost,127.0.0.1,::1"
 os.environ.setdefault("no_proxy", _no_proxy)
 os.environ.setdefault("NO_PROXY", _no_proxy)
 _USER_AGENTS = [
    "Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
@@ -18,31 +33,79 @@ _USER_AGENTS = [
    "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36",
 ]
 _CHROME_CANDIDATES = [
    "google-chrome-stable",
    "google-chrome",
    "chromium",
    "chromium-browser",
 ]
-def _build_options(user_agent: str, headless: bool) -> Options:
+
 def _detect_chrome() -> tuple[str, int]:
    """Return (binary_path, major_version) for the first Chrome found."""
    candidates = list(_CHROME_CANDIDATES)
    if config.chrome_binary:
        candidates.insert(0, config.chrome_binary)
    for binary in candidates:
        resolved = shutil.which(binary) or (binary if Path(binary).exists() else None)
        if not resolved:
            continue
        try:
            out = subprocess.check_output(
                [resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
            ).strip()
            major = int(out.split()[-1].split(".")[0])
            log.debug("Detected Chrome %d via '%s'", major, resolved)
            return resolved, major
        except Exception:
            continue
    log.critical("No Chrome binary found. Install Chrome or set CHROME_BINARY in .env")
    sys.exit(1)
 def _local_driver_path() -> str | None:
    path = Path(config.chromedriver_path)
    if path.exists():
        return str(path)
    log.warning(
        "No patched ChromeDriver at '%s'. Run 'make patch-driver' once. Falling back to auto-download.",
        path,
    )
    return None
 def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
    opts = Options()
    opts.binary_location = chrome_binary
    if headless:
        opts.add_argument("--headless=new")
    opts.add_argument(f"--user-agent={user_agent}")
    opts.add_argument("--no-sandbox")
    opts.add_argument("--disable-dev-shm-usage")
    opts.add_argument("--disable-gpu")
    opts.add_argument("--disable-blink-features=AutomationControlled")
    opts.add_argument("--disable-infobars")
    opts.add_argument("--window-size=1920,1080")
    opts.add_experimental_option("excludeSwitches", ["enable-automation"])
    opts.add_experimental_option("useAutomationExtension", False)
    if config.chrome_binary:
        opts.binary_location = config.chrome_binary
    return opts
 def make_driver(headless: bool | None = None) -> uc.Chrome:
    """Return a stealthed undetected Chrome instance."""
    use_headless = config.headless if headless is None else headless
    chrome_binary, major = _detect_chrome()
    ua = random.choice(_USER_AGENTS)
-    opts = _build_options(ua, use_headless)
+    opts = _build_options(ua, use_headless, chrome_binary)
-    driver = uc.Chrome(options=opts, use_subprocess=True)
+    log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
    driver = uc.Chrome(
        options=opts,
        driver_executable_path=_local_driver_path(),
        version_main=major,
        use_subprocess=True,
    )
    stealth(
        driver,
@@ -54,7 +117,6 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
        fix_hairline=True,
    )
    # Mask navigator.webdriver via CDP
    driver.execute_cdp_cmd(
        "Page.addScriptToEvaluateOnNewDocument",
        {
@@ -65,19 +127,19 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
            """
        },
    )
    log.info("Driver ready (Chrome %d)", major)
    return driver
@contextmanager
 def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
    """Context manager that guarantees driver.quit() on exit."""
    driver = make_driver(headless)
    try:
        yield driver
    finally:
        driver.quit()
        log.debug("Driver session closed")
 def human_delay(lo: float = 0.8, hi: float = 2.5) -> None:
    """Sleep for a random human-like interval."""
    time.sleep(random.uniform(lo, hi))
@@ -4,15 +4,17 @@ Dynamic flow executor — reads the active flow config from DB at runtime and ru
  2. scroll_step  : infinite-scroll until target item is visible or max_scrolls reached
  3. click_step   : click each target item id
 """
 from __future__ import annotations
-import asyncio
+import math
 import random
 import time
 from dataclasses import dataclass, field
 from typing import Any
 import undetected_chromedriver as uc
-from selenium.common.exceptions import NoSuchElementException, TimeoutException
+from selenium.common.exceptions import NoSuchElementException
 from selenium.webdriver.common.action_chains import ActionChains
 from selenium.webdriver.common.by import By
 from selenium.webdriver.common.keys import Keys
@@ -21,6 +23,9 @@ from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 from crawler.driver import human_delay
 from logger import get_logger
 log = get_logger(__name__)
 class DynamicFlowError(Exception):
@@ -71,10 +76,30 @@ def _wait_for(
    selector: str,
    selector_type: str,
    timeout: float = 15.0,
    label: str = "",
 ) -> WebElement:
-    return WebDriverWait(driver, timeout).until(
+    log.debug(
-        EC.presence_of_element_located((_by(selector_type), selector))
+        "Waiting for %s [%s:%s] (timeout=%ss)",
        label or "element",
        selector_type,
        selector,
        timeout,
    )
    try:
        el = WebDriverWait(driver, timeout).until(
            EC.presence_of_element_located((_by(selector_type), selector))
        )
        log.debug("Found %s [%s:%s]", label or "element", selector_type, selector)
        return el
    except Exception as exc:
        log.warning(
            "Selector not found — %s [%s:%s]: %s",
            label or "element",
            selector_type,
            selector,
            exc,
        )
        raise
 class DynamicFlow:
@@ -87,6 +112,7 @@ class DynamicFlow:
            config_id=self.cfg["id"],
            config_name=self.cfg["name"],
        )
        log.info("Starting flow '%s' (id=%s)", self.cfg["name"], self.cfg["id"])
        search_texts: list[str] = self.cfg.get("search_texts_json") or []
        item_ids: list[str] = self.cfg.get("target_item_ids_json") or []
@@ -95,6 +121,7 @@ class DynamicFlow:
            step = self._run_search(text)
            result.steps.append(step)
            if not step.success:
                log.error("Search step failed for '%s' — aborting flow", text)
                return result
            human_delay(0.8, 1.5)
@@ -102,12 +129,21 @@ class DynamicFlow:
                scroll_step = self._run_scroll_and_find(item_id)
                result.steps.append(scroll_step)
                if not scroll_step.success:
-                    continue  # try next item_id, don't abort the whole flow
+                    log.warning(
                        "Scroll/find failed for item '%s' — skipping click", item_id
                    )
                    continue
                click_step = self._run_click(item_id, scroll_step.data.get("element"))
                result.steps.append(click_step)
                human_delay(1.0, 2.5)
        log.info(
            "Flow '%s' finished — success=%s, steps=%d",
            self.cfg["name"],
            result.success,
            len(result.steps),
        )
        return result
    # ------------------------------------------------------------------
@@ -116,44 +152,70 @@ class DynamicFlow:
    def _run_search(self, text: str) -> StepResult:
        t0 = time.monotonic()
        name = f"search:{text}"
        log.info("[%s] Searching for '%s'", name, text)
        try:
            sel = self.cfg["search_box_selector"]
            sel_type = self.cfg["search_box_selector_type"]
            if not sel:
                raise DynamicFlowError("search_box_selector is not configured")
-            box = _wait_for(self.driver, sel, sel_type)
+            box = _wait_for(self.driver, sel, sel_type, label="search box")
            box.clear()
            human_delay(0.3, 0.6)
            # Type character by character for a human feel
            for ch in text:
                box.send_keys(ch)
                time.sleep(0.04)
            log.debug("[%s] Typed search text", name)
            human_delay(0.4, 0.8)
            # Submit: explicit button or Enter
            submit_sel = self.cfg.get("search_submit_selector", "")
            if submit_sel:
                submit_type = self.cfg.get("search_submit_selector_type", "css")
-                btn = _wait_for(self.driver, submit_sel, submit_type, timeout=5.0)
+                log.debug(
                    "[%s] Clicking submit button [%s:%s]", name, submit_type, submit_sel
                )
                btn = _wait_for(
                    self.driver,
                    submit_sel,
                    submit_type,
                    timeout=5.0,
                    label="submit button",
                )
                btn.click()
            else:
                log.debug("[%s] Submitting via Enter key", name)
                box.send_keys(Keys.RETURN)
            # Wait for results container if configured
            results_sel = self.cfg.get("search_results_selector", "")
            results_sel_type = self.cfg.get("search_results_selector_type", "css")
            if results_sel:
-                _wait_for(self.driver, results_sel, "css", timeout=15.0)
+                log.debug(
                    "[%s] Waiting for results container [%s:%s]",
                    name,
                    results_sel_type,
                    results_sel,
                )
                _wait_for(
                    self.driver,
                    results_sel,
                    results_sel_type,
                    timeout=15.0,
                    label="results container",
                )
            human_delay(1.0, 2.0)
-            return StepResult(name=name, success=True, data={"text": text},
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.info("[%s] Search OK — %d ms", name, ms)
            return StepResult(
                name=name, success=True, data={"text": text}, duration_ms=ms
            )
        except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Failed — %s", name, exc)
            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
    # ------------------------------------------------------------------
    # Step: scroll (infinite-scroll) until item selector matches
@@ -161,56 +223,270 @@ class DynamicFlow:
    def _run_scroll_and_find(self, item_id: str) -> StepResult:
        t0 = time.monotonic()
        name = f"scroll_find:{item_id}"
        log.info("[%s] Scrolling to find item '%s'", name, item_id)
        try:
-            item_sel = self._build_item_selector(item_id)
+            item_sel, item_sel_type = self._build_item_selector(item_id)
            item_sel_type = self.cfg.get("item_selector_type", "css")
            scroll_container = self.cfg.get("scroll_container_selector", "").strip()
            scroll_container_type = self.cfg.get(
                "scroll_container_selector_type", "css"
            )
            max_scrolls: int = int(self.cfg.get("max_scrolls", 30))
            pause_ms: int = int(self.cfg.get("scroll_pause_ms", 1200))
            no_new_ms: int = int(self.cfg.get("no_new_content_timeout_ms", 3000))
            pagination_sel: str = self.cfg.get("pagination_selector", "").strip()
            pagination_sel_type: str = self.cfg.get("pagination_selector_type", "css")
            max_pages: int = int(self.cfg.get("max_pages", 10))
            pagination_wait_ms: int = int(self.cfg.get("pagination_wait_ms", 1500))
            log.debug(
                "[%s] Selector [%s:%s], container=[%s:'%s'], max_scrolls=%d, pagination=%s",
                name,
                item_sel_type,
                item_sel,
                scroll_container_type,
                scroll_container or "window",
                max_scrolls,
                (
                    f"[{pagination_sel_type}:{pagination_sel}] max_pages={max_pages}"
                    if pagination_sel
                    else "off"
                ),
            )
            # Check if already visible before scrolling
            found = self._find_item_visible(item_sel, item_sel_type)
            if found:
-                return StepResult(name=name, success=True,
+                log.info("[%s] Item visible immediately (no scroll needed)", name)
-                                  data={"item_id": item_id, "scrolls": 0, "element": found},
+                return StepResult(
-                                  duration_ms=int((time.monotonic() - t0) * 1000))
+                    name=name,
                    success=True,
                    data={
                        "item_id": item_id,
                        "scrolls": 0,
                        "pages": 0,
                        "element": found,
                    },
                    duration_ms=int((time.monotonic() - t0) * 1000),
                )
-            last_height = self._get_scroll_height(scroll_container)
+            last_height = self._get_scroll_height(
                scroll_container, scroll_container_type
            )
            stale_since: float | None = None
            total_scrolls = 0
            page_num = 0
            scroll_on_page = 0
-            for scroll_num in range(1, max_scrolls + 1):
+            while scroll_on_page < max_scrolls:
-                self._scroll_down(scroll_container)
+                # Check for the item BEFORE scrolling so items near the top of a
-                time.sleep(pause_ms / 1000)
+                # freshly-paginated page are seen before we jump to the bottom.
                found = self._find_item_visible(item_sel, item_sel_type)
                if found:
-                    return StepResult(name=name, success=True,
+                    ms = int((time.monotonic() - t0) * 1000)
-                                      data={"item_id": item_id, "scrolls": scroll_num, "element": found},
+                    log.info(
-                                      duration_ms=int((time.monotonic() - t0) * 1000))
+                        "[%s] Found after %d scroll(s) on page %d — %d ms",
                        name,
                        total_scrolls,
                        page_num + 1,
                        ms,
                    )
                    return StepResult(
                        name=name,
                        success=True,
                        data={
                            "item_id": item_id,
                            "scrolls": total_scrolls,
                            "pages": page_num,
                            "element": found,
                        },
                        duration_ms=ms,
                    )
-                new_height = self._get_scroll_height(scroll_container)
+                scroll_on_page += 1
                total_scrolls += 1
                self._scroll_down(scroll_container, scroll_container_type)
                time.sleep(pause_ms / 1000)
                new_height = self._get_scroll_height(
                    scroll_container, scroll_container_type
                )
                if new_height == last_height:
                    if stale_since is None:
                        stale_since = time.monotonic()
-                    elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
+                        log.debug(
-                        raise DynamicFlowError(
+                            "[%s] Page height unchanged at scroll %d — starting stale timer",
-                            f"No new content after {no_new_ms} ms — reached end of page "
+                            name,
-                            f"without finding item '{item_id}'"
+                            scroll_on_page,
                        )
                    elif (time.monotonic() - stale_since) * 1000 >= no_new_ms:
                        # Infinite scroll exhausted — try pagination if configured
                        if pagination_sel and page_num < max_pages:
                            if self._try_paginate(
                                name,
                                pagination_sel,
                                pagination_sel_type,
                                pagination_wait_ms,
                            ):
                                page_num += 1
                                scroll_on_page = 0
                                stale_since = None
                                last_height = self._get_scroll_height(
                                    scroll_container, scroll_container_type
                                )
                                log.info(
                                    "[%s] Advanced to pagination page %d",
                                    name,
                                    page_num + 1,
                                )
                                # Loop continues — item check happens at top of next iteration
                            else:
                                raise DynamicFlowError(
                                    f"Pagination button [{pagination_sel_type}:{pagination_sel}] "
                                    f"not visible on page {page_num + 1}"
                                )
                        elif pagination_sel and page_num >= max_pages:
                            raise DynamicFlowError(
                                f"Item '{item_id}' not found after {page_num} pagination page(s)"
                            )
                        else:
                            raise DynamicFlowError(
                                f"No new content after {no_new_ms} ms — reached end of page "
                                f"without finding item '{item_id}'"
                            )
                else:
                    stale_since = None
                    last_height = new_height
                    log.debug(
                        "[%s] Scroll %d (page %d) — new height %d px",
                        name,
                        scroll_on_page,
                        page_num + 1,
                        new_height,
                    )
            raise DynamicFlowError(
-                f"Item '{item_id}' not found after {max_scrolls} scrolls"
+                f"Item '{item_id}' not found after {total_scrolls} scrolls"
            )
-        except DynamicFlowError:
+        except DynamicFlowError as exc:
-            raise
+            ms = int((time.monotonic() - t0) * 1000)
            log.error("[%s] Failed — %s", name, exc)
            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
        except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Unexpected error — %s", name, exc)
            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
    # ------------------------------------------------------------------
    # Human mouse simulation
    # ------------------------------------------------------------------
    def _human_click(self, el: WebElement, name: str = "") -> None:
        """
        Simulate a human-like mouse approach and click:
          1. Smooth scroll element into viewport
          2. Dispatch JS mousemove events along a curved path
          3. Trigger mouseenter / mouseover on the element
          4. Click via ActionChains with random sub-element offset
          5. Fallback: synthetic JS mouse event sequence
        """
        # Step 1 — bring element into view smoothly
        self.driver.execute_script(
            "arguments[0].scrollIntoView({block:'center', behavior:'smooth'});", el
        )
        time.sleep(random.uniform(0.35, 0.7))
        # Step 2 — get element centre in viewport coordinates
        rect: dict[str, float] = self.driver.execute_script(
            """
            const r = arguments[0].getBoundingClientRect();
            return {x: r.left, y: r.top, w: r.width, h: r.height,
                    vw: window.innerWidth, vh: window.innerHeight};
            """,
            el,
        )
        cx = rect["x"] + rect["w"] / 2
        cy = rect["y"] + rect["h"] / 2
        vw = rect["vw"]
        vh = rect["vh"]
        # Step 3 — build a curved path from a random off-element start
        sx = random.uniform(vw * 0.1, vw * 0.5)
        sy = random.uniform(vh * 0.1, vh * 0.5)
        # Control point for quadratic Bézier — pulled perpendicular to the straight line
        dx, dy = cx - sx, cy - sy
        perp_x = -dy * random.uniform(0.15, 0.4)
        perp_y = dx * random.uniform(0.15, 0.4)
        qx = sx + dx / 2 + perp_x
        qy = sy + dy / 2 + perp_y
        steps = random.randint(10, 18)
        prev_mx, prev_my = sx, sy
        for i in range(1, steps + 1):
            t = i / steps
            # Quadratic Bézier point
            bx = (1 - t) ** 2 * sx + 2 * (1 - t) * t * qx + t**2 * cx
            by = (1 - t) ** 2 * sy + 2 * (1 - t) * t * qy + t**2 * cy
            # Add small Gaussian jitter that fades near the target
            jitter = max(0.0, 1.0 - t) * 4.0
            mx = bx + random.gauss(0, jitter)
            my = by + random.gauss(0, jitter)
            self.driver.execute_script(
                """
                document.dispatchEvent(new MouseEvent('mousemove', {
                    bubbles: true, cancelable: true,
                    clientX: arguments[0], clientY: arguments[1]
                }));
                """,
                int(mx),
                int(my),
            )
            # Variable inter-step pause — faster in the middle, slower near target
            speed = 0.5 + 0.5 * math.sin(math.pi * t)
            time.sleep(random.uniform(0.008, 0.025) / max(speed, 0.3))
            prev_mx, prev_my = mx, my
        # Step 4 — hover events on the element itself
        self.driver.execute_script(
            """
            arguments[0].dispatchEvent(new MouseEvent('mouseenter', {bubbles: true}));
            arguments[0].dispatchEvent(new MouseEvent('mouseover',  {bubbles: true}));
            """,
            el,
        )
        time.sleep(random.uniform(0.1, 0.28))
        # Step 5 — ActionChains click with random offset from centre
        off_x = int(random.uniform(-rect["w"] * 0.28, rect["w"] * 0.28))
        off_y = int(random.uniform(-rect["h"] * 0.28, rect["h"] * 0.28))
        log.debug(
            "[%s] Human click — target=(%.0f,%.0f) offset=(%d,%d)",
            name,
            cx,
            cy,
            off_x,
            off_y,
        )
        try:
            ActionChains(self.driver).move_to_element_with_offset(
                el, off_x, off_y
            ).pause(random.uniform(0.05, 0.14)).click().perform()
            log.debug("[%s] ActionChains click OK", name)
        except Exception as ac_exc:
            log.warning(
                "[%s] ActionChains failed (%s) — synthetic JS click", name, ac_exc
            )
            self.driver.execute_script(
                """
                const opts = {bubbles: true, cancelable: true,
                              clientX: arguments[1], clientY: arguments[2]};
                arguments[0].dispatchEvent(new MouseEvent('mousedown', opts));
                arguments[0].dispatchEvent(new MouseEvent('mouseup',   opts));
                arguments[0].dispatchEvent(new MouseEvent('click',     opts));
                """,
                el,
                int(cx + off_x),
                int(cy + off_y),
            )
    # ------------------------------------------------------------------
    # Step: click the found element
@@ -218,65 +494,205 @@ class DynamicFlow:
    def _run_click(self, item_id: str, element: Any) -> StepResult:
        t0 = time.monotonic()
        name = f"click:{item_id}"
        log.info("[%s] Clicking item '%s'", name, item_id)
        try:
            if element is None:
                raise DynamicFlowError("No element reference from scroll step")
-            el: WebElement = element
+            self._human_click(element, name)
            # Scroll element into view and click
            self.driver.execute_script("arguments[0].scrollIntoView({block:'center'});", el)
            human_delay(0.3, 0.7)
            try:
                el.click()
            except Exception:
                # Fallback: JS click
                self.driver.execute_script("arguments[0].click();", el)
            human_delay(0.8, 1.5)
-            return StepResult(name=name, success=True,
+            ms = int((time.monotonic() - t0) * 1000)
-                              data={"item_id": item_id, "url_after": self.driver.current_url},
+            log.info(
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+                "[%s] Click OK — landed on %s — %d ms",
                name,
                self.driver.current_url,
                ms,
            )
            return StepResult(
                name=name,
                success=True,
                data={"item_id": item_id, "url_after": self.driver.current_url},
                duration_ms=ms,
            )
        except Exception as exc:
-            return StepResult(name=name, success=False, error=str(exc),
+            ms = int((time.monotonic() - t0) * 1000)
-                              duration_ms=int((time.monotonic() - t0) * 1000))
+            log.error("[%s] Failed — %s", name, exc)
            return StepResult(name=name, success=False, error=str(exc), duration_ms=ms)
    # ------------------------------------------------------------------
    # Pagination helper
    # ------------------------------------------------------------------
    def _try_paginate(
        self, step_name: str, sel: str, sel_type: str, wait_ms: int
    ) -> bool:
        """
        Find the next-page button and click it reliably.
        Uses a layered approach rather than full human-mouse simulation because
        pagination buttons are small and offset-based clicking is fragile:
          1. Native Selenium click (triggers real browser events)
          2. JS click() on the element
          3. Synthetic pointer + mouse + click event dispatch
        """
        log.debug("[%s] Looking for pagination button [%s:%s]", step_name, sel_type, sel)
        # Search all matching elements, not just the first visible one,
        # in case the button is present but momentarily off-screen.
        try:
            candidates = _find_all(self.driver, sel, sel_type)
        except Exception as exc:
            log.warning("[%s] Pagination selector failed — %s", step_name, exc)
            return False
        btn: WebElement | None = None
        for el in candidates:
            try:
                if el.is_displayed() and el.is_enabled():
                    btn = el
                    break
            except Exception:
                continue
        if btn is None:
            log.debug("[%s] Pagination button not found / not interactable", step_name)
            return False
        log.debug("[%s] Pagination button found — attempting click", step_name)
        try:
            self.driver.execute_script(
                "arguments[0].scrollIntoView({block:'nearest', behavior:'instant'});", btn
            )
            time.sleep(random.uniform(0.15, 0.35))
            # Layer 1: native Selenium click
            try:
                btn.click()
                log.debug("[%s] Pagination: native click OK", step_name)
            except Exception as e1:
                log.debug("[%s] Pagination: native click failed (%s) — trying JS", step_name, e1)
                # Layer 2: JS .click()
                try:
                    self.driver.execute_script("arguments[0].click();", btn)
                    log.debug("[%s] Pagination: JS click OK", step_name)
                except Exception as e2:
                    log.debug("[%s] Pagination: JS click failed (%s) — dispatching events", step_name, e2)
                    # Layer 3: synthetic pointer + mouse + click events
                    self.driver.execute_script(
                        """
                        const el = arguments[0];
                        const r = el.getBoundingClientRect();
                        const cx = r.left + r.width / 2, cy = r.top + r.height / 2;
                        const opts = {bubbles: true, cancelable: true, clientX: cx, clientY: cy};
                        ['pointerdown','mousedown','pointerup','mouseup','click']
                            .forEach(t => el.dispatchEvent(new (t.startsWith('pointer') ? PointerEvent : MouseEvent)(t, opts)));
                        """,
                        btn,
                    )
                    log.debug("[%s] Pagination: synthetic events dispatched", step_name)
            log.info("[%s] Pagination button clicked — waiting %d ms", step_name, wait_ms)
            time.sleep(wait_ms / 1000)
            return True
        except Exception as exc:
            log.warning("[%s] Pagination click failed — %s", step_name, exc)
            return False
    # ------------------------------------------------------------------
    # Helpers
    # ------------------------------------------------------------------
-    def _build_item_selector(self, item_id: str) -> str:
+    def _build_item_selector(self, item_id: str) -> tuple[str, str]:
        """
        Return (selector, effective_type).
        Priority:
          1. item_url_template — renders the URL pattern and uses XPath
             contains(@href, ...) to find the matching <a> element.
          2. item_selector_type == "a-link" — same href-contains approach but
             driven by item_selector_template instead.
          3. CSS / XPath — item_selector_template used verbatim.
        Href-based searches always use XPath so that arbitrary URL characters
        (colons, brackets, dots, slashes) never break the selector.
        """
        url_tpl: str = self.cfg.get("item_url_template", "").strip()
        if url_tpl:
            href = url_tpl.replace("{item_id}", item_id)
            log.debug("item_url_template resolved to href pattern '%s'", href)
            return f"//a[contains(@href, '{href}')]", "xpath"
        sel_type: str = self.cfg.get("item_selector_type", "css")
        template: str = self.cfg.get("item_selector_template", "")
        if sel_type == "a-link":
            href_fragment = (
                template.replace("{item_id}", item_id) if template else item_id
            )
            return f"//a[contains(@href, '{href_fragment}')]", "xpath"
        if not template:
            raise DynamicFlowError("item_selector_template is not configured")
-        return template.replace("{item_id}", item_id)
+        return template.replace("{item_id}", item_id), sel_type
-    def _find_item_visible(self, selector: str, selector_type: str) -> WebElement | None:
+    def _find_item_visible(
        self, selector: str, selector_type: str
    ) -> WebElement | None:
        try:
            els = _find_all(self.driver, selector, selector_type)
-            for el in els:
+        except Exception as exc:
            log.debug("Selector error [%s:%s]: %s", selector_type, selector, exc)
            return None
        for el in els:
            try:
                if el.is_displayed():
                    return el
-        except NoSuchElementException:
+            except Exception:
-            pass
+                continue
        return None
-    def _get_scroll_height(self, container_sel: str) -> int:
+    def _get_scroll_height(
        self, container_sel: str, container_type: str = "css"
    ) -> int:
        body_h = int(self.driver.execute_script(
            "return Math.max(document.body.scrollHeight, document.documentElement.scrollHeight)"
        ))
        if container_sel:
            try:
-                el = _find(self.driver, container_sel, "css")
+                el = _find(self.driver, container_sel, container_type)
-                return int(self.driver.execute_script("return arguments[0].scrollHeight", el))
+                container_h = int(self.driver.execute_script("return arguments[0].scrollHeight", el))
-            except Exception:
+                return max(body_h, container_h)
-                pass
+            except Exception as exc:
-        return int(self.driver.execute_script("return document.body.scrollHeight"))
+                log.warning(
-
+                    "Could not read scrollHeight from container [%s:'%s']: %s — using body",
-    def _scroll_down(self, container_sel: str) -> None:
+                    container_type, container_sel, exc,
        if container_sel:
            try:
                el = _find(self.driver, container_sel, "css")
                self.driver.execute_script(
                    "arguments[0].scrollTop = arguments[0].scrollHeight", el
                )
-                return
+        return body_h
-            except Exception:
+
-                pass
+    def _scroll_down(self, container_sel: str, container_type: str = "css") -> None:
-        self.driver.execute_script("window.scrollTo(0, document.body.scrollHeight)")
+        if container_sel:
            try:
                el = _find(self.driver, container_sel, container_type)
                self.driver.execute_script(
                    """
                    const el = arguments[0];
                    el.scrollTop = el.scrollHeight;
                    el.dispatchEvent(new Event('scroll', {bubbles: true}));
                    """,
                    el,
                )
            except Exception as exc:
                log.warning(
                    "Could not scroll container [%s:'%s']: %s — falling through to window",
                    container_type, container_sel, exc,
                )
        # Always advance the window too — most infinite-scroll triggers
        # listen to window scroll events, not container scroll events.
        self.driver.execute_script(
            """
            window.scrollTo(0, document.body.scrollHeight);
            window.dispatchEvent(new Event('scroll', {bubbles: true}));
            document.dispatchEvent(new Event('scroll', {bubbles: true}));
            """
        )
@@ -2,11 +2,9 @@
 from __future__ import annotations
 import undetected_chromedriver as uc
 from selenium.common.exceptions import TimeoutException
 from selenium.webdriver.support.ui import WebDriverWait
-from config import config
+from crawler.driver import human_delay
 from crawler.driver import human_delay, make_driver
 class FreshSessionScenario:
@@ -19,8 +17,9 @@ class FreshSessionScenario:
    explicitly delete all cookies after loading to be safe.
    """
-    def __init__(self, driver: uc.Chrome, timeout: int = 30) -> None:
+    def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
        self.driver = driver
        self.url = url
        self.wait = WebDriverWait(driver, timeout)
    def run(self) -> dict[str, object]:
@@ -32,7 +31,7 @@ class FreshSessionScenario:
            "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
        )
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
        human_delay(2.0, 4.0)
        # Wait for the page to reach a ready state
@@ -10,7 +10,6 @@ from selenium.webdriver.common.by import By
 from selenium.webdriver.support import expected_conditions as EC
 from selenium.webdriver.support.ui import WebDriverWait
 from config import config
 from crawler.driver import human_delay
@@ -32,11 +31,13 @@ class OTPLoginScenario:
        driver: uc.Chrome,
        phone: str,
        otp_resolver: Callable[[str], str],
        url: str,
        timeout: int = 30,
    ) -> None:
        self.driver = driver
        self.phone = phone
        self.otp_resolver = otp_resolver
        self.url = url
        self.wait = WebDriverWait(driver, timeout)
    # ------------------------------------------------------------------
@@ -50,7 +51,7 @@ class OTPLoginScenario:
    def run(self) -> dict[str, object]:
        """Execute the OTP login. Returns a result dict."""
-        self.driver.get(config.target_url)
+        self.driver.get(self.url)
        human_delay(1.5, 3.0)
        try:
@@ -0,0 +1,214 @@
 """Huey task definitions and shared batch state for the admin panel."""
 from __future__ import annotations
 import threading
 import time
 from concurrent.futures import ThreadPoolExecutor, as_completed
 from typing import Any
 from huey import MemoryHuey
 huey = MemoryHuey("seed", immediate=False)
 # ── Shared state (all accessed under _lock) ───────────────────────────────────
 _lock = threading.Lock()
 _current_batch_id: str | None = None
 _current_result: Any = None          # Huey Result — kept for revocation
 _batch_meta: dict[str, Any] = {}     # batch_id → aggregate metadata
 _worker_slots: dict[str, dict] = {}  # f"{batch_id}:{slot:04d}" → worker info
 _stop_events: dict[str, threading.Event] = {}
 # ── Public helpers ─────────────────────────────────────────────────────────────
 def get_current_batch_id() -> str | None:
    with _lock:
        return _current_batch_id
 def set_current(batch_id: str, result: Any) -> None:
    global _current_batch_id, _current_result
    with _lock:
        _current_batch_id = batch_id
        _current_result = result
 def stop_current() -> bool:
    """Signal the running batch to stop. Returns True if there was something to stop."""
    with _lock:
        bid = _current_batch_id
        result = _current_result
    if bid is None:
        return False
    # Revoke task if it hasn't started yet (still pending in Huey queue)
    if result is not None:
        try:
            result.revoke(revoke_once=True)
        except Exception:
            pass
    # Signal running workers to stop after their current step
    ev = _stop_events.get(bid)
    if ev:
        ev.set()
    with _lock:
        if bid in _batch_meta:
            _batch_meta[bid]["stopping"] = True
    return True
 def batch_status() -> dict[str, Any]:
    with _lock:
        bid = _current_batch_id
        if bid is None:
            return {"batch_id": None, "running": False}
        meta = dict(_batch_meta.get(bid, {}))
        prefix = f"{bid}:"
        workers = [dict(v) for k, v in _worker_slots.items() if k.startswith(prefix)]
    meta["batch_id"] = bid
    meta["workers_detail"] = sorted(workers, key=lambda w: w.get("slot", 0))
    meta["active_workers"] = sum(1 for w in workers if w.get("status") == "running")
    return meta
 # ── Huey task ─────────────────────────────────────────────────────────────────
@huey.task(name="run_batch")
 def run_batch_task(
    batch_id: str,
    cfg: dict[str, Any],
    workers: int,
    total_runs: int,
    headless: bool,
    stagger_ms: int,
 ) -> dict[str, Any]:
    import asyncio
    from crawler.driver import driver_session
    from crawler.dynamic_flow import DynamicFlow
    from crawler.scenarios.fresh_session import FreshSessionScenario
    from db.repositories.runs import runs_repo
    from logger import get_logger
    log = get_logger(__name__)
    stop_ev = threading.Event()
    with _lock:
        _stop_events[batch_id] = stop_ev
        _batch_meta[batch_id] = {
            "running": True,
            "stopping": False,
            "total": total_runs,
            "completed": 0,
            "succeeded": 0,
            "failed": 0,
            "configured_workers": workers,
            "started_at": time.time(),
            "finished_at": None,
            "error": None,
        }
    def _run_one(slot: int) -> bool:
        slot_key = f"{batch_id}:{slot:04d}"
        if stop_ev.is_set():
            with _lock:
                _worker_slots[slot_key] = {
                    "slot": slot, "status": "skipped",
                    "started_at": None, "finished_at": None, "duration_ms": None, "error": "stopped",
                }
            return False
        with _lock:
            _worker_slots[slot_key] = {
                "slot": slot, "status": "running",
                "started_at": time.time(), "finished_at": None, "duration_ms": None, "error": None,
            }
        t0 = time.monotonic()
        identifier = f"batch-{batch_id[:8]}-{slot:04d}"
        log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
        try:
            target_url: str = cfg.get("target_url") or ""
            with driver_session(headless=headless) as driver:
                FreshSessionScenario(driver, target_url).run()
                result = DynamicFlow(driver, cfg).run() if cfg else None
            total_ms = int((time.monotonic() - t0) * 1000)
            success = result.success if result else True
            reason = result.failure_reason if result else None
            steps = [
                {"name": s.name, "success": s.success, "error": s.error, "duration_ms": s.duration_ms}
                for s in (result.steps if result else [])
            ]
            asyncio.run(runs_repo.insert("batch", identifier, success, reason, steps, total_ms))
            status = "ok" if success else "failed"
            with _lock:
                _worker_slots[slot_key].update({
                    "status": status, "finished_at": time.time(), "duration_ms": total_ms,
                })
            log.info("[batch:%s] slot %d %s — %d ms", batch_id[:8], slot, status, total_ms)
            return success
        except Exception as exc:
            total_ms = int((time.monotonic() - t0) * 1000)
            asyncio.run(runs_repo.insert("batch", identifier, False, str(exc), [], total_ms))
            with _lock:
                _worker_slots[slot_key].update({
                    "status": "error", "error": str(exc),
                    "finished_at": time.time(), "duration_ms": total_ms,
                })
            log.error("[batch:%s] slot %d error — %s", batch_id[:8], slot, exc)
            return False
    succeeded = 0
    failed = 0
    try:
        with ThreadPoolExecutor(
            max_workers=workers,
            thread_name_prefix=f"batch-{batch_id[:8]}",
        ) as pool:
            futures = []
            for i in range(total_runs):
                if stop_ev.is_set():
                    break
                futures.append(pool.submit(_run_one, i))
                if stagger_ms > 0 and i < total_runs - 1:
                    if stop_ev.wait(timeout=stagger_ms / 1000):
                        break  # stop was signalled during the stagger delay
            for fut in as_completed(futures):
                try:
                    ok = fut.result()
                except Exception:
                    ok = False
                if ok:
                    succeeded += 1
                else:
                    failed += 1
                with _lock:
                    _batch_meta[batch_id]["completed"] = succeeded + failed
                    _batch_meta[batch_id]["succeeded"] = succeeded
                    _batch_meta[batch_id]["failed"] = failed
    except Exception as exc:
        with _lock:
            _batch_meta[batch_id]["error"] = str(exc)
    finally:
        with _lock:
            _batch_meta[batch_id]["running"] = False
            _batch_meta[batch_id]["finished_at"] = time.time()
    return {"batch_id": batch_id, "succeeded": succeeded, "failed": failed}
@@ -37,14 +37,22 @@ CREATE TABLE IF NOT EXISTS flow_configs (
    search_box_selector_type    TEXT    NOT NULL DEFAULT 'css',
    search_submit_selector      TEXT    NOT NULL DEFAULT '',
    search_submit_selector_type TEXT    NOT NULL DEFAULT 'css',
-    search_results_selector     TEXT    NOT NULL DEFAULT '',
+    search_results_selector      TEXT    NOT NULL DEFAULT '',
-    scroll_container_selector   TEXT    NOT NULL DEFAULT '',
+    search_results_selector_type TEXT    NOT NULL DEFAULT 'css',
    scroll_container_selector      TEXT    NOT NULL DEFAULT '',
    scroll_container_selector_type TEXT    NOT NULL DEFAULT 'css',
    max_scrolls                 INTEGER NOT NULL DEFAULT 30,
    scroll_pause_ms             INTEGER NOT NULL DEFAULT 1200,
    no_new_content_timeout_ms   INTEGER NOT NULL DEFAULT 3000,
    pagination_selector         TEXT    NOT NULL DEFAULT '',
    pagination_selector_type    TEXT    NOT NULL DEFAULT 'css',
    max_pages                   INTEGER NOT NULL DEFAULT 10,
    pagination_wait_ms          INTEGER NOT NULL DEFAULT 1500,
    item_selector_template      TEXT    NOT NULL DEFAULT '',
    item_selector_type          TEXT    NOT NULL DEFAULT 'css',
-    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]'
+    item_url_template           TEXT    NOT NULL DEFAULT '',
    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]',
    target_url                  TEXT    NOT NULL DEFAULT ''
 );
 """
@@ -55,10 +63,29 @@ def init_db() -> None:
    path.parent.mkdir(parents=True, exist_ok=True)
    conn = sqlite3.connect(str(path))
    conn.executescript(_DDL)
    # Additive migrations for columns added after initial release
    _migrate(conn)
    conn.commit()
    conn.close()
 def _migrate(conn: sqlite3.Connection) -> None:
    existing = {row[1] for row in conn.execute("PRAGMA table_info(flow_configs)")}
    additions = {
        "search_results_selector_type": "TEXT NOT NULL DEFAULT 'css'",
        "item_url_template": "TEXT NOT NULL DEFAULT ''",
        "scroll_container_selector_type": "TEXT NOT NULL DEFAULT 'css'",
        "pagination_selector": "TEXT NOT NULL DEFAULT ''",
        "pagination_selector_type": "TEXT NOT NULL DEFAULT 'css'",
        "max_pages": "INTEGER NOT NULL DEFAULT 10",
        "pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
        "target_url": "TEXT NOT NULL DEFAULT ''",
    }
    for col, definition in additions.items():
        if col not in existing:
            conn.execute(f"ALTER TABLE flow_configs ADD COLUMN {col} {definition}")
@asynccontextmanager
 async def get_db() -> AsyncGenerator[aiosqlite.Connection, None]:
    """Async context manager — yields an open, Row-factory-enabled connection."""
@@ -11,9 +11,13 @@ FlowConfigRow = dict[str, Any]
 _FIELDS = [
    "name", "is_active",
    "search_texts_json", "search_box_selector", "search_box_selector_type",
-    "search_submit_selector", "search_submit_selector_type", "search_results_selector",
+    "search_submit_selector", "search_submit_selector_type",
-    "scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
+    "search_results_selector", "search_results_selector_type",
-    "item_selector_template", "item_selector_type", "target_item_ids_json",
+    "scroll_container_selector", "scroll_container_selector_type",
    "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
    "pagination_selector", "pagination_selector_type", "max_pages", "pagination_wait_ms",
    "item_selector_template", "item_selector_type", "item_url_template", "target_item_ids_json",
    "target_url",
 ]
@@ -0,0 +1,109 @@
 """Central logging configuration for the Seed project.
 Usage anywhere in the codebase:
    from logger import get_logger
    log = get_logger(__name__)
    log.info("message")
    log.warning("watch out")
    log.error("something broke")
 Environment variables:
    LOG_LEVEL   — DEBUG | INFO | WARNING | ERROR | CRITICAL  (default: INFO)
    LOG_FILE    — path to write a rotating log file (optional)
 """
 from __future__ import annotations
 import logging
 import logging.handlers
 import os
 import sys
 from typing import Final
 _LEVEL_MAP: Final = {
    "DEBUG": logging.DEBUG,
    "INFO": logging.INFO,
    "WARNING": logging.WARNING,
    "ERROR": logging.ERROR,
    "CRITICAL": logging.CRITICAL,
 }
 _LOG_LEVEL: int = _LEVEL_MAP.get(os.getenv("LOG_LEVEL", "INFO").upper(), logging.INFO)
 _LOG_FILE: str | None = os.getenv("LOG_FILE")
 # ── ANSI colour codes (console only) ─────────────────────────────────────────
 _RESET  = "\033[0m"
 _BOLD   = "\033[1m"
 _GREY   = "\033[90m"
 _CYAN   = "\033[96m"
 _YELLOW = "\033[93m"
 _RED    = "\033[91m"
 _BRED   = "\033[1;91m"
 _LEVEL_COLOURS: Final[dict[int, str]] = {
    logging.DEBUG:    _GREY,
    logging.INFO:     _CYAN,
    logging.WARNING:  _YELLOW,
    logging.ERROR:    _RED,
    logging.CRITICAL: _BRED,
 }
 class _ColourFormatter(logging.Formatter):
    _FMT = "{colour}{level:<8}{reset} {grey}{asctime}  {name}{reset}  {msg}"
    def format(self, record: logging.LogRecord) -> str:
        colour = _LEVEL_COLOURS.get(record.levelno, "")
        level  = record.levelname
        grey   = _GREY if sys.stderr.isatty() else ""
        reset  = _RESET if sys.stderr.isatty() else ""
        col    = colour if sys.stderr.isatty() else ""
        self.datefmt = "%H:%M:%S"
        base = super().format(record)
        return self._FMT.format(
            colour=col, level=level, reset=reset,
            grey=grey, asctime=self.formatTime(record, self.datefmt),
            name=record.name, msg=record.getMessage(),
        ) + (f"\n{record.exc_text}" if record.exc_info and self.formatException(record.exc_info) else "")
 _FILE_FMT = logging.Formatter(
    fmt="%(asctime)s  %(levelname)-8s  %(name)s  %(message)s",
    datefmt="%Y-%m-%dT%H:%M:%S",
 )
 def _build_root_logger() -> logging.Logger:
    root = logging.getLogger("seed")
    root.setLevel(_LOG_LEVEL)
    if root.handlers:
        return root  # already configured (e.g. imported twice)
    # Console handler
    console = logging.StreamHandler(sys.stderr)
    console.setLevel(_LOG_LEVEL)
    console.setFormatter(_ColourFormatter())
    root.addHandler(console)
    # Optional rotating file handler
    if _LOG_FILE:
        fh = logging.handlers.RotatingFileHandler(
            _LOG_FILE, maxBytes=10 * 1024 * 1024, backupCount=5, encoding="utf-8"
        )
        fh.setLevel(_LOG_LEVEL)
        fh.setFormatter(_FILE_FMT)
        root.addHandler(fh)
    root.propagate = False
    return root
 _root = _build_root_logger()
 def get_logger(name: str) -> logging.Logger:
    """Return a child logger namespaced under 'seed'."""
    if name.startswith("seed.") or name == "seed":
        return logging.getLogger(name)
    return logging.getLogger(f"seed.{name}")
@@ -8,15 +8,18 @@ import time
 import uvicorn
 from config import config
 from db import flow_config_repo, init_db
 from db.repositories.runs import runs_repo
-from config import config
+from logger import get_logger
 log = get_logger(__name__)
 def _load_active_flow() -> dict[str, object]:
    cfg = asyncio.run(flow_config_repo.get_active())
    if cfg is None:
-        print("[WARN] No active flow config — running without dynamic flow steps.", file=sys.stderr)
+        log.warning("No active flow config — running without dynamic flow steps.")
    return cfg or {}
@@ -32,7 +35,7 @@ def _record(
    total_ms = int((time.monotonic() - t0) * 1000)
    if exc is not None:
        asyncio.run(runs_repo.insert(scenario, identifier, False, str(exc), [], total_ms))
-        print(f"[FAIL] {identifier}: {exc}", file=sys.stderr)
+        log.error("%s failed: %s", identifier, exc)
        return
    if isinstance(flow_result, DynamicFlowResult):
@@ -44,12 +47,13 @@ def _record(
            scenario, identifier, flow_result.success,
            flow_result.failure_reason, steps_data, total_ms,
        ))
-        status = "OK" if flow_result.success else f"FAIL ({flow_result.failure_reason})"
+        if flow_result.success:
            log.info("%s OK — %d ms", identifier, total_ms)
        else:
            log.error("%s FAIL (%s) — %d ms", identifier, flow_result.failure_reason, total_ms)
    else:
        asyncio.run(runs_repo.insert(scenario, identifier, True, None, [], total_ms))
-        status = "OK (no flow config)"
+        log.info("%s OK (no flow config) — %d ms", identifier, total_ms)
    print(f"[{status}] {identifier} — {total_ms} ms")
 def run_otp(phone: str) -> None:
@@ -58,14 +62,14 @@ def run_otp(phone: str) -> None:
    from crawler.scenarios.otp_login import OTPLoginScenario
    def otp_resolver(p: str) -> str:
        # TODO: wire up your SMS gateway / webhook here.
        raise NotImplementedError("Implement otp_resolver to fetch OTP from your SMS gateway")
    flow_cfg = _load_active_flow()
    target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
    t0 = time.monotonic()
    try:
        with driver_session() as driver:
-            OTPLoginScenario(driver, phone, otp_resolver).run()
+            OTPLoginScenario(driver, phone, otp_resolver, target_url).run()
            result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
    except Exception as exc:
        _record("otp", phone, None, exc, t0)
@@ -79,10 +83,11 @@ def run_fresh() -> None:
    from crawler.scenarios.fresh_session import FreshSessionScenario
    flow_cfg = _load_active_flow()
    target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
    t0 = time.monotonic()
    try:
        with driver_session() as driver:
-            FreshSessionScenario(driver).run()
+            FreshSessionScenario(driver, target_url).run()
            result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
    except Exception as exc:
        _record("fresh", "fresh", None, exc, t0)
@@ -90,6 +95,34 @@ def run_fresh() -> None:
    _record("fresh", "fresh", result, None, t0)
 def run_batch(workers: int, total_runs: int, stagger_ms: int, headless: bool) -> None:
    from crawler.batch import run_batch as _run_batch
    flow_cfg = _load_active_flow()
    if not flow_cfg:
        log.error("No active flow config — activate one in the admin panel first.")
        sys.exit(1)
    completed_count = [0]
    def _progress(completed: int, succeeded: int, failed: int) -> None:
        completed_count[0] = completed
        log.info("Progress: %d/%d  (ok=%d  fail=%d)", completed, total_runs, succeeded, failed)
    result = _run_batch(
        cfg=flow_cfg,
        workers=workers,
        total_runs=total_runs,
        headless=headless,
        stagger_ms=stagger_ms,
        on_progress=_progress,
    )
    log.info(
        "Batch complete — succeeded=%d/%d  failed=%d",
        result["succeeded"], result["total"], result["failed"],
    )
 def run_admin() -> None:
    init_db()
    from admin.main import app
@@ -107,6 +140,12 @@ def main() -> None:
    sub.add_parser("fresh", help="Run Scenario 2 (fresh session)")
    batch_p = sub.add_parser("batch", help="Run many parallel fresh-session runners")
    batch_p.add_argument("--workers", type=int, default=3, help="Parallel Chrome instances (default: 3)")
    batch_p.add_argument("--runs", type=int, default=10, help="Total runs to complete (default: 10)")
    batch_p.add_argument("--stagger", type=int, default=500, help="Ms delay between worker launches (default: 500)")
    batch_p.add_argument("--no-headless", dest="headless", action="store_false", help="Show browser windows")
    args = parser.parse_args()
    if args.cmd == "admin":
@@ -114,12 +153,19 @@ def main() -> None:
    elif args.cmd == "otp":
        phones = [args.phone] if args.phone else config.phone_numbers
        if not phones:
-            print("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone", file=sys.stderr)
+            log.error("No phone numbers configured. Set PHONE_NUMBERS in .env or pass --phone")
            sys.exit(1)
        for phone in phones:
            run_otp(phone)
    elif args.cmd == "fresh":
        run_fresh()
    elif args.cmd == "batch":
        run_batch(
            workers=args.workers,
            total_runs=args.runs,
            stagger_ms=args.stagger,
            headless=args.headless,
        )
 if __name__ == "__main__":
@@ -14,6 +14,7 @@ dependencies = [
    "pydantic>=2.7.0",
    "python-dotenv>=1.0.1",
    "setuptools>=70.0.0",
    "huey>=2.5.0",
 ]
 [tool.pyright]
@@ -0,0 +1,106 @@
 """
 Download and patch ChromeDriver once.  Run via: make patch-driver
 Uses the official Chrome for Testing API to fetch the exact ChromeDriver
 version matching the installed Chrome, then patches it with undetected_chromedriver.
 """
 from __future__ import annotations
 import io
 import shutil
 import stat
 import subprocess
 import sys
 import urllib.request
 import zipfile
 from pathlib import Path
 import undetected_chromedriver as uc
 from config import config
 from logger import get_logger
 log = get_logger(__name__)
 def _get_chrome_full_version() -> tuple[int, str]:
    candidates = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
    if config.chrome_binary:
        candidates.insert(0, config.chrome_binary)
    for binary in candidates:
        if not shutil.which(binary) and not Path(binary).exists():
            continue
        try:
            out = subprocess.check_output(
                [binary, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
            ).strip()
            full = out.split()[-1]
            major = int(full.split(".")[0])
            log.info("Detected Chrome %s via '%s'", full, binary)
            return major, full
        except Exception:
            continue
    log.critical("Could not detect Chrome version. Set CHROME_BINARY in .env")
    sys.exit(1)
 def _latest_chromedriver_version(major: int) -> str:
    url = f"https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_{major}"
    try:
        with urllib.request.urlopen(url, timeout=15) as resp:
            version = resp.read().decode().strip()
            log.info("Latest ChromeDriver for Chrome %d: %s", major, version)
            return version
    except Exception as exc:
        log.critical("Could not fetch ChromeDriver version for Chrome %d: %s", major, exc)
        sys.exit(1)
 def _download_chromedriver(version: str, dest: Path) -> None:
    zip_url = (
        f"https://storage.googleapis.com/chrome-for-testing-public"
        f"/{version}/linux64/chromedriver-linux64.zip"
    )
    log.info("Downloading ChromeDriver %s …", version)
    try:
        with urllib.request.urlopen(zip_url, timeout=60) as resp:
            data = resp.read()
    except Exception as exc:
        log.critical("Download failed: %s", exc)
        sys.exit(1)
    with zipfile.ZipFile(io.BytesIO(data)) as zf:
        binary_name = next(
            n for n in zf.namelist() if n.endswith("/chromedriver") or n == "chromedriver"
        )
        dest.write_bytes(zf.read(binary_name))
    dest.chmod(dest.stat().st_mode | stat.S_IEXEC | stat.S_IXGRP | stat.S_IXOTH)
    log.info("Extracted to %s", dest)
 def _patch(dest: Path) -> None:
    log.info("Patching binary …")
    patcher = uc.Patcher(executable_path=str(dest))
    patcher.patch_exe()
    log.info("Patch applied successfully")
 def main() -> None:
    dest = Path(config.chromedriver_path)
    dest.parent.mkdir(parents=True, exist_ok=True)
    major, _ = _get_chrome_full_version()
    cd_version = _latest_chromedriver_version(major)
    _download_chromedriver(cd_version, dest)
    _patch(dest)
    log.info("ChromeDriver %s ready at %s", cd_version, dest)
    log.info("Re-run 'make patch-driver' only when Chrome updates to a new major version.")
 if __name__ == "__main__":
    main()
@@ -1,2 +0,0 @@
 # Force resolution through the official PyPI index
 index-url = "https://pypi.org/simple"
		`@@ -1,2 +0,0 @@`
			`# Force resolution through the official PyPI index`
			`index-url = "https://pypi.org/simple"`