wip

admin/main.py

@@ -1,6 +1,7 @@
 """FastAPI admin application."""
 from __future__ import annotations
 
+import threading
 from pathlib import Path
 
 from fastapi import FastAPI
@@ -21,6 +22,20 @@ app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
 @app.on_event("startup")
 def on_startup() -> None:
     init_db()
+    _start_huey_consumer()
+
+
+def _start_huey_consumer() -> None:
+    from huey.consumer import Consumer
+    from crawler.tasks import huey
+
+    class _ThreadConsumer(Consumer):
+        def _set_signal_handlers(self) -> None:
+            pass  # signal.signal() only works on the main thread
+
+    consumer = _ThreadConsumer(huey, workers=1, periodic=False)
+    t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
+    t.start()
 
 
 @app.get("/")

admin/routes.py

@@ -2,6 +2,7 @@
 from __future__ import annotations
 
 import secrets
+import uuid
 from typing import Annotated, Any
 
 from fastapi import APIRouter, Depends, HTTPException, status
@@ -17,6 +18,9 @@ from admin.models import (
     upsert_flow_config,
 )
 from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
 
 router = APIRouter()
 security = HTTPBasic()
@@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel):
     search_submit_selector: str = ""
     search_submit_selector_type: str = "css"
     search_results_selector: str = ""
+    search_results_selector_type: str = "css"
     scroll_container_selector: str = ""
+    scroll_container_selector_type: str = "css"
     max_scrolls: int = 30
     scroll_pause_ms: int = 1200
     no_new_content_timeout_ms: int = 3000
+    pagination_selector: str = ""
+    pagination_selector_type: str = "css"
+    max_pages: int = 10
+    pagination_wait_ms: int = 1500
     item_selector_template: str = ""
     item_selector_type: str = "css"
+    item_url_template: str = ""
     target_item_ids: list[str] = []
+    target_url: str = ""
 
 
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
@@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
         "search_submit_selector": body.search_submit_selector,
         "search_submit_selector_type": body.search_submit_selector_type,
         "search_results_selector": body.search_results_selector,
+        "search_results_selector_type": body.search_results_selector_type,
         "scroll_container_selector": body.scroll_container_selector,
+        "scroll_container_selector_type": body.scroll_container_selector_type,
         "max_scrolls": body.max_scrolls,
         "scroll_pause_ms": body.scroll_pause_ms,
         "no_new_content_timeout_ms": body.no_new_content_timeout_ms,
+        "pagination_selector": body.pagination_selector,
+        "pagination_selector_type": body.pagination_selector_type,
+        "max_pages": body.max_pages,
+        "pagination_wait_ms": body.pagination_wait_ms,
         "item_selector_template": body.item_selector_template,
         "item_selector_type": body.item_selector_type,
+        "item_url_template": body.item_url_template,
         "target_item_ids_json": body.target_item_ids,
+        "target_url": body.target_url,
     }
 
 
@@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None:
     if not existing:
         raise HTTPException(status_code=404, detail="Not found")
     await delete_flow_config(cfg_id)
+
+
+# ---------------------------------------------------------------------------
+# Batch runs (managed via Huey task queue)
+# ---------------------------------------------------------------------------
+
+class BatchRunIn(BaseModel):
+    workers: int = 3
+    total_runs: int = 10
+    stagger_ms: int = 500
+    headless: bool = True
+
+
+@router.post("/api/run-batch", status_code=202)
+async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status, run_batch_task, set_current
+
+    current = _status()
+    if current.get("running"):
+        raise HTTPException(status_code=409, detail="A batch is already running")
+
+    from admin.models import get_active_flow_config
+    active_cfg = await get_active_flow_config()
+    if not active_cfg:
+        raise HTTPException(status_code=422, detail="No active flow config — activate one first")
+
+    batch_id = uuid.uuid4().hex
+    result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms)
+    set_current(batch_id, result)
+
+    log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers)
+    return {"batch_id": batch_id, "status": "queued"}
+
+
+@router.post("/api/stop-batch", status_code=200)
+async def stop_batch(_: Auth) -> dict[str, object]:
+    from crawler.tasks import stop_current
+    stopped = stop_current()
+    if not stopped:
+        raise HTTPException(status_code=404, detail="No active batch to stop")
+    return {"status": "stopping"}
+
+
+@router.get("/api/batch-status")
+async def batch_status(_: Auth) -> dict[str, object]:
+    from crawler.tasks import batch_status as _status
+    return _status()  # type: ignore[return-value]

admin/static/dashboard.html

@@ -102,6 +102,17 @@
     }
     .form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; }
     .form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; }
+    details.adv { margin-top: 20px; border: 1px solid var(--border); border-radius: 9px; }
+    details.adv > summary {
+      list-style: none; cursor: pointer; padding: 11px 14px;
+      font-size: 12px; font-weight: 600; color: var(--muted);
+      display: flex; align-items: center; gap: 7px; user-select: none;
+    }
+    details.adv > summary::-webkit-details-marker { display: none; }
+    details.adv > summary::before { content: '▶'; font-size: 9px; transition: transform .15s; }
+    details.adv[open] > summary::before { transform: rotate(90deg); }
+    details.adv > summary:hover { color: var(--text); }
+    .adv-body { padding: 4px 14px 14px; }
     .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; }
     .fg { margin-bottom: 14px; }
     .fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; }
@@ -124,6 +135,9 @@
     .modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; }
     .error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; }
     .error-msg.hidden { display: none; }
+    .remember-row { display: flex; align-items: center; gap: 8px; margin-top: 12px; }
+    .remember-row input[type=checkbox] { width: 15px; height: 15px; accent-color: var(--accent); cursor: pointer; }
+    .remember-row label { color: var(--muted); font-size: 13px; cursor: pointer; user-select: none; }
   </style>
 </head>
 <body>
@@ -134,7 +148,11 @@
     <h2>Seed Admin</h2>
     <div class="fg"><label>Username</label><input id="inp-user" type="text" autocomplete="username" /></div>
     <div class="fg"><label>Password</label><input id="inp-pass" type="password" autocomplete="current-password" /></div>
-    <button class="btn btn-primary" style="width:100%;margin-top:4px" id="login-btn">Sign in</button>
+    <div class="remember-row">
+      <input type="checkbox" id="inp-remember" />
+      <label for="inp-remember">Remember me</label>
+    </div>
+    <button class="btn btn-primary" style="width:100%;margin-top:14px" id="login-btn">Sign in</button>
     <p class="error-msg hidden" id="auth-error">Invalid credentials</p>
   </div>
 </div>
@@ -149,80 +167,130 @@
       <label>Config Name</label>
       <input id="cfg-name" type="text" placeholder="e.g. Product Search Flow" />
     </div>
+
     <div class="fg">
-      <label style="display:flex;align-items:center;gap:8px;cursor:pointer">
-        <input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
-      </label>
+      <label>Target URL</label>
+      <input id="cfg-target-url" type="url" placeholder="https://example.com" />
     </div>
 
-    <!-- Search -->
-    <div class="form-section">Search Step</div>
-    <div class="fg">
+    <div class="fg" style="margin-top:6px">
       <label>Search Texts <span style="color:var(--muted)">(one per line)</span></label>
       <textarea id="cfg-search-texts" placeholder="shoes&#10;blue sneakers&#10;running shoes"></textarea>
     </div>
-    <div class="form-row">
-      <div class="fg">
-        <label>Search Box Selector</label>
-        <input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
-      </div>
-      <div class="fg">
-        <label>Type</label>
-        <select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
-      </div>
-    </div>
-    <div class="form-row">
-      <div class="fg">
-        <label>Submit Button Selector <span style="color:var(--muted)">(leave blank → Enter key)</span></label>
-        <input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
-      </div>
-      <div class="fg">
-        <label>Type</label>
-        <select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
-      </div>
-    </div>
-    <div class="fg">
-      <label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
-      <input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
-    </div>
 
-    <!-- Scroll -->
-    <div class="form-section">Scroll Step (Infinite Scroll)</div>
-    <div class="fg">
-      <label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
-      <input id="cfg-scroll-container" type="text" placeholder=".product-list" />
-    </div>
-    <div class="form-row">
-      <div class="fg">
-        <label>Max Scrolls</label>
-        <input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
-      </div>
-      <div class="fg">
-        <label>Scroll Pause (ms)</label>
-        <input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
-      </div>
-    </div>
-    <div class="fg">
-      <label>No New Content Timeout (ms) <span style="color:var(--muted)">— bail if DOM height unchanged for this long</span></label>
-      <input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
-    </div>
-
-    <!-- Item -->
-    <div class="form-section">Item Selection</div>
-    <div class="fg">
-      <label>Item Selector Template</label>
-      <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or .result-card" />
-      <p class="hint">Use <code>{item_id}</code> as a placeholder — it is replaced per item ID below.</p>
-    </div>
-    <div class="fg">
-      <label>Selector Type</label>
-      <select id="cfg-item-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
-    </div>
     <div class="fg">
       <label>Target Item IDs <span style="color:var(--muted)">(one per line)</span></label>
       <textarea id="cfg-item-ids" placeholder="prod-1234&#10;prod-5678"></textarea>
     </div>
 
+    <!-- Advanced -->
+    <details class="adv" id="adv-details">
+      <summary>Advanced Settings</summary>
+      <div class="adv-body">
+
+        <div class="fg">
+          <label style="display:flex;align-items:center;gap:8px;cursor:pointer">
+            <input id="cfg-active" type="checkbox" style="width:auto" /> Set as active config
+          </label>
+        </div>
+
+        <div class="form-section">Search Step</div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Search Box Selector</label>
+            <input id="cfg-search-sel" type="text" placeholder="input[name='q']" />
+          </div>
+          <div class="fg">
+            <label>Type</label>
+            <select id="cfg-search-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Submit Button Selector <span style="color:var(--muted)">(blank → Enter key)</span></label>
+            <input id="cfg-submit-sel" type="text" placeholder="button[type='submit']" />
+          </div>
+          <div class="fg">
+            <label>Type</label>
+            <select id="cfg-submit-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Results Container Selector <span style="color:var(--muted)">(waited for after search)</span></label>
+            <input id="cfg-results-sel" type="text" placeholder=".search-results, #results-list" />
+          </div>
+          <div class="fg">
+            <label>Type</label>
+            <select id="cfg-results-sel-type"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+
+        <div class="form-section">Scroll</div>
+        <div class="fg">
+          <label>Scroll Container Selector <span style="color:var(--muted)">(blank = window)</span></label>
+          <div style="display:flex;gap:8px;align-items:center">
+            <input id="cfg-scroll-container" type="text" placeholder=".product-list" style="flex:1" />
+            <select id="cfg-scroll-container-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Max Scrolls</label>
+            <input id="cfg-max-scrolls" type="number" min="1" max="500" value="30" />
+          </div>
+          <div class="fg">
+            <label>Scroll Pause (ms)</label>
+            <input id="cfg-scroll-pause" type="number" min="200" max="10000" value="1200" />
+          </div>
+        </div>
+        <div class="fg">
+          <label>No New Content Timeout (ms)</label>
+          <input id="cfg-no-content-timeout" type="number" min="500" max="30000" value="3000" />
+        </div>
+
+        <div class="form-section">Pagination <span style="font-weight:400;color:var(--muted)">(optional)</span></div>
+        <div class="fg">
+          <label>Next Page Button Selector <span style="color:var(--muted)">(blank = disabled)</span></label>
+          <div style="display:flex;gap:8px;align-items:center">
+            <input id="cfg-pagination-sel" type="text" placeholder=".pagination .next, a[rel='next']" style="flex:1" />
+            <select id="cfg-pagination-sel-type" style="width:auto"><option value="css">CSS</option><option value="xpath">XPath</option></select>
+          </div>
+        </div>
+        <div class="form-row">
+          <div class="fg">
+            <label>Max Pages</label>
+            <input id="cfg-max-pages" type="number" min="1" max="200" value="10" />
+          </div>
+          <div class="fg">
+            <label>Wait After Page Turn (ms)</label>
+            <input id="cfg-pagination-wait" type="number" min="200" max="15000" value="1500" />
+          </div>
+        </div>
+
+        <div class="form-section">Item Selection</div>
+        <div class="fg">
+          <label>Item URL Template <span style="color:var(--muted)">(href pattern, highest priority)</span></label>
+          <input id="cfg-item-url-tpl" type="text" placeholder="/products/{item_id}" />
+          <p class="hint">Finds <code>&lt;a href&gt;</code> whose href contains this pattern. Use <code>{item_id}</code> as placeholder.</p>
+        </div>
+        <div class="fg">
+          <label>Item Selector Template <span style="color:var(--muted)">(fallback)</span></label>
+          <input id="cfg-item-sel" type="text" placeholder="[data-id='{item_id}'] or /products/{item_id}" />
+        </div>
+        <div class="fg">
+          <label>Selector Type</label>
+          <select id="cfg-item-sel-type" style="width:auto">
+            <option value="css">CSS</option>
+            <option value="xpath">XPath</option>
+            <option value="a-link">A-Link (href contains)</option>
+          </select>
+          <p class="hint" id="alink-hint" style="display:none">Template matched against <code>&lt;a href&gt;</code> — e.g. <code>/products/{item_id}</code>.</p>
+        </div>
+
+      </div>
+    </details>
+
     <div class="form-footer">
       <button class="btn btn-ghost" id="form-cancel">Cancel</button>
       <button class="btn btn-primary" id="form-save">Save Config</button>
@@ -244,6 +312,7 @@
 <div class="tabs">
   <div class="tab active" data-tab="dashboard">Dashboard</div>
   <div class="tab" data-tab="flows">Flow Configs</div>
+  <div class="tab" data-tab="batch">Batch Run</div>
 </div>
 
 <!-- ── Dashboard Tab ──────────────────────────────────────────── -->
@@ -288,8 +357,12 @@
       <div style="font-size:16px;font-weight:600;margin-bottom:4px">Flow Configurations</div>
       <div style="color:var(--muted);font-size:13px">Only one config is active at a time. The crawler picks it up on every run.</div>
     </div>
-    <button class="btn btn-primary" id="btn-new-config">+ New Config</button>
+    <div style="display:flex;gap:8px">
+      <button class="btn btn-ghost" id="btn-import-config">↑ Import</button>
+      <button class="btn btn-primary" id="btn-new-config">+ New Config</button>
+    </div>
   </div>
+  <input type="file" id="import-file-input" accept=".json" style="display:none">
 
   <div class="config-list" id="config-list">
     <p class="empty">No configs yet. Create one to get started.</p>
@@ -297,6 +370,96 @@
 </main>
 </div>
 
+<!-- ── Batch Run Tab ─────────────────────────────────────────── -->
+<div class="tab-panel" id="tab-batch">
+<main>
+  <div style="font-size:16px;font-weight:600;margin-bottom:4px">Batch Run</div>
+  <div style="color:var(--muted);font-size:13px;margin-bottom:24px">Launch many parallel crawler sessions using the active flow config.</div>
+
+  <div style="display:grid;grid-template-columns:320px 1fr;gap:20px;max-width:1100px">
+
+    <!-- Controls -->
+    <div class="panel" style="align-self:start">
+      <div class="section-title">Configuration</div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Workers <span style="color:var(--muted)">(parallel Chrome instances)</span></label>
+        <input id="batch-workers" type="number" min="1" max="50" value="3" />
+      </div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Total Runs</label>
+        <input id="batch-runs" type="number" min="1" max="10000" value="10" />
+      </div>
+      <div class="fg" style="margin-bottom:14px">
+        <label>Stagger Between Starts (ms)</label>
+        <input id="batch-stagger" type="number" min="0" max="30000" value="500" />
+      </div>
+      <div style="display:flex;align-items:center;gap:8px;margin-bottom:20px">
+        <input id="batch-headless" type="checkbox" checked style="width:16px;height:16px;cursor:pointer" />
+        <label for="batch-headless" style="cursor:pointer;margin:0">Headless</label>
+      </div>
+      <div style="display:flex;gap:10px">
+        <button class="btn btn-primary" id="batch-start-btn" style="flex:1">▶ Start</button>
+        <button class="btn btn-danger" id="batch-stop-btn" style="flex:1;display:none">■ Stop</button>
+      </div>
+      <div id="batch-error" style="color:var(--red);font-size:12px;margin-top:10px;display:none"></div>
+    </div>
+
+    <!-- Status -->
+    <div class="panel">
+      <div style="display:flex;align-items:center;justify-content:space-between;margin-bottom:14px">
+        <div class="section-title" style="margin:0">Live Status</div>
+        <span id="bs-active-badge" style="display:none;font-size:12px;color:var(--accent);font-weight:600"></span>
+      </div>
+      <div id="batch-idle-msg" style="color:var(--muted);font-size:13px">No batch running.</div>
+      <div id="batch-status-body" style="display:none">
+        <div style="margin-bottom:16px">
+          <div style="display:flex;justify-content:space-between;margin-bottom:6px">
+            <span style="color:var(--muted);font-size:12px">Progress</span>
+            <span id="batch-progress-label" style="font-size:12px">0 / 0</span>
+          </div>
+          <div style="background:var(--surface2);border-radius:4px;height:8px;overflow:hidden">
+            <div id="batch-progress-bar" style="height:100%;background:var(--accent);width:0%;transition:width .3s"></div>
+          </div>
+        </div>
+        <div style="display:grid;grid-template-columns:1fr 1fr 1fr;gap:10px;margin-bottom:16px">
+          <div class="card" style="padding:12px">
+            <div class="label">Completed</div>
+            <div class="value accent" id="bs-completed">0</div>
+          </div>
+          <div class="card" style="padding:12px">
+            <div class="label">Succeeded</div>
+            <div class="value green" id="bs-succeeded">0</div>
+          </div>
+          <div class="card" style="padding:12px">
+            <div class="label">Failed</div>
+            <div class="value red" id="bs-failed">0</div>
+          </div>
+        </div>
+        <div style="font-size:12px;color:var(--muted);margin-bottom:14px">
+          Workers: <strong id="bs-workers">–</strong> &nbsp;·&nbsp;
+          Started: <strong id="bs-started">–</strong>
+          <span id="bs-finished-row" style="display:none"> &nbsp;·&nbsp; Finished: <strong id="bs-finished">–</strong></span>
+        </div>
+
+        <!-- Per-worker table -->
+        <div class="section-title">Workers</div>
+        <table id="bs-worker-table" style="font-size:12px">
+          <thead>
+            <tr>
+              <th>#</th>
+              <th>Status</th>
+              <th>Duration</th>
+              <th>Error</th>
+            </tr>
+          </thead>
+          <tbody id="bs-worker-tbody"></tbody>
+        </table>
+      </div>
+    </div>
+  </div>
+</main>
+</div>
+
 <script>
 /* ─────────────────────────── Auth ─────────────────────────── */
 let _creds = null;
@@ -309,24 +472,73 @@ async function apiFetch(path, opts = {}) {
   return res;
 }
 
-function showAuth() { document.getElementById('auth-modal').classList.remove('hidden'); }
-function hideAuth() { document.getElementById('auth-modal').classList.add('hidden'); }
+const _STORAGE_KEY = 'seed_admin_creds';
+
+function showAuth() {
+  document.getElementById('auth-modal').classList.remove('hidden');
+}
+function hideAuth() {
+  document.getElementById('auth-modal').classList.add('hidden');
+}
+
+function _loadSaved() {
+  try {
+    const raw = localStorage.getItem(_STORAGE_KEY);
+    if (!raw) return null;
+    return JSON.parse(raw);
+  } catch { return null; }
+}
+
+function _saveCreds(u, p) {
+  localStorage.setItem(_STORAGE_KEY, JSON.stringify({ u, p }));
+}
+
+function _clearCreds() {
+  localStorage.removeItem(_STORAGE_KEY);
+}
+
+async function _attemptLogin(u, p) {
+  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
+  return res;
+}
 
 document.getElementById('login-btn').addEventListener('click', async () => {
   const u = document.getElementById('inp-user').value.trim();
   const p = document.getElementById('inp-pass').value;
-  const res = await fetch('/api/stats', { headers: { 'Authorization': `Basic ${b64(u, p)}` } });
+  const remember = document.getElementById('inp-remember').checked;
+  const res = await _attemptLogin(u, p);
   if (res.ok) {
     _creds = { u, p };
+    remember ? _saveCreds(u, p) : _clearCreds();
     document.getElementById('auth-error').classList.add('hidden');
     hideAuth();
     await Promise.all([loadStats(), loadConfigs()]);
     startAutoRefresh();
   } else {
+    _clearCreds();
     document.getElementById('auth-error').classList.remove('hidden');
   }
 });
-document.getElementById('inp-pass').addEventListener('keydown', e => { if (e.key === 'Enter') document.getElementById('login-btn').click(); });
+document.getElementById('inp-pass').addEventListener('keydown', e => {
+  if (e.key === 'Enter') document.getElementById('login-btn').click();
+});
+
+// Auto-login from saved credentials on page load
+(async () => {
+  const saved = _loadSaved();
+  if (!saved) return;
+  const res = await _attemptLogin(saved.u, saved.p);
+  if (res.ok) {
+    _creds = saved;
+    document.getElementById('inp-user').value = saved.u;
+    document.getElementById('inp-remember').checked = true;
+    hideAuth();
+    await Promise.all([loadStats(), loadConfigs()]);
+    startAutoRefresh();
+  } else {
+    _clearCreds(); // saved creds are stale (password changed)
+  }
+})();
 
 /* ─────────────────────────── Tabs ─────────────────────────── */
 document.querySelectorAll('.tab').forEach(tab => {
@@ -336,6 +548,7 @@ document.querySelectorAll('.tab').forEach(tab => {
     tab.classList.add('active');
     document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
     if (tab.dataset.tab === 'flows') loadConfigs();
+    if (tab.dataset.tab === 'batch') pollBatchStatus();
   });
 });
 
@@ -426,11 +639,15 @@ function renderConfigs() {
         </div>
         <div class="config-meta" style="margin-top:4px">
           Item selector: <code style="font-size:11px">${esc(c.item_selector_template || '—')}</code>
+          ${c.item_url_template ? `&nbsp;·&nbsp; href pattern: <code style="font-size:11px">${esc(c.item_url_template)}</code>` : ''}
+          ${c.pagination_selector ? `&nbsp;·&nbsp; Pagination: <code style="font-size:11px">${esc(c.pagination_selector)}</code> (max ${c.max_pages} pages)` : ''}
         </div>
       </div>
       <div class="config-actions">
         ${!c.is_active ? `<button class="btn btn-ghost btn-sm" onclick="activateConfig(${c.id})">Activate</button>` : ''}
         <button class="btn btn-ghost btn-sm" onclick="openEditForm(${c.id})">Edit</button>
+        <button class="btn btn-ghost btn-sm" onclick="duplicateConfig(${c.id})">Duplicate</button>
+        <button class="btn btn-ghost btn-sm" onclick="exportConfig(${c.id})">↓ Export</button>
         <button class="btn btn-danger btn-sm" onclick="deleteConfig(${c.id})">Delete</button>
       </div>
     </div>
@@ -449,25 +666,143 @@ async function deleteConfig(id) {
   if (res && (res.ok || res.status === 204)) await loadConfigs();
 }
 
+function exportConfig(id) {
+  const c = _configs.find(x => x.id === id);
+  if (!c) return;
+  const out = {
+    name: c.name,
+    search_texts: c.search_texts_json || [],
+    search_box_selector: c.search_box_selector,
+    search_box_selector_type: c.search_box_selector_type,
+    search_submit_selector: c.search_submit_selector,
+    search_submit_selector_type: c.search_submit_selector_type,
+    search_results_selector: c.search_results_selector,
+    search_results_selector_type: c.search_results_selector_type,
+    scroll_container_selector: c.scroll_container_selector,
+    scroll_container_selector_type: c.scroll_container_selector_type,
+    max_scrolls: c.max_scrolls,
+    scroll_pause_ms: c.scroll_pause_ms,
+    no_new_content_timeout_ms: c.no_new_content_timeout_ms,
+    pagination_selector: c.pagination_selector,
+    pagination_selector_type: c.pagination_selector_type,
+    max_pages: c.max_pages,
+    pagination_wait_ms: c.pagination_wait_ms,
+    item_selector_template: c.item_selector_template,
+    item_selector_type: c.item_selector_type,
+    item_url_template: c.item_url_template,
+    target_item_ids: c.target_item_ids_json || [],
+    target_url: c.target_url || '',
+  };
+  const blob = new Blob([JSON.stringify(out, null, 2)], { type: 'application/json' });
+  const url = URL.createObjectURL(blob);
+  const a = document.createElement('a');
+  a.href = url; a.download = `${c.name.replace(/[^a-z0-9_-]/gi, '_')}.json`;
+  a.click(); URL.revokeObjectURL(url);
+}
+
+async function duplicateConfig(id) {
+  const c = _configs.find(x => x.id === id);
+  if (!c) return;
+  const body = {
+    name: c.name + ' (copy)',
+    is_active: false,
+    search_texts: c.search_texts_json || [],
+    search_box_selector: c.search_box_selector || '',
+    search_box_selector_type: c.search_box_selector_type || 'css',
+    search_submit_selector: c.search_submit_selector || '',
+    search_submit_selector_type: c.search_submit_selector_type || 'css',
+    search_results_selector: c.search_results_selector || '',
+    search_results_selector_type: c.search_results_selector_type || 'css',
+    scroll_container_selector: c.scroll_container_selector || '',
+    scroll_container_selector_type: c.scroll_container_selector_type || 'css',
+    max_scrolls: c.max_scrolls ?? 30,
+    scroll_pause_ms: c.scroll_pause_ms ?? 1200,
+    no_new_content_timeout_ms: c.no_new_content_timeout_ms ?? 3000,
+    pagination_selector: c.pagination_selector || '',
+    pagination_selector_type: c.pagination_selector_type || 'css',
+    max_pages: c.max_pages ?? 10,
+    pagination_wait_ms: c.pagination_wait_ms ?? 1500,
+    item_selector_template: c.item_selector_template || '',
+    item_selector_type: c.item_selector_type || 'css',
+    item_url_template: c.item_url_template || '',
+    target_item_ids: c.target_item_ids_json || [],
+    target_url: c.target_url || '',
+  };
+  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+  if (res && res.ok) await loadConfigs();
+}
+
+document.getElementById('btn-import-config').addEventListener('click', () => {
+  document.getElementById('import-file-input').click();
+});
+document.getElementById('import-file-input').addEventListener('change', async (e) => {
+  const file = e.target.files[0];
+  if (!file) return;
+  e.target.value = '';
+  let data;
+  try { data = JSON.parse(await file.text()); } catch { alert('Invalid JSON file'); return; }
+  const body = {
+    name: data.name || file.name.replace(/\.json$/, ''),
+    is_active: false,
+    search_texts: data.search_texts || [],
+    search_box_selector: data.search_box_selector || '',
+    search_box_selector_type: data.search_box_selector_type || 'css',
+    search_submit_selector: data.search_submit_selector || '',
+    search_submit_selector_type: data.search_submit_selector_type || 'css',
+    search_results_selector: data.search_results_selector || '',
+    search_results_selector_type: data.search_results_selector_type || 'css',
+    scroll_container_selector: data.scroll_container_selector || '',
+    scroll_container_selector_type: data.scroll_container_selector_type || 'css',
+    max_scrolls: data.max_scrolls ?? 30,
+    scroll_pause_ms: data.scroll_pause_ms ?? 1200,
+    no_new_content_timeout_ms: data.no_new_content_timeout_ms ?? 3000,
+    pagination_selector: data.pagination_selector || '',
+    pagination_selector_type: data.pagination_selector_type || 'css',
+    max_pages: data.max_pages ?? 10,
+    pagination_wait_ms: data.pagination_wait_ms ?? 1500,
+    item_selector_template: data.item_selector_template || '',
+    item_selector_type: data.item_selector_type || 'css',
+    item_url_template: data.item_url_template || '',
+    target_item_ids: data.target_item_ids || [],
+    target_url: data.target_url || '',
+  };
+  const res = await apiFetch('/api/flow-configs', { method: 'POST', headers: { 'Content-Type': 'application/json' }, body: JSON.stringify(body) });
+  if (res && res.ok) await loadConfigs();
+});
+
 /* ─── Form ─── */
-function openNewForm() {
-  document.getElementById('form-title').textContent = 'New Flow Config';
-  document.getElementById('cfg-id').value = '';
-  document.getElementById('cfg-name').value = '';
+function _resetAdvanced() {
   document.getElementById('cfg-active').checked = false;
-  document.getElementById('cfg-search-texts').value = '';
   document.getElementById('cfg-search-sel').value = '';
   document.getElementById('cfg-search-sel-type').value = 'css';
   document.getElementById('cfg-submit-sel').value = '';
   document.getElementById('cfg-submit-sel-type').value = 'css';
   document.getElementById('cfg-results-sel').value = '';
+  document.getElementById('cfg-results-sel-type').value = 'css';
   document.getElementById('cfg-scroll-container').value = '';
+  document.getElementById('cfg-scroll-container-type').value = 'css';
   document.getElementById('cfg-max-scrolls').value = '30';
   document.getElementById('cfg-scroll-pause').value = '1200';
   document.getElementById('cfg-no-content-timeout').value = '3000';
+  document.getElementById('cfg-pagination-sel').value = '';
+  document.getElementById('cfg-pagination-sel-type').value = 'css';
+  document.getElementById('cfg-max-pages').value = '10';
+  document.getElementById('cfg-pagination-wait').value = '1500';
   document.getElementById('cfg-item-sel').value = '';
   document.getElementById('cfg-item-sel-type').value = 'css';
+  document.getElementById('alink-hint').style.display = 'none';
+  document.getElementById('cfg-item-url-tpl').value = '';
+}
+
+function openNewForm() {
+  document.getElementById('form-title').textContent = 'New Flow Config';
+  document.getElementById('cfg-id').value = '';
+  document.getElementById('cfg-name').value = '';
+  document.getElementById('cfg-target-url').value = '';
+  document.getElementById('cfg-search-texts').value = '';
   document.getElementById('cfg-item-ids').value = '';
+  _resetAdvanced();
+  document.getElementById('adv-details').removeAttribute('open');
   document.getElementById('config-overlay').classList.remove('hidden');
 }
 
@@ -477,20 +812,31 @@ function openEditForm(id) {
   document.getElementById('form-title').textContent = 'Edit Flow Config';
   document.getElementById('cfg-id').value = id;
   document.getElementById('cfg-name').value = c.name || '';
-  document.getElementById('cfg-active').checked = !!c.is_active;
+  document.getElementById('cfg-target-url').value = c.target_url || '';
   document.getElementById('cfg-search-texts').value = (c.search_texts_json || []).join('\n');
+  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
+  document.getElementById('cfg-active').checked = !!c.is_active;
   document.getElementById('cfg-search-sel').value = c.search_box_selector || '';
   document.getElementById('cfg-search-sel-type').value = c.search_box_selector_type || 'css';
   document.getElementById('cfg-submit-sel').value = c.search_submit_selector || '';
   document.getElementById('cfg-submit-sel-type').value = c.search_submit_selector_type || 'css';
   document.getElementById('cfg-results-sel').value = c.search_results_selector || '';
+  document.getElementById('cfg-results-sel-type').value = c.search_results_selector_type || 'css';
   document.getElementById('cfg-scroll-container').value = c.scroll_container_selector || '';
+  document.getElementById('cfg-scroll-container-type').value = c.scroll_container_selector_type || 'css';
   document.getElementById('cfg-max-scrolls').value = c.max_scrolls ?? 30;
   document.getElementById('cfg-scroll-pause').value = c.scroll_pause_ms ?? 1200;
   document.getElementById('cfg-no-content-timeout').value = c.no_new_content_timeout_ms ?? 3000;
+  document.getElementById('cfg-pagination-sel').value = c.pagination_selector || '';
+  document.getElementById('cfg-pagination-sel-type').value = c.pagination_selector_type || 'css';
+  document.getElementById('cfg-max-pages').value = c.max_pages ?? 10;
+  document.getElementById('cfg-pagination-wait').value = c.pagination_wait_ms ?? 1500;
   document.getElementById('cfg-item-sel').value = c.item_selector_template || '';
-  document.getElementById('cfg-item-sel-type').value = c.item_selector_type || 'css';
-  document.getElementById('cfg-item-ids').value = (c.target_item_ids_json || []).join('\n');
+  const _ist = c.item_selector_type || 'css';
+  document.getElementById('cfg-item-sel-type').value = _ist;
+  document.getElementById('alink-hint').style.display = _ist === 'a-link' ? '' : 'none';
+  document.getElementById('cfg-item-url-tpl').value = c.item_url_template || '';
+  document.getElementById('adv-details').setAttribute('open', '');
   document.getElementById('config-overlay').classList.remove('hidden');
 }
 
@@ -508,19 +854,30 @@ function collectForm() {
     search_submit_selector: document.getElementById('cfg-submit-sel').value.trim(),
     search_submit_selector_type: document.getElementById('cfg-submit-sel-type').value,
     search_results_selector: document.getElementById('cfg-results-sel').value.trim(),
+    search_results_selector_type: document.getElementById('cfg-results-sel-type').value,
     scroll_container_selector: document.getElementById('cfg-scroll-container').value.trim(),
+    scroll_container_selector_type: document.getElementById('cfg-scroll-container-type').value,
     max_scrolls: parseInt(document.getElementById('cfg-max-scrolls').value) || 30,
     scroll_pause_ms: parseInt(document.getElementById('cfg-scroll-pause').value) || 1200,
     no_new_content_timeout_ms: parseInt(document.getElementById('cfg-no-content-timeout').value) || 3000,
+    pagination_selector: document.getElementById('cfg-pagination-sel').value.trim(),
+    pagination_selector_type: document.getElementById('cfg-pagination-sel-type').value,
+    max_pages: parseInt(document.getElementById('cfg-max-pages').value) || 10,
+    pagination_wait_ms: parseInt(document.getElementById('cfg-pagination-wait').value) || 1500,
     item_selector_template: document.getElementById('cfg-item-sel').value.trim(),
     item_selector_type: document.getElementById('cfg-item-sel-type').value,
+    item_url_template: document.getElementById('cfg-item-url-tpl').value.trim(),
     target_item_ids: document.getElementById('cfg-item-ids').value.split('\n').map(s => s.trim()).filter(Boolean),
+    target_url: document.getElementById('cfg-target-url').value.trim(),
   };
 }
 
 document.getElementById('btn-new-config').addEventListener('click', openNewForm);
 document.getElementById('form-cancel').addEventListener('click', closeForm);
 document.getElementById('config-overlay').addEventListener('click', e => { if (e.target === e.currentTarget) closeForm(); });
+document.getElementById('cfg-item-sel-type').addEventListener('change', e => {
+  document.getElementById('alink-hint').style.display = e.target.value === 'a-link' ? '' : 'none';
+});
 
 document.getElementById('form-save').addEventListener('click', async () => {
   const body = collectForm();
@@ -540,8 +897,9 @@ document.getElementById('form-save').addEventListener('click', async () => {
     closeForm();
     await loadConfigs();
   } else {
-    alert('Failed to save config. Check console.');
-    console.error(await res?.text());
+    const detail = res ? await res.text().catch(() => `HTTP ${res.status}`) : 'No response (network error)';
+    console.error('Save failed:', detail);
+    alert(`Save failed (${res?.status ?? '?'}):\n${detail}\n\nIf columns are missing, restart the admin panel to apply DB migrations.`);
   }
 });
 
@@ -553,6 +911,162 @@ function fmtTime(iso) {
   if (!iso) return '–';
   return new Date(iso).toLocaleString();
 }
+
+/* ─────────────────────────── Batch Run ─────────────────────── */
+let _batchPollTimer = null;
+
+const _STATUS_COLOR = {
+  running: 'var(--accent)',
+  ok:      'var(--green)',
+  failed:  'var(--red)',
+  error:   'var(--red)',
+  skipped: 'var(--muted)',
+  pending: 'var(--muted)',
+};
+
+function fmtDuration(ms) {
+  if (ms == null) return '–';
+  if (ms < 1000) return ms + ' ms';
+  return (ms / 1000).toFixed(1) + ' s';
+}
+
+function fmtTs(ts) {
+  if (!ts) return '–';
+  return new Date(ts * 1000).toLocaleTimeString();
+}
+
+function renderBatchStatus(s) {
+  const idle     = document.getElementById('batch-idle-msg');
+  const body     = document.getElementById('batch-status-body');
+  const errEl    = document.getElementById('batch-error');
+  const startBtn = document.getElementById('batch-start-btn');
+  const stopBtn  = document.getElementById('batch-stop-btn');
+  const badge    = document.getElementById('bs-active-badge');
+
+  if (!s.batch_id) {
+    idle.style.display = '';
+    body.style.display = 'none';
+    startBtn.style.display = '';
+    stopBtn.style.display = 'none';
+    badge.style.display = 'none';
+    return;
+  }
+
+  idle.style.display = 'none';
+  body.style.display = '';
+
+  const pct = s.total ? Math.round(s.completed / s.total * 100) : 0;
+  document.getElementById('batch-progress-bar').style.width = pct + '%';
+  document.getElementById('batch-progress-label').textContent = `${s.completed} / ${s.total}`;
+  document.getElementById('bs-completed').textContent = s.completed ?? 0;
+  document.getElementById('bs-succeeded').textContent = s.succeeded ?? 0;
+  document.getElementById('bs-failed').textContent = s.failed ?? 0;
+  document.getElementById('bs-workers').textContent = s.configured_workers ?? '–';
+  document.getElementById('bs-started').textContent = fmtTs(s.started_at);
+
+  const finRow = document.getElementById('bs-finished-row');
+  if (s.finished_at) {
+    finRow.style.display = '';
+    document.getElementById('bs-finished').textContent = fmtTs(s.finished_at);
+  } else {
+    finRow.style.display = 'none';
+  }
+
+  // Active worker badge
+  const active = s.active_workers ?? 0;
+  if (s.running && active > 0) {
+    badge.textContent = `${active} / ${s.configured_workers} active`;
+    badge.style.display = '';
+  } else {
+    badge.style.display = 'none';
+  }
+
+  // Per-worker table
+  const tbody = document.getElementById('bs-worker-tbody');
+  const workers = s.workers_detail || [];
+  tbody.innerHTML = workers.map(w => {
+    const color = _STATUS_COLOR[w.status] || 'var(--muted)';
+    const label = w.status === 'running' ? '⟳ running' : w.status;
+    const err = w.error && w.status !== 'skipped' ? `<span style="color:var(--red)">${esc(w.error)}</span>` : '–';
+    return `<tr>
+      <td style="font-family:monospace">${w.slot}</td>
+      <td style="color:${color};font-weight:600">${label}</td>
+      <td>${fmtDuration(w.duration_ms)}</td>
+      <td style="max-width:220px;overflow:hidden;text-overflow:ellipsis;white-space:nowrap">${err}</td>
+    </tr>`;
+  }).join('');
+
+  if (s.error) {
+    errEl.textContent = 'Error: ' + s.error;
+    errEl.style.display = '';
+  } else {
+    errEl.style.display = 'none';
+  }
+
+  if (s.running) {
+    startBtn.style.display = 'none';
+    stopBtn.style.display = '';
+    stopBtn.disabled = s.stopping || false;
+    stopBtn.textContent = s.stopping ? '■ Stopping…' : '■ Stop';
+  } else {
+    startBtn.style.display = '';
+    stopBtn.style.display = 'none';
+  }
+}
+
+async function pollBatchStatus() {
+  const res = await apiFetch('/api/batch-status');
+  if (!res || !res.ok) return;
+  const s = await res.json();
+  renderBatchStatus(s);
+  if (s.running && !_batchPollTimer) startBatchPoll();
+}
+
+function startBatchPoll() {
+  if (_batchPollTimer) return;
+  _batchPollTimer = setInterval(async () => {
+    const r = await apiFetch('/api/batch-status');
+    if (!r || !r.ok) return;
+    const st = await r.json();
+    renderBatchStatus(st);
+    if (!st.running) {
+      clearInterval(_batchPollTimer);
+      _batchPollTimer = null;
+    }
+  }, 1500);
+}
+
+document.getElementById('batch-start-btn').addEventListener('click', async () => {
+  const workers  = parseInt(document.getElementById('batch-workers').value) || 3;
+  const runs     = parseInt(document.getElementById('batch-runs').value) || 10;
+  const stagger  = parseInt(document.getElementById('batch-stagger').value) || 0;
+  const headless = document.getElementById('batch-headless').checked;
+
+  const res = await apiFetch('/api/run-batch', {
+    method: 'POST',
+    headers: { 'Content-Type': 'application/json' },
+    body: JSON.stringify({ workers, total_runs: runs, stagger_ms: stagger, headless }),
+  });
+
+  if (!res) return;
+  if (res.status === 409) { alert('A batch is already running.'); return; }
+  if (res.status === 422) { const d = await res.json(); alert(d.detail || 'No active flow config.'); return; }
+  if (!res.ok) { alert('Failed to start batch.'); return; }
+
+  await pollBatchStatus();
+  startBatchPoll();
+});
+
+document.getElementById('batch-stop-btn').addEventListener('click', async () => {
+  const btn = document.getElementById('batch-stop-btn');
+  btn.disabled = true;
+  btn.textContent = '■ Stopping…';
+  const res = await apiFetch('/api/stop-batch', { method: 'POST' });
+  if (!res || !res.ok) {
+    btn.disabled = false;
+    btn.textContent = '■ Stop';
+  }
+});
 </script>
 </body>
 </html>