Navigate before initializing page state

admin/routes.py

@@ -6,12 +6,13 @@ import secrets
 from datetime import UTC, datetime
 from pathlib import Path
 from typing import Annotated, Any
+from urllib.parse import urlsplit
 
 import speedtest  # type: ignore[import-untyped]
 from fastapi import APIRouter, Depends, HTTPException, status
 from fastapi.responses import FileResponse
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from admin.models import (
     delete_flow_config,
@@ -91,6 +92,17 @@ class FlowConfigIn(BaseModel):
     scenario: str = "dynamic"
     stop_on_first_click: bool = False
 
+    @field_validator("target_url")
+    @classmethod
+    def validate_target_url(cls, value: str) -> str:
+        value = value.strip()
+        if value and "://" not in value:
+            value = f"https://{value}"
+        parsed = urlsplit(value)
+        if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+            raise ValueError("Target URL must be a complete HTTP(S) URL")
+        return value
+
 
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
     return {
@@ -181,6 +193,11 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
         raise HTTPException(
             status_code=404, detail=f"Flow config {body.config_id} not found"
         )
+    if not str(cfg.get("target_url") or "").strip():
+        raise HTTPException(
+            status_code=422,
+            detail="The selected flow config has no Target URL. Edit and save it first.",
+        )
 
     result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
     log.info(

admin/static/dashboard.html

@@ -183,7 +183,7 @@
 
     <div class="fg">
       <label>Target URL</label>
-      <input id="cfg-target-url" type="url" placeholder="https://example.com" />
+      <input id="cfg-target-url" type="url" placeholder="https://example.com" required />
     </div>
 
     <div class="fg" style="display:flex;align-items:center;gap:10px;margin-bottom:14px">
@@ -1189,6 +1189,7 @@ document.getElementById('cfg-digipay-match').addEventListener('change', e => {
 document.getElementById('form-save').addEventListener('click', async () => {
   const body = collectForm();
   if (!body.name) { alert('Config name is required.'); return; }
+  if (!body.target_url) { alert('Target URL is required.'); return; }
 
   const cfgId = document.getElementById('cfg-id').value;
   const isEdit = !!cfgId;
@@ -1408,6 +1409,11 @@ function startBatchPoll() {
 document.getElementById('batch-start-btn').addEventListener('click', async () => {
   const configId = parseInt(document.getElementById('batch-config-select').value);
   if (!configId) { alert('Please select a flow config.'); return; }
+  const selectedConfig = _configs.find(c => c.id === configId);
+  if (!selectedConfig?.target_url) {
+    alert('The selected flow config has no Target URL. Edit and save it first.');
+    return;
+  }
   const workers  = parseInt(document.getElementById('batch-workers').value) || 3;
   const runs     = parseInt(document.getElementById('batch-runs').value) || 10;
   const stagger  = parseInt(document.getElementById('batch-stagger').value) || 0;
@@ -1420,8 +1426,11 @@ document.getElementById('batch-start-btn').addEventListener('click', async () =>
   });
 
   if (!res) return;
-  if (res.status === 404) { const d = await res.json(); alert(d.detail || 'Flow config not found.'); return; }
+  if (!res.ok) {
-  if (!res.ok) { alert('Failed to add task.'); return; }
+    const d = await res.json().catch(() => ({}));
+    alert(d.detail || 'Failed to add task.');
+    return;
+  }
 
   const data = await res.json();
   await pollBatchStatus();

crawler/batch.py

@@ -27,9 +27,14 @@ def _single_run(
 
     identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
     t0 = time.monotonic()
-    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+    target_url = str(cfg.get("target_url") or "").strip()
+    log.info(
+        "[batch:%s] Run %d starting — target=%s",
+        batch_id[:8],
+        run_index,
+        target_url or "<missing>",
+    )
     try:
-        target_url: str = cfg.get("target_url") or ""
         with driver_session(headless=headless, recording_name=identifier) as driver:
             FreshSessionScenario(driver, target_url).run()
             result = DynamicFlow(driver, cfg).run() if cfg else None

crawler/scenarios/fresh_session.py

@@ -1,8 +1,11 @@
 """Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
 from __future__ import annotations
 
+from typing import Any, cast
+from urllib.parse import urlsplit
+
 import undetected_chromedriver as uc
-from selenium.common.exceptions import TimeoutException
+from selenium.common.exceptions import TimeoutException, WebDriverException
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
@@ -10,56 +13,100 @@ from logger import get_logger
 
 log = get_logger(__name__)
 
+_BLANK_URLS = {"", "about:blank", "data:,"}
+
+
+class FreshSessionError(RuntimeError):
+    """The browser could not leave its initial blank page."""
+
+
+def _normalise_url(url: str) -> str:
+    value = url.strip()
+    if value and "://" not in value:
+        value = f"https://{value}"
+    parsed = urlsplit(value)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise FreshSessionError(
+            "target_url must be a complete HTTP(S) URL, for example https://example.com"
+        )
+    return value
+
 
 class FreshSessionScenario:
     """
     Opens the target URL in a brand-new browser profile (no stored cookies,
     localStorage, or cache) so each run appears as a first-time visitor.
 
-    Because undetected-chromedriver already creates an isolated temp profile
+    Undetected-chromedriver creates an isolated temporary profile per instance,
-    per instance, simply spinning up a new driver is sufficient — but we also
+    so no cookie or storage clearing is needed before navigation.
-    explicitly delete all cookies after loading to be safe.
     """
 
     def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
         self.driver = driver
-        self.url = url
+        self.url = _normalise_url(url)
         self.wait = WebDriverWait(driver, timeout)
 
     def run(self) -> dict[str, object]:
-        """Navigate to the target, clear any residual state, return page info."""
+        """Navigate to the target and return page information."""
-        self.driver.delete_all_cookies()
+        # Each driver already owns a brand-new temporary profile. Do not issue
-
+        # renderer commands on Chrome's initial blank tab: on small servers that
-        # Clear localStorage / sessionStorage via JS
+        # renderer can stall before navigation is ever attempted.
-        self.driver.execute_script(
+        log.info("Navigating browser to %s", self.url)
-            "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
+        try:
+            navigation = cast(
+                dict[str, Any],
+                self.driver.execute_cdp_cmd(  # type: ignore[reportUnknownMemberType]
+                    "Page.navigate", {"url": self.url}
+                ),
             )
-
+            error_text = navigation.get("errorText")
-        if not self.url:
+            if error_text:
-            return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
+                raise FreshSessionError(f"Chrome rejected navigation: {error_text}")
-
+        except FreshSessionError:
+            raise
+        except WebDriverException:
+            # Older Chrome builds may not expose Page.navigate. Keep a normal
+            # WebDriver fallback, but only after navigation has been attempted.
             try:
                 self.driver.get(self.url)
             except TimeoutException:
-            log.warning(
+                log.warning("Page load timed out; stopping navigation")
-                "Page load exceeded the browser timeout; stopping navigation and continuing"
+                self.driver.execute_cdp_cmd(  # type: ignore[reportUnknownMemberType]
+                    "Page.stopLoading", {}
                 )
-            self.driver.execute_cdp_cmd("Page.stopLoading", {})
+
-        human_delay(2.0, 4.0)
+        try:
+            self.wait.until(lambda d: d.current_url not in _BLANK_URLS)
+        except TimeoutException as exc:
+            raise FreshSessionError(
+                f"navigation to {self.url} never left Chrome's blank page"
+            ) from exc
+
+        human_delay(1.0, 2.0)
 
         # Interactive is sufficient for the dynamic flow and avoids waiting on
         # analytics, ads, or other background resources indefinitely.
-        try:
+        def document_is_ready(driver: uc.Chrome) -> bool:
-            self.wait.until(
+            state = cast(
-                lambda d: d.execute_script("return document.readyState")
+                str,
-                in ("interactive", "complete")
+                driver.execute_script(  # type: ignore[reportUnknownMemberType]
+                    "return document.readyState"
+                ),
             )
+            return state in ("interactive", "complete")
+
+        try:
+            self.wait.until(document_is_ready)
         except TimeoutException:
             log.warning("Document did not report a ready state; continuing with the current DOM")
 
         title = self.driver.title
         current_url = self.driver.current_url
-        cookies = self.driver.get_cookies()
+        cookies = cast(
+            list[dict[str, Any]],
+            self.driver.get_cookies(),  # type: ignore[reportUnknownMemberType]
+        )
+        log.info("Browser reached %s", current_url)
 
         return {
             "title": title,

crawler/tasks.py

@@ -304,11 +304,16 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
 
         t0 = time.monotonic()
         identifier = f"batch-{batch_id[:8]}-{slot:04d}"
-        log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
+        target_url = str(cfg.get("target_url") or "").strip()
+        scenario: str = cfg.get("scenario") or "dynamic"
+        log.info(
+            "[batch:%s] slot %d starting — target=%s",
+            batch_id[:8],
+            slot,
+            target_url or "<missing>",
+        )
 
         try:
-            target_url: str = cfg.get("target_url") or ""
-            scenario: str = cfg.get("scenario") or "dynamic"
             with driver_session(headless=headless, recording_name=identifier) as driver:
                 FreshSessionScenario(driver, target_url).run()
                 if cfg: