diff --git a/admin/routes.py b/admin/routes.py
index 37949f3..b94ca4a 100644
--- a/admin/routes.py
+++ b/admin/routes.py
@@ -6,12 +6,13 @@ import secrets
from datetime import UTC, datetime
from pathlib import Path
from typing import Annotated, Any
+from urllib.parse import urlsplit
import speedtest # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.responses import FileResponse
from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
from admin.models import (
delete_flow_config,
@@ -91,6 +92,17 @@ class FlowConfigIn(BaseModel):
scenario: str = "dynamic"
stop_on_first_click: bool = False
+ @field_validator("target_url")
+ @classmethod
+ def validate_target_url(cls, value: str) -> str:
+ value = value.strip()
+ if value and "://" not in value:
+ value = f"https://{value}"
+ parsed = urlsplit(value)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ raise ValueError("Target URL must be a complete HTTP(S) URL")
+ return value
+
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
return {
@@ -181,6 +193,11 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
raise HTTPException(
status_code=404, detail=f"Flow config {body.config_id} not found"
)
+ if not str(cfg.get("target_url") or "").strip():
+ raise HTTPException(
+ status_code=422,
+ detail="The selected flow config has no Target URL. Edit and save it first.",
+ )
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
log.info(
diff --git a/admin/static/dashboard.html b/admin/static/dashboard.html
index 9e9d357..2de0f8c 100644
--- a/admin/static/dashboard.html
+++ b/admin/static/dashboard.html
@@ -183,7 +183,7 @@
-
+
@@ -1189,6 +1189,7 @@ document.getElementById('cfg-digipay-match').addEventListener('change', e => {
document.getElementById('form-save').addEventListener('click', async () => {
const body = collectForm();
if (!body.name) { alert('Config name is required.'); return; }
+ if (!body.target_url) { alert('Target URL is required.'); return; }
const cfgId = document.getElementById('cfg-id').value;
const isEdit = !!cfgId;
@@ -1408,6 +1409,11 @@ function startBatchPoll() {
document.getElementById('batch-start-btn').addEventListener('click', async () => {
const configId = parseInt(document.getElementById('batch-config-select').value);
if (!configId) { alert('Please select a flow config.'); return; }
+ const selectedConfig = _configs.find(c => c.id === configId);
+ if (!selectedConfig?.target_url) {
+ alert('The selected flow config has no Target URL. Edit and save it first.');
+ return;
+ }
const workers = parseInt(document.getElementById('batch-workers').value) || 3;
const runs = parseInt(document.getElementById('batch-runs').value) || 10;
const stagger = parseInt(document.getElementById('batch-stagger').value) || 0;
@@ -1420,8 +1426,11 @@ document.getElementById('batch-start-btn').addEventListener('click', async () =>
});
if (!res) return;
- if (res.status === 404) { const d = await res.json(); alert(d.detail || 'Flow config not found.'); return; }
- if (!res.ok) { alert('Failed to add task.'); return; }
+ if (!res.ok) {
+ const d = await res.json().catch(() => ({}));
+ alert(d.detail || 'Failed to add task.');
+ return;
+ }
const data = await res.json();
await pollBatchStatus();
diff --git a/crawler/batch.py b/crawler/batch.py
index 79ef517..5286648 100644
--- a/crawler/batch.py
+++ b/crawler/batch.py
@@ -27,9 +27,14 @@ def _single_run(
identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
t0 = time.monotonic()
- log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+ target_url = str(cfg.get("target_url") or "").strip()
+ log.info(
+ "[batch:%s] Run %d starting — target=%s",
+ batch_id[:8],
+ run_index,
+ target_url or "",
+ )
try:
- target_url: str = cfg.get("target_url") or ""
with driver_session(headless=headless, recording_name=identifier) as driver:
FreshSessionScenario(driver, target_url).run()
result = DynamicFlow(driver, cfg).run() if cfg else None
diff --git a/crawler/scenarios/fresh_session.py b/crawler/scenarios/fresh_session.py
index a48f019..ad86d09 100644
--- a/crawler/scenarios/fresh_session.py
+++ b/crawler/scenarios/fresh_session.py
@@ -1,8 +1,11 @@
"""Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
from __future__ import annotations
+from typing import Any, cast
+from urllib.parse import urlsplit
+
import undetected_chromedriver as uc
-from selenium.common.exceptions import TimeoutException
+from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
@@ -10,56 +13,100 @@ from logger import get_logger
log = get_logger(__name__)
+_BLANK_URLS = {"", "about:blank", "data:,"}
+
+
+class FreshSessionError(RuntimeError):
+ """The browser could not leave its initial blank page."""
+
+
+def _normalise_url(url: str) -> str:
+ value = url.strip()
+ if value and "://" not in value:
+ value = f"https://{value}"
+ parsed = urlsplit(value)
+ if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+ raise FreshSessionError(
+ "target_url must be a complete HTTP(S) URL, for example https://example.com"
+ )
+ return value
+
class FreshSessionScenario:
"""
Opens the target URL in a brand-new browser profile (no stored cookies,
localStorage, or cache) so each run appears as a first-time visitor.
- Because undetected-chromedriver already creates an isolated temp profile
- per instance, simply spinning up a new driver is sufficient — but we also
- explicitly delete all cookies after loading to be safe.
+ Undetected-chromedriver creates an isolated temporary profile per instance,
+ so no cookie or storage clearing is needed before navigation.
"""
def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
self.driver = driver
- self.url = url
+ self.url = _normalise_url(url)
self.wait = WebDriverWait(driver, timeout)
def run(self) -> dict[str, object]:
- """Navigate to the target, clear any residual state, return page info."""
- self.driver.delete_all_cookies()
-
- # Clear localStorage / sessionStorage via JS
- self.driver.execute_script(
- "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
- )
-
- if not self.url:
- return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
+ """Navigate to the target and return page information."""
+ # Each driver already owns a brand-new temporary profile. Do not issue
+ # renderer commands on Chrome's initial blank tab: on small servers that
+ # renderer can stall before navigation is ever attempted.
+ log.info("Navigating browser to %s", self.url)
+ try:
+ navigation = cast(
+ dict[str, Any],
+ self.driver.execute_cdp_cmd( # type: ignore[reportUnknownMemberType]
+ "Page.navigate", {"url": self.url}
+ ),
+ )
+ error_text = navigation.get("errorText")
+ if error_text:
+ raise FreshSessionError(f"Chrome rejected navigation: {error_text}")
+ except FreshSessionError:
+ raise
+ except WebDriverException:
+ # Older Chrome builds may not expose Page.navigate. Keep a normal
+ # WebDriver fallback, but only after navigation has been attempted.
+ try:
+ self.driver.get(self.url)
+ except TimeoutException:
+ log.warning("Page load timed out; stopping navigation")
+ self.driver.execute_cdp_cmd( # type: ignore[reportUnknownMemberType]
+ "Page.stopLoading", {}
+ )
try:
- self.driver.get(self.url)
- except TimeoutException:
- log.warning(
- "Page load exceeded the browser timeout; stopping navigation and continuing"
- )
- self.driver.execute_cdp_cmd("Page.stopLoading", {})
- human_delay(2.0, 4.0)
+ self.wait.until(lambda d: d.current_url not in _BLANK_URLS)
+ except TimeoutException as exc:
+ raise FreshSessionError(
+ f"navigation to {self.url} never left Chrome's blank page"
+ ) from exc
+
+ human_delay(1.0, 2.0)
# Interactive is sufficient for the dynamic flow and avoids waiting on
# analytics, ads, or other background resources indefinitely.
- try:
- self.wait.until(
- lambda d: d.execute_script("return document.readyState")
- in ("interactive", "complete")
+ def document_is_ready(driver: uc.Chrome) -> bool:
+ state = cast(
+ str,
+ driver.execute_script( # type: ignore[reportUnknownMemberType]
+ "return document.readyState"
+ ),
)
+ return state in ("interactive", "complete")
+
+ try:
+ self.wait.until(document_is_ready)
except TimeoutException:
log.warning("Document did not report a ready state; continuing with the current DOM")
title = self.driver.title
current_url = self.driver.current_url
- cookies = self.driver.get_cookies()
+ cookies = cast(
+ list[dict[str, Any]],
+ self.driver.get_cookies(), # type: ignore[reportUnknownMemberType]
+ )
+ log.info("Browser reached %s", current_url)
return {
"title": title,
diff --git a/crawler/tasks.py b/crawler/tasks.py
index 16bcbdb..6d50c1a 100644
--- a/crawler/tasks.py
+++ b/crawler/tasks.py
@@ -304,11 +304,16 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
t0 = time.monotonic()
identifier = f"batch-{batch_id[:8]}-{slot:04d}"
- log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
+ target_url = str(cfg.get("target_url") or "").strip()
+ scenario: str = cfg.get("scenario") or "dynamic"
+ log.info(
+ "[batch:%s] slot %d starting — target=%s",
+ batch_id[:8],
+ slot,
+ target_url or "",
+ )
try:
- target_url: str = cfg.get("target_url") or ""
- scenario: str = cfg.get("scenario") or "dynamic"
with driver_session(headless=headless, recording_name=identifier) as driver:
FreshSessionScenario(driver, target_url).run()
if cfg: