nfel/crawle-snapp
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

Navigate before initializing page state

Browse Source
This commit is contained in:
nfel
2026-08-03 00:21:55 +03:30
parent 66c8c19e0c
commit e6ee587bf0
5 changed files with 119 additions and 36 deletions
Download Patch File Download Diff File Expand all files Collapse all files
admin/routes.py
+18 -1
View File
@@ -6,12 +6,13 @@ import secrets
from datetime import UTC, datetime
from pathlib import Path
from typing import Annotated, Any
from urllib.parse import urlsplit
import speedtest # type: ignore[import-untyped]
from fastapi import APIRouter, Depends, HTTPException, status
from fastapi.responses import FileResponse
from fastapi.security import HTTPBasic, HTTPBasicCredentials
from pydantic import BaseModel
from pydantic import BaseModel, field_validator
from admin.models import (
delete_flow_config,
@@ -91,6 +92,17 @@ class FlowConfigIn(BaseModel):
scenario: str = "dynamic"
stop_on_first_click: bool = False
@field_validator("target_url")
@classmethod
def validate_target_url(cls, value: str) -> str:
value = value.strip()
if value and "://" not in value:
value = f"https://{value}"
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise ValueError("Target URL must be a complete HTTP(S) URL")
return value
def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
return {
@@ -181,6 +193,11 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
raise HTTPException(
status_code=404, detail=f"Flow config {body.config_id} not found"
)
if not str(cfg.get("target_url") or "").strip():
raise HTTPException(
status_code=422,
detail="The selected flow config has no Target URL. Edit and save it first.",
)
result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
log.info(
admin/static/dashboard.html
+12 -3
View File
@@ -183,7 +183,7 @@
<div class="fg">
<label>Target URL</label>
<input id="cfg-target-url" type="url" placeholder="https://example.com" />
<input id="cfg-target-url" type="url" placeholder="https://example.com" required />
</div>
<div class="fg" style="display:flex;align-items:center;gap:10px;margin-bottom:14px">
@@ -1189,6 +1189,7 @@ document.getElementById('cfg-digipay-match').addEventListener('change', e => {
document.getElementById('form-save').addEventListener('click', async () => {
const body = collectForm();
if (!body.name) { alert('Config name is required.'); return; }
if (!body.target_url) { alert('Target URL is required.'); return; }
const cfgId = document.getElementById('cfg-id').value;
const isEdit = !!cfgId;
@@ -1408,6 +1409,11 @@ function startBatchPoll() {
document.getElementById('batch-start-btn').addEventListener('click', async () => {
const configId = parseInt(document.getElementById('batch-config-select').value);
if (!configId) { alert('Please select a flow config.'); return; }
const selectedConfig = _configs.find(c => c.id === configId);
if (!selectedConfig?.target_url) {
alert('The selected flow config has no Target URL. Edit and save it first.');
return;
}
const workers = parseInt(document.getElementById('batch-workers').value) || 3;
const runs = parseInt(document.getElementById('batch-runs').value) || 10;
const stagger = parseInt(document.getElementById('batch-stagger').value) || 0;
@@ -1420,8 +1426,11 @@ document.getElementById('batch-start-btn').addEventListener('click', async () =>
});
if (!res) return;
if (res.status === 404) { const d = await res.json(); alert(d.detail || 'Flow config not found.'); return; }
if (!res.ok) { alert('Failed to add task.'); return; }
if (!res.ok) {
const d = await res.json().catch(() => ({}));
alert(d.detail || 'Failed to add task.');
return;
}
const data = await res.json();
await pollBatchStatus();
crawler/batch.py
+7 -2
View File
@@ -27,9 +27,14 @@ def _single_run(
identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
t0 = time.monotonic()
log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
target_url = str(cfg.get("target_url") or "").strip()
log.info(
"[batch:%s] Run %d starting — target=%s",
batch_id[:8],
run_index,
target_url or "<missing>",
)
try:
target_url: str = cfg.get("target_url") or ""
with driver_session(headless=headless, recording_name=identifier) as driver:
FreshSessionScenario(driver, target_url).run()
result = DynamicFlow(driver, cfg).run() if cfg else None
crawler/scenarios/fresh_session.py
+74 -27
View File
@@ -1,8 +1,11 @@
"""Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
from __future__ import annotations
from typing import Any, cast
from urllib.parse import urlsplit
import undetected_chromedriver as uc
from selenium.common.exceptions import TimeoutException
from selenium.common.exceptions import TimeoutException, WebDriverException
from selenium.webdriver.support.ui import WebDriverWait
from crawler.driver import human_delay
@@ -10,56 +13,100 @@ from logger import get_logger
log = get_logger(__name__)
_BLANK_URLS = {"", "about:blank", "data:,"}
class FreshSessionError(RuntimeError):
"""The browser could not leave its initial blank page."""
def _normalise_url(url: str) -> str:
value = url.strip()
if value and "://" not in value:
value = f"https://{value}"
parsed = urlsplit(value)
if parsed.scheme not in {"http", "https"} or not parsed.netloc:
raise FreshSessionError(
"target_url must be a complete HTTP(S) URL, for example https://example.com"
)
return value
class FreshSessionScenario:
"""
Opens the target URL in a brand-new browser profile (no stored cookies,
localStorage, or cache) so each run appears as a first-time visitor.
Because undetected-chromedriver already creates an isolated temp profile
per instance, simply spinning up a new driver is sufficient — but we also
explicitly delete all cookies after loading to be safe.
Undetected-chromedriver creates an isolated temporary profile per instance,
so no cookie or storage clearing is needed before navigation.
"""
def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
self.driver = driver
self.url = url
self.url = _normalise_url(url)
self.wait = WebDriverWait(driver, timeout)
def run(self) -> dict[str, object]:
"""Navigate to the target, clear any residual state, return page info."""
self.driver.delete_all_cookies()
# Clear localStorage / sessionStorage via JS
self.driver.execute_script(
"try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
)
if not self.url:
return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
"""Navigate to the target and return page information."""
# Each driver already owns a brand-new temporary profile. Do not issue
# renderer commands on Chrome's initial blank tab: on small servers that
# renderer can stall before navigation is ever attempted.
log.info("Navigating browser to %s", self.url)
try:
navigation = cast(
dict[str, Any],
self.driver.execute_cdp_cmd( # type: ignore[reportUnknownMemberType]
"Page.navigate", {"url": self.url}
),
)
error_text = navigation.get("errorText")
if error_text:
raise FreshSessionError(f"Chrome rejected navigation: {error_text}")
except FreshSessionError:
raise
except WebDriverException:
# Older Chrome builds may not expose Page.navigate. Keep a normal
# WebDriver fallback, but only after navigation has been attempted.
try:
self.driver.get(self.url)
except TimeoutException:
log.warning("Page load timed out; stopping navigation")
self.driver.execute_cdp_cmd( # type: ignore[reportUnknownMemberType]
"Page.stopLoading", {}
)
try:
self.driver.get(self.url)
except TimeoutException:
log.warning(
"Page load exceeded the browser timeout; stopping navigation and continuing"
)
self.driver.execute_cdp_cmd("Page.stopLoading", {})
human_delay(2.0, 4.0)
self.wait.until(lambda d: d.current_url not in _BLANK_URLS)
except TimeoutException as exc:
raise FreshSessionError(
f"navigation to {self.url} never left Chrome's blank page"
) from exc
human_delay(1.0, 2.0)
# Interactive is sufficient for the dynamic flow and avoids waiting on
# analytics, ads, or other background resources indefinitely.
try:
self.wait.until(
lambda d: d.execute_script("return document.readyState")
in ("interactive", "complete")
def document_is_ready(driver: uc.Chrome) -> bool:
state = cast(
str,
driver.execute_script( # type: ignore[reportUnknownMemberType]
"return document.readyState"
),
)
return state in ("interactive", "complete")
try:
self.wait.until(document_is_ready)
except TimeoutException:
log.warning("Document did not report a ready state; continuing with the current DOM")
title = self.driver.title
current_url = self.driver.current_url
cookies = self.driver.get_cookies()
cookies = cast(
list[dict[str, Any]],
self.driver.get_cookies(), # type: ignore[reportUnknownMemberType]
)
log.info("Browser reached %s", current_url)
return {
"title": title,
crawler/tasks.py
+8 -3
View File
@@ -304,11 +304,16 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
t0 = time.monotonic()
identifier = f"batch-{batch_id[:8]}-{slot:04d}"
log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
target_url = str(cfg.get("target_url") or "").strip()
scenario: str = cfg.get("scenario") or "dynamic"
log.info(
"[batch:%s] slot %d starting — target=%s",
batch_id[:8],
slot,
target_url or "<missing>",
)
try:
target_url: str = cfg.get("target_url") or ""
scenario: str = cfg.get("scenario") or "dynamic"
with driver_session(headless=headless, recording_name=identifier) as driver:
FreshSessionScenario(driver, target_url).run()
if cfg: