Remove all third-party apt sources before apt-get update

The Playwright image ships with NodeSource, GitHub CLI, and other
third-party apt sources that all time out from Iran. rm -rf the
entire sources.list.d directory, then add back only the local
Ubuntu mirror (ubuntu.parsvds.com).

Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>

.claude/settings.local.json

@@ -0,0 +1,32 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(python -m venv .venv)",
+      "Bash(.venv/bin/pip install *)",
+      "Read(//home/sana/.config/uv/**)",
+      "Read(//home/sana/.pip/**)",
+      "Bash(make admin *)",
+      "Bash(curl -s -o /dev/null -w \"%{http_code}\" http://127.0.0.1:8000/)",
+      "Bash(kill %1)",
+      "Bash(uv run *)",
+      "Bash(curl -sv http://127.0.0.1:8000/)",
+      "Bash(no_proxy=\"127.0.0.1\" curl -s -o /dev/null -w \"%{http_code}\" http://127.0.0.1:8000/)",
+      "Bash(echo \"exit: $?\")",
+      "Bash(google-chrome --version)",
+      "Bash(google-chrome-stable --version)",
+      "Bash(chromium --version)",
+      "Bash(make patch-driver *)",
+      "Bash(rtk ls *)",
+      "Bash(./drivers/chromedriver *)",
+      "Bash(rtk grep *)",
+      "Bash(rtk git *)",
+      "Bash(python *)",
+      "Bash(rtk read *)",
+      "Bash(uv add *)",
+      "Bash(wait)",
+      "Bash(fuser -k 8000/tcp)",
+      "Bash(fuser -k 9000/tcp)",
+      "Bash(rtk find *)"
+    ]
+  }
+}

.dockerignore

@@ -0,0 +1,20 @@
+.git
+.gitignore
+.env
+.env.*
+!.env.example
+.venv
+venv
+__pycache__
+*.py[cod]
+.pytest_cache
+.mypy_cache
+.ruff_cache
+.claude
+.agents
+.codex
+data
+logs
+drivers
+*.db
+*.swp

.env.example

@@ -18,6 +18,12 @@ HEADLESS=true
 # CHROME_BINARY=/usr/bin/google-chrome-stable  # optional: explicit Chrome binary path
 # CHROMEDRIVER_PATH=drivers/chromedriver       # pre-patched driver (run: make patch-driver)
 
+# Worker browser-session recordings (Docker Compose enables these by default)
+RECORD_SESSIONS=false
+RECORDING_FPS=12
+RECORDING_MAX_FILES=3
+RECORDINGS_DIR=recordings
+
 # Scenario 1 — comma-separated phone numbers
 PHONE_NUMBERS=+989100000001,+989100000002,+989100000003
 

.qwen/settings.json

@@ -0,0 +1,10 @@
+{
+  "permissions": {
+    "allow": [
+      "Bash(git add *)",
+      "Bash(git commit *)",
+      "Bash(git push *)"
+    ]
+  },
+  "$version": 4
+}

.qwen/skills/auto-skill-playwright-base-docker/SKILL.md

@@ -0,0 +1,83 @@
+---
+name: playwright-base-docker
+description: Use mcr.microsoft.com/playwright as a pre-baked Chrome base image for Dockerized crawler/automation projects
+source: auto-skill
+extracted_at: '2026-08-04T18:21:03.395Z'
+---
+
+## Problem
+
+Dockerizing headless Chrome automation requires Chromium, matching ChromeDriver, Xvfb, ffmpeg, and browser patching — which bloats the image and adds build steps.
+
+## Approach
+
+Use `mcr.microsoft.com/playwright:<version>-noble` (or similar variant) as the base image for worker/crawler services. Playwright ships with Chromium, ChromeDriver, and common browser automation dependencies pre-installed and pre-patched.
+
+### Key steps
+
+1. **Define multi-stage Dockerfile with separate targets**
+   - `admin` target: lightweight `python:3.11-slim` (no Chrome needed for FastAPI)
+   - `worker` target: `mcr.microsoft.com/playwright:v1.62.0-noble` (Chrome + deps included)
+
+2. **Do NOT copy `.venv` from a different Python version builder stage**
+   - The Playwright Noble image ships **Python 3.12** while `python:3.11-slim` ships 3.11.
+   - Copying a 3.11-built venv into the image produces **broken shebangs** — `exec /app/.venv/bin/rq: no such file or directory` (the binaries exist but their `#!` line points to a non-existent Python 3.11).
+   - **Fix**: install `uv` inside the worker stage and run `uv sync` natively so the venv targets the image's Python.
+
+3. **Installing `uv` in the Playwright image**
+   - Playwright ships Python but **not** `pip` or `uv` in PATH.
+   - **Working approach**:
+     ```dockerfile
+     RUN python -m ensurepip --upgrade \
+         && python -m pip install --no-cache-dir --break-system-packages uv==0.11.31 \
+         && uv sync --frozen --no-dev --no-install-project
+     ```
+   - `ensurepip` bootstraps pip in the same shell, then `python -m pip` installs the `uv` CLI into `/usr/local/bin/uv` (already in PATH).
+   - Do **NOT** try to set `ENV PATH` in a separate layer before calling `uv` — the install script doesn't modify PATH, and the binary won't be found.
+   - Avoid `curl -LsSf https://astral.sh/uv/install.sh | sh` in a RUN followed by `uv` in the next — the install script is a one-shot that doesn't persist PATH.
+
+4. **Set environment variables** for Chrome location and headless mode:
+   ```dockerfile
+   ENV CHROME_BINARY=/usr/bin/chromium \
+       CHROMEDRIVER_PATH=/usr/bin/chromedriver \
+       HEADLESS=true
+   ```
+
+5. **Skip `patch_driver.py`** — Playwright's Chromium is already pre-patched with undetected-chromedriver-friendly modifications.
+
+6. **Update docker-compose.yml** with explicit build targets:
+   ```yaml
+   admin:
+     build:
+       context: .
+       dockerfile: Dockerfile
+       target: admin
+   worker:
+     build:
+       context: .
+       dockerfile: Dockerfile
+       target: worker
+   ```
+
+### Common pitfalls
+
+- **`//` in `FROM` lines doesn't work**: `FROM oci.reg.darano.ir//mcr.microsoft.com/playwright:latest AS worker` will fail with `invalid reference format`. The Docker parser confuses `//` with the `AS` syntax. If your registry requires `//` for internal routing, configure it at the **Docker daemon level** in `/etc/docker/daemon.json` under `registry-mirrors`, not in the Dockerfile.
+- **Don't copy venv from python:3.11-slim to Playwright**: Python version mismatch → broken shebangs. Always build the venv natively in the target stage.
+- **Don't rely on `pip` being available**: Playwright ships Python without pip. Use `python -m ensurepip` first.
+
+## What you still need
+
+- **ffmpeg** and **Xvfb**: The Playwright image includes these for session recording. If you need them for headless mode only (no recording), they are optional.
+- **security_opt**: Workers still need `cap_add: [SYS_ADMIN]` and `security_opt: [seccomp:unconfined]` for undetected-chromedriver to fully spoof browser fingerprints.
+- **shm_size**: Keep `shm_size: "2gb"` on worker containers to prevent Chrome OOM crashes.
+
+## Files changed
+
+- `Dockerfile` — split into `admin` and `worker` targets
+- `docker-compose.yml` — explicit `target` per service, separate image names
+
+## Why this matters
+
+- **Faster builds** — no Chromium download, no ChromeDriver version matching, no patching script
+- **Smaller admin image** — admin panel doesn't need any browser dependencies
+- **Reliable driver matching** — Playwright maintains its own bundled Chromium/ChromeDriver pair, eliminating version mismatches

Dockerfile

@@ -1,60 +1,108 @@
+# Usage:
+#   make build              # Build all Docker images (admin + worker)
+#   docker compose up -d    # Start all services
+#
+# - admin stage: python:3.11-slim (FastAPI only, no Chrome)
+# - worker stage: Playwright Noble with Python + Chromium pre-installed
+#   from your local registry mirror.
+
+# ── Build args ─────────────────────────────────────────────────────────────────
+ARG BASE_IMAGE=python:3.11-slim
+
 # ── Stage 1: dependency resolver ─────────────────────────────────────────────
 FROM python:3.11-slim AS builder
 
-# Install uv
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
-
 WORKDIR /app
 
-# Copy only dependency manifests first (cache layer)
+# python:3.11-slim has neither curl nor pip. Bootstrap pip via ensurepip,
+# then use it to install uv from the local PyPI mirror.
+RUN python -m ensurepip --upgrade \
+    && python -m pip install --break-system-packages \
+        --index-url https://pypi.reg.darano.ir/simple/ \
+        --trusted-host pypi.reg.darano.ir \
+        uv
+
+ENV PATH="/usr/local/bin:$PATH" \
+    UV_DEFAULT_INDEX="https://pypi.reg.darano.ir"
+
 COPY pyproject.toml uv.lock* ./
 
-# Install deps into an isolated prefix so we can copy them cleanly
 RUN uv sync --frozen --no-dev --no-install-project
 
-# ── Stage 2: runtime ──────────────────────────────────────────────────────────
-FROM python:3.11-slim AS runtime
-
-# ── Chrome + system deps ──────────────────────────────────────────────────────
-RUN apt-get update && apt-get install -y --no-install-recommends \
-        wget gnupg ca-certificates curl unzip \
-        # X11 / rendering libs needed even in headless mode
-        libnss3 libatk1.0-0 libatk-bridge2.0-0 libcups2 libdrm2 \
-        libxkbcommon0 libxcomposite1 libxdamage1 libxfixes3 libxrandr2 \
-        libgbm1 libasound2 libpango-1.0-0 libcairo2 libx11-xcb1 \
-    && wget -qO- https://dl.google.com/linux/linux_signing_key.pub \
-        | gpg --dearmor -o /usr/share/keyrings/google-chrome.gpg \
-    && echo "deb [arch=amd64 signed-by=/usr/share/keyrings/google-chrome.gpg] \
-        http://dl.google.com/linux/chrome/deb/ stable main" \
-        > /etc/apt/sources.list.d/google-chrome.list \
-    && apt-get update \
-    && apt-get install -y --no-install-recommends google-chrome-stable \
-    && rm -rf /var/lib/apt/lists/*
-
-# ── uv + venv from builder ────────────────────────────────────────────────────
-COPY --from=ghcr.io/astral-sh/uv:latest /uv /usr/local/bin/uv
-COPY --from=builder /app/.venv /app/.venv
+# ── Stage 2a: admin runtime (python:3.11-slim, no Chrome needed) ──────────────
+FROM ${BASE_IMAGE} AS admin
 
 WORKDIR /app
 
-# Make the venv's binaries the default Python
 ENV PATH="/app/.venv/bin:$PATH" \
     PYTHONUNBUFFERED=1 \
     PYTHONDONTWRITEBYTECODE=1 \
-    # Tell undetected-chromedriver where Chrome lives
-    CHROME_BINARY=/usr/bin/google-chrome-stable \
-    # Always run headless inside Docker
-    HEADLESS=true
+    UV_DEFAULT_INDEX="https://pypi.reg.darano.ir"
 
-# ── Application code ──────────────────────────────────────────────────────────
 COPY . .
+COPY --from=builder /app/.venv /app/.venv
 
-RUN mkdir -p data logs
+RUN mkdir -p data logs recordings \
+    && useradd -m -u 1001 seed \
+    && chown -R seed:seed /app
 
-# Non-root user for safety
-RUN useradd -m -u 1001 seed && chown -R seed:seed /app
 USER seed
 
 EXPOSE 8000
 
 CMD ["python", "main.py", "admin"]
+
+# ── Stage 2b: worker runtime (Playwright Noble + Python installed) ─────────────
+# Base: oci.reg.darano.ir/mcr.microsoft.com/playwright:v1.62.0-noble
+# This image ships: Ubuntu 24.04 + Chromium + Firefox + WebKit + system libs.
+# We add Python + uv ourselves.
+
+FROM oci.reg.darano.ir/mcr.microsoft.com/playwright:v1.62.0-noble AS worker
+
+# Remove ALL third-party apt sources before running apt-get update.
+# The Playwright image ships with NodeSource, GitHub CLI, etc. that
+# all time out from our VPS location (Iran). Only keep Ubuntu repos.
+RUN rm -rf /etc/apt/sources.list.d/*
+
+# Install Python 3.12 + uv, then sync deps — all in one RUN.
+# Use a mirror for apt since the VPS has network restrictions (Iran).
+RUN printf 'Types: deb\nURIs: http://ubuntu.parsvds.com/ubuntu/\nSuites: noble noble-updates noble-security\nComponents: main restricted universe multiverse\nSigned-By: /usr/share/keyrings/ubuntu-archive-keyring.gpg\n' \
+      > /etc/apt/sources.list.d/ubuntu.sources \
+    && apt-get update && apt-get install -y --no-install-recommends \
+      python3 python3-venv \
+    && rm -rf /var/lib/apt/lists/* \
+    && curl -LsSf https://astral.sh/uv/install.sh | sh \
+    && . /usr/local/env 2>/dev/null || true \
+    || pip install --break-system-packages \
+        --index-url https://pypi.reg.darano.ir/simple/ \
+        --trusted-host pypi.reg.darano.ir \
+        uv
+
+ENV PATH="/usr/local/bin:$PATH" \
+    UV_DEFAULT_INDEX="https://pypi.reg.darano.ir" \
+    PIP_INDEX_URL="https://pypi.reg.darano.ir/simple/" \
+    # Chromium is pre-installed at /usr/bin/chromium
+    CHROME_BINARY=/usr/bin/chromium \
+    CHROMEDRIVER_PATH=/usr/bin/chromedriver \
+    HEADLESS=true
+
+WORKDIR /app
+
+# Copy manifests first (stable cache layer)
+COPY pyproject.toml uv.lock* ./
+
+# Install project deps into a venv targeting this image's Python 3.12
+RUN uv sync --frozen --no-dev --no-install-project
+
+# Copy app code
+COPY . .
+
+# mkdir for volumes, user for security
+RUN mkdir -p data logs recordings \
+    && useradd -m -u 1001 seed 2>/dev/null || true \
+    && chown -R seed:seed /app 2>/dev/null || true
+
+USER seed
+
+# CMD overridden in docker-compose.yml to use RQ worker
+CMD ["rq", "worker", "--url", "redis://redis:6379/0", "batch"]

Makefile

@@ -85,10 +85,13 @@ batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500)
 
 # ── Docker ────────────────────────────────────────────────────────────────────
 
-build: ## Build the Docker image
+build: ## Build all Docker images (admin + worker)
 	docker compose build
 
-up: ## Start admin panel in Docker (detached)
+worker-build: ## Rebuild worker with Playwright base image
+	docker compose build worker
+
+up: ## Start all services in Docker (detached)
 	docker compose up -d
 	@echo "$(GREEN)Admin panel:$(RESET) http://localhost:8000"
 
@@ -101,16 +104,19 @@ logs: ## Follow container logs
 shell: ## Open a shell in the running admin container
 	docker compose exec admin bash
 
-# Crawler one-shots inside Docker
+# Worker one-off commands (override the default RQ worker command)
 docker-otp: ## Run OTP scenario in Docker (PHONE=+98...)
 ifdef PHONE
-	docker compose run --rm crawler python main.py otp --phone $(PHONE)
+	docker compose run --rm -e PHONE=$(PHONE) worker python main.py otp --phone $(PHONE)
 else
-	docker compose run --rm crawler python main.py otp
+	docker compose run --rm worker python main.py otp
 endif
 
 docker-fresh: ## Run fresh-session scenario in Docker
-	docker compose run --rm crawler python main.py fresh
+	docker compose run --rm worker python main.py fresh
+
+docker-admin-up: ## Start only the admin panel in Docker (detached)
+	docker compose up -d admin
 
 # ── Housekeeping ──────────────────────────────────────────────────────────────
 

README.md

@@ -169,6 +169,9 @@ make down
 ```
 
 The SQLite database is stored in the `seed-data` Docker volume and persists across rebuilds.
+Worker browser sessions are recorded at 12 FPS into the shared `seed-recordings`
+volume. The admin panel's **Recordings** tab lists and downloads the newest three
+completed sessions.
 
 ---
 
@@ -186,6 +189,10 @@ Copy `.env.example` to `.env` and set values before running anything.
 | `DB_PATH` | `data/tracker.db` | Path to the SQLite file |
 | `HEADLESS` | `true` | Set to `false` to watch the browser (local only) |
 | `CHROME_BINARY` | _(auto)_ | Explicit path to Chrome, e.g. `/usr/bin/google-chrome-stable` |
+| `RECORD_SESSIONS` | `false` | Record browser sessions; Docker workers override this to `true` |
+| `RECORDING_FPS` | `12` | Recording frame rate |
+| `RECORDING_MAX_FILES` | `3` | Number of completed recordings retained across all workers |
+| `RECORDINGS_DIR` | `recordings` | Recording directory; Docker uses the shared `/app/recordings` volume |
 | `PHONE_NUMBERS` | _(empty)_ | Comma-separated list of phone numbers for Scenario 1 |
 | `FLOW_STEPS` | `step_home` | Comma-separated step names for the legacy step-registry runner |
 

admin/main.py

@@ -30,8 +30,9 @@ def on_startup() -> None:
     if os.environ.get("SKIP_INTERNAL_WORKER", "").lower() in ("1", "true", "yes"):
         return
     redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+    rq_executable = str(Path(sys.executable).with_name("rq"))
     _rq_worker = subprocess.Popen(
-        [sys.executable, "-m", "rq", "worker", "--url", redis_url, "batch"],
+        [rq_executable, "worker", "--url", redis_url, "batch"],
         stdout=subprocess.DEVNULL,
         stderr=subprocess.DEVNULL,
     )
@@ -50,3 +51,8 @@ def on_shutdown() -> None:
 @app.get("/")
 def dashboard() -> FileResponse:
     return FileResponse(str(_STATIC / "dashboard.html"))
+
+
+@app.get("/favicon.ico", include_in_schema=False)
+def favicon() -> FileResponse:
+    return FileResponse(str(_STATIC / "favicon.svg"), media_type="image/svg+xml")

admin/routes.py

@@ -3,12 +3,16 @@
 from __future__ import annotations
 
 import secrets
+from datetime import UTC, datetime
+from pathlib import Path
 from typing import Annotated, Any
+from urllib.parse import urlsplit
 
 import speedtest  # type: ignore[import-untyped]
 from fastapi import APIRouter, Depends, HTTPException, status
+from fastapi.responses import FileResponse
 from fastapi.security import HTTPBasic, HTTPBasicCredentials
-from pydantic import BaseModel
+from pydantic import BaseModel, field_validator
 
 from admin.models import (
     delete_flow_config,
@@ -88,6 +92,17 @@ class FlowConfigIn(BaseModel):
     scenario: str = "dynamic"
     stop_on_first_click: bool = False
 
+    @field_validator("target_url")
+    @classmethod
+    def validate_target_url(cls, value: str) -> str:
+        value = value.strip()
+        if value and "://" not in value:
+            value = f"https://{value}"
+        parsed = urlsplit(value)
+        if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+            raise ValueError("Target URL must be a complete HTTP(S) URL")
+        return value
+
 
 def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]:
     return {
@@ -178,6 +193,11 @@ async def add_to_batch_queue(body: BatchTaskIn, _: Auth) -> dict[str, object]:
         raise HTTPException(
             status_code=404, detail=f"Flow config {body.config_id} not found"
         )
+    if not str(cfg.get("target_url") or "").strip():
+        raise HTTPException(
+            status_code=422,
+            detail="The selected flow config has no Target URL. Edit and save it first.",
+        )
 
     result = enqueue(cfg, body.workers, body.total_runs, body.stagger_ms, body.headless)
     log.info(
@@ -232,6 +252,63 @@ async def list_batch_runs(_: Auth) -> list[dict[str, object]]:
     return await batch_runs_repo.list()  # type: ignore[return-value]
 
 
+# ---------------------------------------------------------------------------
+# Browser-session recordings
+# ---------------------------------------------------------------------------
+
+
+def _recording_path(filename: str) -> Path:
+    if filename != Path(filename).name or Path(filename).suffix.lower() != ".mp4":
+        raise HTTPException(status_code=404, detail="Recording not found")
+    directory = Path(config.recordings_dir).resolve()
+    path = directory / filename
+    if path.is_symlink() or not path.is_file():
+        raise HTTPException(status_code=404, detail="Recording not found")
+    return path
+
+
+@router.get("/api/recordings")
+def list_recordings(_: Auth) -> list[dict[str, object]]:
+    directory = Path(config.recordings_dir)
+    if not directory.is_dir():
+        return []
+
+    rows: list[dict[str, object]] = []
+    for path in directory.glob("*.mp4"):
+        if path.is_symlink():
+            continue
+        try:
+            stat_result = path.stat()
+        except FileNotFoundError:
+            continue
+        rows.append(
+            {
+                "name": path.name,
+                "size_bytes": stat_result.st_size,
+                "created_at": datetime.fromtimestamp(
+                    stat_result.st_mtime, tz=UTC
+                ).isoformat(),
+            }
+        )
+    rows.sort(key=lambda row: str(row["created_at"]), reverse=True)
+    return rows
+
+
+@router.get("/api/recordings/{filename}")
+def download_recording(filename: str, _: Auth) -> FileResponse:
+    path = _recording_path(filename)
+    return FileResponse(path, media_type="video/mp4", filename=path.name)
+
+
+@router.delete("/api/recordings/{filename}", status_code=204)
+def delete_recording(filename: str, _: Auth) -> None:
+    path = _recording_path(filename)
+    try:
+        path.unlink()
+    except FileNotFoundError:
+        raise HTTPException(status_code=404, detail="Recording not found") from None
+
+
 # ---------------------------------------------------------------------------
 # Log viewer
 # ---------------------------------------------------------------------------

admin/static/dashboard.html

@@ -4,6 +4,7 @@
   <meta charset="UTF-8" />
   <meta name="viewport" content="width=device-width, initial-scale=1.0" />
   <title>Seed — Admin</title>
+  <link rel="icon" href="/static/favicon.svg" type="image/svg+xml" />
   <link rel="preconnect" href="https://fonts.googleapis.com" />
   <link rel="preconnect" href="https://fonts.gstatic.com" crossorigin />
   <link href="https://fonts.googleapis.com/css2?family=Vazirmatn:wght@300;400;500;600;700&display=swap" rel="stylesheet" />
@@ -183,7 +184,7 @@
 
     <div class="fg">
       <label>Target URL</label>
-      <input id="cfg-target-url" type="url" placeholder="https://example.com" />
+      <input id="cfg-target-url" type="url" placeholder="https://example.com" required />
     </div>
 
     <div class="fg" style="display:flex;align-items:center;gap:10px;margin-bottom:14px">
@@ -348,6 +349,7 @@
   <div class="tab active" data-tab="dashboard">Dashboard</div>
   <div class="tab" data-tab="flows">Flow Configs</div>
   <div class="tab" data-tab="batch">Batch Run</div>
+  <div class="tab" data-tab="recordings">Recordings</div>
   <div class="tab" data-tab="logs">Logs</div>
 </div>
 
@@ -589,6 +591,29 @@
 </main>
 </div>
 
+<!-- ── Recordings Tab ──────────────────────────────────────────── -->
+<div class="tab-panel" id="tab-recordings">
+<main>
+  <div class="panel" style="max-width:1100px">
+    <div style="display:flex;align-items:center;gap:12px;margin-bottom:14px">
+      <div>
+        <div class="section-title" style="margin:0">Browser Session Recordings</div>
+        <div style="color:var(--muted);font-size:12px;margin-top:4px">The newest three completed worker sessions are retained at 12 FPS.</div>
+      </div>
+      <button class="btn" onclick="loadRecordings()" style="margin-left:auto;padding:5px 14px;font-size:12px">↻ Refresh</button>
+    </div>
+    <div style="overflow-x:auto">
+      <table>
+        <thead><tr><th>Recorded</th><th>Session</th><th>Size</th><th>Actions</th></tr></thead>
+        <tbody id="recordings-tbody">
+          <tr><td colspan="4" class="empty">No recordings yet.</td></tr>
+        </tbody>
+      </table>
+    </div>
+  </div>
+</main>
+</div>
+
 <!-- ── Logs Tab ───────────────────────────────────────────────── -->
 <div class="tab-panel" id="tab-logs">
 <main>
@@ -712,6 +737,7 @@ document.querySelectorAll('.tab').forEach(tab => {
     document.getElementById(`tab-${tab.dataset.tab}`).classList.add('active');
     if (tab.dataset.tab === 'flows') loadConfigs();
     if (tab.dataset.tab === 'batch') { loadConfigs(); pollBatchStatus(); loadBatchHistory(); }
+    if (tab.dataset.tab === 'recordings') loadRecordings();
     if (tab.dataset.tab === 'logs') { loadLogs(); _startLogPoll(); }
     else _stopLogPoll();
   });
@@ -1164,6 +1190,7 @@ document.getElementById('cfg-digipay-match').addEventListener('change', e => {
 document.getElementById('form-save').addEventListener('click', async () => {
   const body = collectForm();
   if (!body.name) { alert('Config name is required.'); return; }
+  if (!body.target_url) { alert('Target URL is required.'); return; }
 
   const cfgId = document.getElementById('cfg-id').value;
   const isEdit = !!cfgId;
@@ -1383,6 +1410,11 @@ function startBatchPoll() {
 document.getElementById('batch-start-btn').addEventListener('click', async () => {
   const configId = parseInt(document.getElementById('batch-config-select').value);
   if (!configId) { alert('Please select a flow config.'); return; }
+  const selectedConfig = _configs.find(c => c.id === configId);
+  if (!selectedConfig?.target_url) {
+    alert('The selected flow config has no Target URL. Edit and save it first.');
+    return;
+  }
   const workers  = parseInt(document.getElementById('batch-workers').value) || 3;
   const runs     = parseInt(document.getElementById('batch-runs').value) || 10;
   const stagger  = parseInt(document.getElementById('batch-stagger').value) || 0;
@@ -1395,8 +1427,11 @@ document.getElementById('batch-start-btn').addEventListener('click', async () =>
   });
 
   if (!res) return;
-  if (res.status === 404) { const d = await res.json(); alert(d.detail || 'Flow config not found.'); return; }
-  if (!res.ok) { alert('Failed to add task.'); return; }
+  if (!res.ok) {
+    const d = await res.json().catch(() => ({}));
+    alert(d.detail || 'Failed to add task.');
+    return;
+  }
 
   const data = await res.json();
   await pollBatchStatus();
@@ -1421,6 +1456,56 @@ async function loadBatchHistory() {
   renderBatchHistory(await res.json());
 }
 
+/* ─────────────────────────── Recordings ─────────────────────── */
+function fmtBytes(bytes) {
+  if (!bytes) return '0 B';
+  const units = ['B', 'KB', 'MB', 'GB'];
+  const index = Math.min(Math.floor(Math.log(bytes) / Math.log(1024)), units.length - 1);
+  return `${(bytes / Math.pow(1024, index)).toFixed(index ? 1 : 0)} ${units[index]}`;
+}
+
+async function loadRecordings() {
+  const res = await apiFetch('/api/recordings');
+  if (!res || !res.ok) return;
+  const rows = await res.json();
+  const tbody = document.getElementById('recordings-tbody');
+  if (!rows.length) {
+    tbody.innerHTML = '<tr><td colspan="4" class="empty">No recordings yet.</td></tr>';
+    return;
+  }
+  tbody.innerHTML = rows.map(row => {
+    const encodedName = encodeURIComponent(row.name);
+    return `<tr>
+      <td style="white-space:nowrap;color:var(--muted)">${fmtTime(row.created_at)}</td>
+      <td style="font-family:monospace;font-size:12px">${esc(row.name)}</td>
+      <td style="white-space:nowrap;color:var(--muted)">${fmtBytes(row.size_bytes)}</td>
+      <td style="white-space:nowrap">
+        <button class="btn" onclick="downloadRecording('${encodedName}')" style="padding:4px 10px;font-size:12px">↓ Download</button>
+        <button class="btn btn-danger btn-sm" onclick="deleteRecording('${encodedName}')">Delete</button>
+      </td>
+    </tr>`;
+  }).join('');
+}
+
+async function downloadRecording(encodedName) {
+  const res = await apiFetch(`/api/recordings/${encodedName}`);
+  if (!res || !res.ok) return;
+  const blobUrl = URL.createObjectURL(await res.blob());
+  const link = document.createElement('a');
+  link.href = blobUrl;
+  link.download = decodeURIComponent(encodedName);
+  document.body.appendChild(link);
+  link.click();
+  link.remove();
+  URL.revokeObjectURL(blobUrl);
+}
+
+async function deleteRecording(encodedName) {
+  if (!confirm(`Delete recording ${decodeURIComponent(encodedName)}?`)) return;
+  const res = await apiFetch(`/api/recordings/${encodedName}`, { method: 'DELETE' });
+  if (res && res.ok) await loadRecordings();
+}
+
 /* ─────────────────────────── Logs ──────────────────────────── */
 let _logPollTimer = null;
 

admin/static/favicon.svg

@@ -0,0 +1,7 @@
+<svg xmlns="http://www.w3.org/2000/svg" viewBox="0 0 64 64">
+  <rect width="64" height="64" rx="14" fill="#6366f1"/>
+  <path d="M32 49V27" fill="none" stroke="#fff" stroke-width="5" stroke-linecap="round"/>
+  <path d="M31 30C19 30 12 23 13 13c11-1 19 5 18 17Z" fill="#a7f3d0"/>
+  <path d="M33 35c12 0 19-7 18-17-11-1-19 5-18 17Z" fill="#d1fae5"/>
+  <path d="M22 50h20" fill="none" stroke="#fff" stroke-width="5" stroke-linecap="round"/>
+</svg>

config.py

@@ -18,6 +18,10 @@ class Config:
     headless: bool = os.getenv("HEADLESS", "true").lower() == "true"
     chrome_binary: str | None = os.getenv("CHROME_BINARY")
     chromedriver_path: str = os.getenv("CHROMEDRIVER_PATH", "drivers/chromedriver")
+    record_sessions: bool = os.getenv("RECORD_SESSIONS", "false").lower() == "true"
+    recording_fps: int = int(os.getenv("RECORDING_FPS", "12"))
+    recording_max_files: int = int(os.getenv("RECORDING_MAX_FILES", "3"))
+    recordings_dir: str = os.getenv("RECORDINGS_DIR", "recordings")
 
     # Scenario 1 — SMS-OTP: list of phone numbers (one per line in env or comma-separated)
     phone_numbers: list[str] = field(default_factory=list)

crawler/batch.py

@@ -27,10 +27,19 @@ def _single_run(
 
     identifier = f"batch-{batch_id[:8]}-{run_index:04d}"
     t0 = time.monotonic()
-    log.info("[batch:%s] Run %d starting", batch_id[:8], run_index)
+    target_url = str(cfg.get("target_url") or "").strip()
+    log.info(
+        "[batch:%s] Run %d starting — target=%s",
+        batch_id[:8],
+        run_index,
+        target_url or "<missing>",
+    )
     try:
-        target_url: str = cfg.get("target_url") or ""
-        with driver_session(headless=headless) as driver:
+        with driver_session(
+            headless=headless,
+            recording_name=identifier,
+            initial_url=target_url,
+        ) as driver:
             FreshSessionScenario(driver, target_url).run()
             result = DynamicFlow(driver, cfg).run() if cfg else None
 

crawler/driver.py

@@ -4,19 +4,24 @@ from __future__ import annotations
 
 import os
 import random
+import re
 import shutil
+import signal
 import subprocess
 import sys
+import tempfile
 import time
 from contextlib import contextmanager
 from pathlib import Path
-from typing import Generator
+from typing import Any, Generator, cast
 
 import undetected_chromedriver as uc
+from selenium.common.exceptions import WebDriverException
 from selenium.webdriver.chrome.options import Options
 from selenium_stealth import stealth
 
 from config import config
+from crawler.recording import SessionRecorder
 from logger import get_logger
 
 log = get_logger(__name__)
@@ -56,7 +61,10 @@ def _detect_chrome() -> tuple[str, int]:
             out = subprocess.check_output(
                 [resolved, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
             ).strip()
-            major = int(out.split()[-1].split(".")[0])
+            match = re.search(r"\b(\d+(?:\.\d+){3})\b", out)
+            if match is None:
+                continue
+            major = int(match.group(1).split(".")[0])
             log.debug("Detected Chrome %d via '%s'", major, resolved)
             return resolved, major
         except Exception:
@@ -77,41 +85,102 @@ def _local_driver_path() -> str | None:
     return None
 
 
-def _build_options(user_agent: str, headless: bool, chrome_binary: str) -> Options:
+def _build_options(
+    user_agent: str,
+    headless: bool,
+    chrome_binary: str,
+    profile_dir: str | None = None,
+    display: str | None = None,
+) -> Options:
     opts = Options()
     opts.binary_location = chrome_binary
+    # Never let ChromeDriver session creation block on the initial New Tab
+    # renderer. Scenarios perform their own explicit URL/readiness waits.
+    opts.page_load_strategy = "none"
     if headless:
         opts.add_argument("--headless=new")
+    if profile_dir:
+        opts.add_argument(f"--user-data-dir={profile_dir}")
+    if display:
+        opts.add_argument(f"--display={display}")
+        opts.add_argument("--ozone-platform=x11")
     opts.add_argument(f"--user-agent={user_agent}")
     opts.add_argument("--no-sandbox")
-    opts.add_argument("--disable-dev-shm-usage")
     opts.add_argument("--disable-gpu")
+    opts.add_argument("--disable-logging")
+    opts.add_argument("--no-first-run")
+    opts.add_argument("--no-default-browser-check")
+    opts.add_argument("--disable-background-networking")
+    opts.add_argument("--disable-component-update")
+    opts.add_argument("--disable-default-apps")
+    opts.add_argument("--disable-sync")
+    opts.add_argument("--renderer-process-limit=2")
     opts.add_argument("--disable-blink-features=AutomationControlled")
     opts.add_argument("--disable-infobars")
     opts.add_argument("--window-size=1920,1080")
     return opts
 
 
-def make_driver(headless: bool | None = None) -> uc.Chrome:
+def make_driver(
+    headless: bool | None = None,
+    profile_dir: str | None = None,
+    display: str | None = None,
+    initial_url: str | None = None,
+) -> uc.Chrome:
     """Return a stealthed undetected Chrome instance."""
     use_headless = config.headless if headless is None else headless
     chrome_binary, major = _detect_chrome()
     ua = random.choice(_USER_AGENTS)
-    opts = _build_options(ua, use_headless, chrome_binary)
+    opts = _build_options(
+        ua,
+        use_headless,
+        chrome_binary,
+        profile_dir,
+        display,
+    )
 
-    log.debug("Starting ChromeDriver (headless=%s, Chrome %d)", use_headless, major)
+    log.info(
+        "Starting ChromeDriver (headless=%s, Chrome %d, initial_url=%s)",
+        use_headless,
+        major,
+        initial_url or "<none>",
+    )
 
     driver = uc.Chrome(
         options=opts,
         driver_executable_path=_local_driver_path(),
         version_main=major,
         use_subprocess=True,
+        # undetected-chromedriver captures Chrome stdout/stderr in pipes but
+        # does not drain them. Fatal-only logging prevents a full pipe from
+        # freezing Chrome during renderer startup in Docker.
+        log_level=3,
     )
+    log.info("ChromeDriver connected")
 
-    if not use_headless:
+    if initial_url and initial_url.startswith(("http://", "https://")):
+        navigation = cast(
+            dict[str, Any],
+            driver.execute_cdp_cmd(  # type: ignore[reportUnknownMemberType]
+                "Page.navigate", {"url": initial_url}
+            ),
+        )
+        error_text = navigation.get("errorText")
+        if error_text:
+            raise WebDriverException(
+                f"Chrome rejected initial navigation to {initial_url}: {error_text}"
+            )
+        log.info("Initial navigation dispatched to %s", initial_url)
+
+    log.info("Applying stealth settings")
+
+    if display:
+        driver.set_window_size(1920, 1080)
+    elif not use_headless:
         driver.maximize_window()
     else:
         driver.set_window_size(1920, 1080)
+    driver.set_page_load_timeout(45)
 
     stealth(
         driver,
@@ -135,13 +204,67 @@ def make_driver(headless: bool | None = None) -> uc.Chrome:
     return driver
 
 
+def _profile_processes(profile_dir: str) -> list[int]:
+    marker = profile_dir.encode()
+    pids: list[int] = []
+    for entry in Path("/proc").iterdir():
+        if not entry.name.isdigit():
+            continue
+        try:
+            if marker in (entry / "cmdline").read_bytes():
+                pids.append(int(entry.name))
+        except (FileNotFoundError, PermissionError, ProcessLookupError):
+            continue
+    return pids
+
+
+def _terminate_profile_processes(profile_dir: str) -> None:
+    for sig in (signal.SIGTERM, signal.SIGKILL):
+        pids = _profile_processes(profile_dir)
+        for pid in pids:
+            try:
+                os.kill(pid, sig)
+            except ProcessLookupError:
+                pass
+        if sig == signal.SIGTERM and pids:
+            time.sleep(0.5)
+
+
 @contextmanager
-def driver_session(headless: bool | None = None) -> Generator[uc.Chrome, None, None]:
-    driver = make_driver(headless)
+def driver_session(
+    headless: bool | None = None,
+    recording_name: str | None = None,
+    initial_url: str | None = None,
+) -> Generator[uc.Chrome, None, None]:
+    profile_dir = tempfile.mkdtemp(prefix="seed-chrome-")
+    driver: uc.Chrome | None = None
+    recorder: SessionRecorder | None = None
+    display: str | None = None
+    effective_headless = config.headless if headless is None else headless
     try:
+        if config.record_sessions:
+            recorder = SessionRecorder(recording_name)
+            if recorder.start():
+                display = recorder.display
+                effective_headless = False
+        driver = make_driver(
+            effective_headless,
+            profile_dir=profile_dir,
+            display=display,
+            initial_url=initial_url,
+        )
         yield driver
     finally:
-        driver.quit()
+        _terminate_profile_processes(profile_dir)
+        if driver is not None:
+            try:
+                driver.quit()
+            except Exception as exc:
+                log.warning("Driver shutdown error: %s", exc)
+        _terminate_profile_processes(profile_dir)
+        if recorder is not None:
+            recorder.stop()
+        shutil.rmtree(profile_dir, ignore_errors=True)
         log.debug("Driver session closed")
 
 

crawler/recording.py

@@ -0,0 +1,186 @@
+"""Per-session Xvfb/FFmpeg recording with shared-volume retention."""
+
+from __future__ import annotations
+
+import fcntl
+import os
+import re
+import signal
+import subprocess
+import threading
+import time
+import uuid
+from datetime import UTC, datetime
+from pathlib import Path
+
+from config import config
+from logger import get_logger
+
+log = get_logger(__name__)
+
+_DISPLAY_LOCK = threading.Lock()
+_SAFE_NAME = re.compile(r"[^A-Za-z0-9_.-]+")
+_WIDTH = 1920
+_HEIGHT = 1080
+
+
+def _safe_label(label: str | None) -> str:
+    cleaned = _SAFE_NAME.sub("-", label or "session").strip("-._")
+    return cleaned[:80] or "session"
+
+
+def _stop_process(process: subprocess.Popen[bytes] | None, *, interrupt: bool = False) -> None:
+    if process is None or process.poll() is not None:
+        return
+    try:
+        process.send_signal(signal.SIGINT if interrupt else signal.SIGTERM)
+        process.wait(timeout=10)
+    except subprocess.TimeoutExpired:
+        process.kill()
+        process.wait(timeout=5)
+
+
+def _prune_recordings(directory: Path) -> None:
+    keep = max(1, config.recording_max_files)
+    lock_path = directory / ".retention.lock"
+    with lock_path.open("a+b") as lock_file:
+        fcntl.flock(lock_file.fileno(), fcntl.LOCK_EX)
+        recordings = sorted(
+            directory.glob("*.mp4"),
+            key=lambda path: path.stat().st_mtime_ns,
+            reverse=True,
+        )
+        for stale in recordings[keep:]:
+            try:
+                stale.unlink()
+                log.info("Removed expired recording %s", stale.name)
+            except FileNotFoundError:
+                pass
+
+
+class SessionRecorder:
+    """Own one virtual display and one FFmpeg recorder."""
+
+    def __init__(self, label: str | None = None) -> None:
+        self.label = _safe_label(label)
+        self.display: str | None = None
+        self._xvfb: subprocess.Popen[bytes] | None = None
+        self._ffmpeg: subprocess.Popen[bytes] | None = None
+        self._temp_path: Path | None = None
+        self._final_path: Path | None = None
+
+    def start(self) -> bool:
+        directory = Path(config.recordings_dir)
+        directory.mkdir(parents=True, exist_ok=True)
+        stamp = datetime.now(UTC).strftime("%Y%m%dT%H%M%S.%fZ")
+        token = uuid.uuid4().hex[:8]
+        self._final_path = directory / f"{self.label}-{stamp}-{token}.mp4"
+        self._temp_path = directory / f".{self._final_path.name}.part.mp4"
+
+        try:
+            with _DISPLAY_LOCK:
+                self._start_xvfb()
+            self._start_ffmpeg()
+        except Exception as exc:
+            log.error("Could not start session recording: %s", exc)
+            self.stop(publish=False)
+            return False
+
+        log.info(
+            "Recording session to %s at %d FPS",
+            self._final_path.name,
+            config.recording_fps,
+        )
+        return True
+
+    def _start_xvfb(self) -> None:
+        for display_number in range(100, 1000):
+            socket_path = Path(f"/tmp/.X11-unix/X{display_number}")
+            if socket_path.exists():
+                continue
+            display = f":{display_number}"
+            process = subprocess.Popen(
+                [
+                    "Xvfb",
+                    display,
+                    "-screen",
+                    "0",
+                    f"{_WIDTH}x{_HEIGHT}x24",
+                    "-nolisten",
+                    "tcp",
+                    "-ac",
+                ],
+                stdout=subprocess.DEVNULL,
+                stderr=subprocess.PIPE,
+            )
+            deadline = time.monotonic() + 5
+            while time.monotonic() < deadline:
+                if process.poll() is not None:
+                    stderr = (process.stderr.read() if process.stderr else b"").decode(
+                        "utf-8", errors="replace"
+                    )
+                    raise RuntimeError(f"Xvfb exited early: {stderr.strip()}")
+                if socket_path.exists():
+                    self.display = display
+                    self._xvfb = process
+                    return
+                time.sleep(0.05)
+            _stop_process(process)
+        raise RuntimeError("no free X display was available")
+
+    def _start_ffmpeg(self) -> None:
+        if self.display is None or self._temp_path is None:
+            raise RuntimeError("virtual display is not ready")
+        self._ffmpeg = subprocess.Popen(
+            [
+                "ffmpeg",
+                "-y",
+                "-nostdin",
+                "-loglevel",
+                "error",
+                "-f",
+                "x11grab",
+                "-framerate",
+                str(config.recording_fps),
+                "-video_size",
+                f"{_WIDTH}x{_HEIGHT}",
+                "-i",
+                f"{self.display}.0",
+                "-c:v",
+                "libx264",
+                "-preset",
+                "ultrafast",
+                "-threads",
+                "1",
+                "-crf",
+                "28",
+                "-pix_fmt",
+                "yuv420p",
+                str(self._temp_path),
+            ],
+            stdout=subprocess.DEVNULL,
+            stderr=subprocess.PIPE,
+        )
+        time.sleep(0.2)
+        if self._ffmpeg.poll() is not None:
+            stderr = (self._ffmpeg.stderr.read() if self._ffmpeg.stderr else b"").decode(
+                "utf-8", errors="replace"
+            )
+            raise RuntimeError(f"FFmpeg exited early: {stderr.strip()}")
+
+    def stop(self, *, publish: bool = True) -> Path | None:
+        _stop_process(self._ffmpeg, interrupt=True)
+        _stop_process(self._xvfb)
+        self._ffmpeg = None
+        self._xvfb = None
+
+        if self._temp_path is None or self._final_path is None:
+            return None
+        if not publish or not self._temp_path.exists() or self._temp_path.stat().st_size == 0:
+            self._temp_path.unlink(missing_ok=True)
+            return None
+
+        os.replace(self._temp_path, self._final_path)
+        _prune_recordings(self._final_path.parent)
+        log.info("Recording saved: %s", self._final_path.name)
+        return self._final_path

crawler/scenarios/fresh_session.py

@@ -1,10 +1,37 @@
 """Scenario 2 — fresh cookie jar per run so the site treats the visitor as a new user."""
 from __future__ import annotations
 
+from typing import Any, cast
+from urllib.parse import urlsplit
+
 import undetected_chromedriver as uc
+from selenium.common.exceptions import TimeoutException, WebDriverException
 from selenium.webdriver.support.ui import WebDriverWait
 
 from crawler.driver import human_delay
+from logger import get_logger
+
+log = get_logger(__name__)
+
+class FreshSessionError(RuntimeError):
+    """The browser could not leave its initial blank page."""
+
+
+def _normalise_url(url: str) -> str:
+    value = url.strip()
+    if value and "://" not in value:
+        value = f"https://{value}"
+    parsed = urlsplit(value)
+    if parsed.scheme not in {"http", "https"} or not parsed.netloc:
+        raise FreshSessionError(
+            "target_url must be a complete HTTP(S) URL, for example https://example.com"
+        )
+    return value
+
+
+def _is_web_url(url: str) -> bool:
+    parsed = urlsplit(url)
+    return parsed.scheme in {"http", "https"} and bool(parsed.netloc)
 
 
 class FreshSessionScenario:
@@ -12,39 +39,84 @@ class FreshSessionScenario:
     Opens the target URL in a brand-new browser profile (no stored cookies,
     localStorage, or cache) so each run appears as a first-time visitor.
 
-    Because undetected-chromedriver already creates an isolated temp profile
-    per instance, simply spinning up a new driver is sufficient — but we also
-    explicitly delete all cookies after loading to be safe.
+    Undetected-chromedriver creates an isolated temporary profile per instance,
+    so no cookie or storage clearing is needed before navigation.
     """
 
     def __init__(self, driver: uc.Chrome, url: str, timeout: int = 30) -> None:
         self.driver = driver
-        self.url = url
+        self.url = _normalise_url(url)
         self.wait = WebDriverWait(driver, timeout)
 
     def run(self) -> dict[str, object]:
-        """Navigate to the target, clear any residual state, return page info."""
-        self.driver.delete_all_cookies()
+        """Navigate to the target and return page information."""
+        # Each driver already owns a brand-new temporary profile. Do not issue
+        # renderer commands on Chrome's initial blank tab: on small servers that
+        # renderer can stall before navigation is ever attempted.
+        log.info("Navigating browser to %s", self.url)
+        try:
+            navigation = cast(
+                dict[str, Any],
+                self.driver.execute_cdp_cmd(  # type: ignore[reportUnknownMemberType]
+                    "Page.navigate", {"url": self.url}
+                ),
+            )
+            error_text = navigation.get("errorText")
+            if error_text:
+                raise FreshSessionError(f"Chrome rejected navigation: {error_text}")
+        except FreshSessionError:
+            raise
+        except WebDriverException:
+            # Older Chrome builds may not expose Page.navigate. Keep a normal
+            # WebDriver fallback, but only after navigation has been attempted.
+            try:
+                self.driver.get(self.url)
+            except TimeoutException:
+                log.warning("Page load timed out; stopping navigation")
+                self.driver.execute_cdp_cmd(  # type: ignore[reportUnknownMemberType]
+                    "Page.stopLoading", {}
+                )
 
-        # Clear localStorage / sessionStorage via JS
-        self.driver.execute_script(
-            "try { window.localStorage.clear(); window.sessionStorage.clear(); } catch(e) {}"
-        )
+        try:
+            # Chrome's internal New Tab URLs (chrome://newtab and
+            # chrome://new-tab-page) are not successful navigation. Wait until
+            # the address bar actually contains an HTTP(S) page or redirect.
+            self.wait.until(lambda d: _is_web_url(d.current_url))
+        except TimeoutException as exc:
+            try:
+                current_url = self.driver.current_url
+            except WebDriverException:
+                current_url = "<unavailable>"
+            raise FreshSessionError(
+                f"navigation to {self.url} never reached a web page "
+                f"(Chrome remained at {current_url})"
+            ) from exc
 
-        if not self.url:
-            return {"title": "", "url": "", "cookie_count": 0, "cookies": []}
+        human_delay(1.0, 2.0)
 
-        self.driver.get(self.url)
-        human_delay(2.0, 4.0)
+        # Interactive is sufficient for the dynamic flow and avoids waiting on
+        # analytics, ads, or other background resources indefinitely.
+        def document_is_ready(driver: uc.Chrome) -> bool:
+            state = cast(
+                str,
+                driver.execute_script(  # type: ignore[reportUnknownMemberType]
+                    "return document.readyState"
+                ),
+            )
+            return state in ("interactive", "complete")
 
-        # Wait for the page to reach a ready state
-        self.wait.until(
-            lambda d: d.execute_script("return document.readyState") == "complete"
-        )
+        try:
+            self.wait.until(document_is_ready)
+        except TimeoutException:
+            log.warning("Document did not report a ready state; continuing with the current DOM")
 
         title = self.driver.title
         current_url = self.driver.current_url
-        cookies = self.driver.get_cookies()
+        cookies = cast(
+            list[dict[str, Any]],
+            self.driver.get_cookies(),  # type: ignore[reportUnknownMemberType]
+        )
+        log.info("Browser reached %s", current_url)
 
         return {
             "title": title,

crawler/tasks.py

@@ -304,12 +304,21 @@ def run_batch_job(batch_id: str, item: dict[str, Any]) -> None:
 
         t0 = time.monotonic()
         identifier = f"batch-{batch_id[:8]}-{slot:04d}"
-        log.info("[batch:%s] slot %d starting", batch_id[:8], slot)
+        target_url = str(cfg.get("target_url") or "").strip()
+        scenario: str = cfg.get("scenario") or "dynamic"
+        log.info(
+            "[batch:%s] slot %d starting — target=%s",
+            batch_id[:8],
+            slot,
+            target_url or "<missing>",
+        )
 
         try:
-            target_url: str = cfg.get("target_url") or ""
-            scenario: str = cfg.get("scenario") or "dynamic"
-            with driver_session(headless=headless) as driver:
+            with driver_session(
+                headless=headless,
+                recording_name=identifier,
+                initial_url=target_url,
+            ) as driver:
                 FreshSessionScenario(driver, target_url).run()
                 if cfg:
                     if scenario == "digipay":

docker-compose.yml

@@ -15,13 +15,17 @@ services:
 
   # ── Admin panel (FastAPI) ─────────────────────────────────────────────────────
   admin:
-    build: .
-    image: seed:latest
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: admin
+    image: seed-admin:latest
+    hostname: crawler-admin
     container_name: seed-admin
     restart: unless-stopped
     command: python main.py admin
     ports:
-      - "${ADMIN_PORT:-8000}:8000"
+      - "${COMPOSE_ADMIN_PORT:-127.0.0.1:8800}:8000"
     env_file:
       - .env
     environment:
@@ -31,9 +35,11 @@ services:
       REDIS_URL: "redis://redis:6379/0"
       SKIP_INTERNAL_WORKER: "true"
       HEADLESS: "true"
+      RECORDINGS_DIR: /app/recordings
     volumes:
       - seed-data:/app/data
       - seed-logs:/app/logs
+      - seed-recordings:/app/recordings
     depends_on:
       redis:
         condition: service_healthy
@@ -43,14 +49,21 @@ services:
       timeout: 5s
       retries: 3
       start_period: 10s
+    networks:
+      - proxy-network
+      - default
 
   # ── RQ worker (Chrome crawler, runs batch jobs) ───────────────────────────────
   worker:
-    build: .
-    image: seed:latest
-    container_name: seed-worker
+    build:
+      context: .
+      dockerfile: Dockerfile
+      target: worker
+    image: seed-worker:latest
     restart: unless-stopped
-    command: python -m rq worker --url redis://redis:6379/0 batch
+    deploy:
+      replicas: 3
+    command: ["rq", "worker", "--url", "redis://redis:6379/0", "batch"]
     env_file:
       - .env
     environment:
@@ -58,24 +71,36 @@ services:
       LOG_FILE: /app/logs/seed.log
       REDIS_URL: "redis://redis:6379/0"
       HEADLESS: "true"
-      CHROME_BINARY: /usr/bin/google-chrome-stable
+      RECORD_SESSIONS: "true"
+      RECORDING_FPS: "12"
+      RECORDING_MAX_FILES: "3"
+      RECORDINGS_DIR: /app/recordings
     volumes:
       - seed-data:/app/data
       - seed-logs:/app/logs
+      - seed-recordings:/app/recordings
     depends_on:
       redis:
         condition: service_healthy
-    # Chrome needs extra shared memory to avoid OOM crashes
-    shm_size: "256mb"
+    shm_size: "2gb"
     cap_add:
       - SYS_ADMIN
     security_opt:
       - seccomp:unconfined
 
+
+networks:
+  proxy-network:
+    external: true
+  default:
+    name: crawle-snapp_default
+
 volumes:
   seed-data:
     driver: local
   seed-logs:
     driver: local
+  seed-recordings:
+    driver: local
   redis-data:
     driver: local

main.py

@@ -73,7 +73,7 @@ def run_otp(phone: str, cfg_id: int | None = None) -> None:
     target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
     t0 = time.monotonic()
     try:
-        with driver_session() as driver:
+        with driver_session(initial_url=target_url) as driver:
             OTPLoginScenario(driver, phone, otp_resolver, target_url).run()
             result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
     except Exception as exc:
@@ -91,7 +91,7 @@ def run_fresh(cfg_id: int | None = None) -> None:
     target_url: str = flow_cfg.get("target_url") or "" if flow_cfg else ""
     t0 = time.monotonic()
     try:
-        with driver_session() as driver:
+        with driver_session(initial_url=target_url) as driver:
             FreshSessionScenario(driver, target_url).run()
             result = DynamicFlow(driver, flow_cfg).run() if flow_cfg else None
     except Exception as exc:

pyproject.toml

@@ -31,3 +31,8 @@ venv = ".venv"
 dev = [
     "pyright>=1.1.370",
 ]
+
+[[tool.uv.index]]
+name = "pypi"
+url = "https://pypi.org/simple"
+default = true

scripts/patch_driver.py

@@ -7,6 +7,7 @@ version matching the installed Chrome, then patches it with undetected_chromedri
 from __future__ import annotations
 
 import io
+import re
 import shutil
 import stat
 import subprocess
@@ -22,6 +23,13 @@ from logger import get_logger
 
 log = get_logger(__name__)
 
+_DOWNLOAD_HEADERS = {
+    "User-Agent": (
+        "Mozilla/5.0 (X11; Linux x86_64) AppleWebKit/537.36 "
+        "(KHTML, like Gecko) Chrome/124.0.0.0 Safari/537.36"
+    )
+}
+
 
 def _get_chrome_full_version() -> tuple[int, str]:
     candidates = ["google-chrome", "google-chrome-stable", "chromium", "chromium-browser"]
@@ -35,7 +43,10 @@ def _get_chrome_full_version() -> tuple[int, str]:
             out = subprocess.check_output(
                 [binary, "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
             ).strip()
-            full = out.split()[-1]
+            match = re.search(r"\b(\d+(?:\.\d+){3})\b", out)
+            if match is None:
+                continue
+            full = match.group(1)
             major = int(full.split(".")[0])
             log.info("Detected Chrome %s via '%s'", full, binary)
             return major, full
@@ -49,7 +60,8 @@ def _get_chrome_full_version() -> tuple[int, str]:
 def _latest_chromedriver_version(major: int) -> str:
     url = f"https://googlechromelabs.github.io/chrome-for-testing/LATEST_RELEASE_{major}"
     try:
-        with urllib.request.urlopen(url, timeout=15) as resp:
+        request = urllib.request.Request(url, headers=_DOWNLOAD_HEADERS)
+        with urllib.request.urlopen(request, timeout=15) as resp:
             version = resp.read().decode().strip()
             log.info("Latest ChromeDriver for Chrome %d: %s", major, version)
             return version
@@ -65,7 +77,8 @@ def _download_chromedriver(version: str, dest: Path) -> None:
     )
     log.info("Downloading ChromeDriver %s …", version)
     try:
-        with urllib.request.urlopen(zip_url, timeout=60) as resp:
+        request = urllib.request.Request(zip_url, headers=_DOWNLOAD_HEADERS)
+        with urllib.request.urlopen(request, timeout=60) as resp:
             data = resp.read()
     except Exception as exc:
         log.critical("Download failed: %s", exc)
@@ -88,13 +101,42 @@ def _patch(dest: Path) -> None:
     log.info("Patch applied successfully")
 
 
+def _validate_existing_driver(dest: Path, chrome_major: int) -> bool:
+    if not dest.exists():
+        return False
+
+    try:
+        out = subprocess.check_output(
+            [str(dest), "--version"], stderr=subprocess.DEVNULL, text=True, timeout=5
+        ).strip()
+        driver_major = int(out.split()[1].split(".")[0])
+    except Exception as exc:
+        log.critical("Could not inspect ChromeDriver at %s: %s", dest, exc)
+        sys.exit(1)
+
+    if driver_major != chrome_major:
+        log.critical(
+            "ChromeDriver major %d does not match Chrome major %d",
+            driver_major,
+            chrome_major,
+        )
+        sys.exit(1)
+
+    log.info("Using packaged ChromeDriver %s", out.split()[1])
+    return True
+
+
 def main() -> None:
     dest = Path(config.chromedriver_path)
     dest.parent.mkdir(parents=True, exist_ok=True)
 
     major, _ = _get_chrome_full_version()
-    cd_version = _latest_chromedriver_version(major)
+    if _validate_existing_driver(dest, major):
+        _patch(dest)
+        log.info("ChromeDriver ready at %s", dest)
+        return
 
+    cd_version = _latest_chromedriver_version(major)
     _download_chromedriver(cd_version, dest)
     _patch(dest)