diff --git a/.env.example b/.env.example index 378928f..f44ca5e 100644 --- a/.env.example +++ b/.env.example @@ -1,6 +1,8 @@ # Copy to .env and fill in -TARGET_URL=https://example.com +# Logging +LOG_LEVEL=INFO # DEBUG | INFO | WARNING | ERROR | CRITICAL +# LOG_FILE=logs/seed.log # optional rotating file log # Admin panel credentials ADMIN_USERNAME=admin @@ -13,7 +15,8 @@ DB_PATH=data/tracker.db # Chrome: set to false to watch the browser window HEADLESS=true -# CHROME_BINARY=/usr/bin/chromium # optional: explicit Chrome path +# CHROME_BINARY=/usr/bin/google-chrome-stable # optional: explicit Chrome binary path +# CHROMEDRIVER_PATH=drivers/chromedriver # pre-patched driver (run: make patch-driver) # Scenario 1 — comma-separated phone numbers PHONE_NUMBERS=+989100000001,+989100000002,+989100000003 diff --git a/.gitignore b/.gitignore index dd58e5f..873cc70 100644 --- a/.gitignore +++ b/.gitignore @@ -5,3 +5,4 @@ __pycache__/ data/ *.db *.swp +drivers/ diff --git a/Makefile b/Makefile index 908845d..b340d76 100644 --- a/Makefile +++ b/Makefile @@ -11,7 +11,7 @@ GREEN := $(shell tput setaf 2 2>/dev/null) CYAN := $(shell tput setaf 6 2>/dev/null) .PHONY: help install sync lock lint typecheck \ - admin otp fresh \ + admin otp fresh batch \ build up down logs shell \ clean @@ -46,6 +46,10 @@ typecheck: ## Run Pyright type-checker # ── Crawler ─────────────────────────────────────────────────────────────────── +patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chrome updates) + @mkdir -p drivers + PYTHONPATH=. $(PYTHON) scripts/patch_driver.py + admin: ## Start the admin panel (http://localhost:8000) @mkdir -p $(DATA_DIR) $(PYTHON) main.py admin @@ -62,6 +66,14 @@ fresh: ## Run Scenario 2 — fresh cookie session @mkdir -p $(DATA_DIR) $(PYTHON) main.py fresh +WORKERS ?= 3 +RUNS ?= 10 +STAGGER ?= 500 + +batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500) + @mkdir -p $(DATA_DIR) + $(PYTHON) main.py batch --workers $(WORKERS) --runs $(RUNS) --stagger $(STAGGER) + # ── Docker ──────────────────────────────────────────────────────────────────── build: ## Build the Docker image diff --git a/admin/main.py b/admin/main.py index 0b35b59..afdb55c 100644 --- a/admin/main.py +++ b/admin/main.py @@ -1,6 +1,7 @@ """FastAPI admin application.""" from __future__ import annotations +import threading from pathlib import Path from fastapi import FastAPI @@ -21,6 +22,20 @@ app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static") @app.on_event("startup") def on_startup() -> None: init_db() + _start_huey_consumer() + + +def _start_huey_consumer() -> None: + from huey.consumer import Consumer + from crawler.tasks import huey + + class _ThreadConsumer(Consumer): + def _set_signal_handlers(self) -> None: + pass # signal.signal() only works on the main thread + + consumer = _ThreadConsumer(huey, workers=1, periodic=False) + t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer") + t.start() @app.get("/") diff --git a/admin/routes.py b/admin/routes.py index 248b19b..11a4bb7 100644 --- a/admin/routes.py +++ b/admin/routes.py @@ -2,6 +2,7 @@ from __future__ import annotations import secrets +import uuid from typing import Annotated, Any from fastapi import APIRouter, Depends, HTTPException, status @@ -17,6 +18,9 @@ from admin.models import ( upsert_flow_config, ) from config import config +from logger import get_logger + +log = get_logger(__name__) router = APIRouter() security = HTTPBasic() @@ -59,13 +63,21 @@ class FlowConfigIn(BaseModel): search_submit_selector: str = "" search_submit_selector_type: str = "css" search_results_selector: str = "" + search_results_selector_type: str = "css" scroll_container_selector: str = "" + scroll_container_selector_type: str = "css" max_scrolls: int = 30 scroll_pause_ms: int = 1200 no_new_content_timeout_ms: int = 3000 + pagination_selector: str = "" + pagination_selector_type: str = "css" + max_pages: int = 10 + pagination_wait_ms: int = 1500 item_selector_template: str = "" item_selector_type: str = "css" + item_url_template: str = "" target_item_ids: list[str] = [] + target_url: str = "" def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: @@ -78,13 +90,21 @@ def _to_db_dict(body: FlowConfigIn) -> dict[str, Any]: "search_submit_selector": body.search_submit_selector, "search_submit_selector_type": body.search_submit_selector_type, "search_results_selector": body.search_results_selector, + "search_results_selector_type": body.search_results_selector_type, "scroll_container_selector": body.scroll_container_selector, + "scroll_container_selector_type": body.scroll_container_selector_type, "max_scrolls": body.max_scrolls, "scroll_pause_ms": body.scroll_pause_ms, "no_new_content_timeout_ms": body.no_new_content_timeout_ms, + "pagination_selector": body.pagination_selector, + "pagination_selector_type": body.pagination_selector_type, + "max_pages": body.max_pages, + "pagination_wait_ms": body.pagination_wait_ms, "item_selector_template": body.item_selector_template, "item_selector_type": body.item_selector_type, + "item_url_template": body.item_url_template, "target_item_ids_json": body.target_item_ids, + "target_url": body.target_url, } @@ -133,3 +153,50 @@ async def delete_config(cfg_id: int, _: Auth) -> None: if not existing: raise HTTPException(status_code=404, detail="Not found") await delete_flow_config(cfg_id) + + +# --------------------------------------------------------------------------- +# Batch runs (managed via Huey task queue) +# --------------------------------------------------------------------------- + +class BatchRunIn(BaseModel): + workers: int = 3 + total_runs: int = 10 + stagger_ms: int = 500 + headless: bool = True + + +@router.post("/api/run-batch", status_code=202) +async def start_batch(body: BatchRunIn, _: Auth) -> dict[str, object]: + from crawler.tasks import batch_status as _status, run_batch_task, set_current + + current = _status() + if current.get("running"): + raise HTTPException(status_code=409, detail="A batch is already running") + + from admin.models import get_active_flow_config + active_cfg = await get_active_flow_config() + if not active_cfg: + raise HTTPException(status_code=422, detail="No active flow config — activate one first") + + batch_id = uuid.uuid4().hex + result = run_batch_task(batch_id, active_cfg, body.workers, body.total_runs, body.headless, body.stagger_ms) + set_current(batch_id, result) + + log.info("Batch %s enqueued — %d runs, %d workers", batch_id[:8], body.total_runs, body.workers) + return {"batch_id": batch_id, "status": "queued"} + + +@router.post("/api/stop-batch", status_code=200) +async def stop_batch(_: Auth) -> dict[str, object]: + from crawler.tasks import stop_current + stopped = stop_current() + if not stopped: + raise HTTPException(status_code=404, detail="No active batch to stop") + return {"status": "stopping"} + + +@router.get("/api/batch-status") +async def batch_status(_: Auth) -> dict[str, object]: + from crawler.tasks import batch_status as _status + return _status() # type: ignore[return-value] diff --git a/admin/static/dashboard.html b/admin/static/dashboard.html index 5dfcfaf..e1cd963 100644 --- a/admin/static/dashboard.html +++ b/admin/static/dashboard.html @@ -102,6 +102,17 @@ } .form-box h2 { font-size: 17px; font-weight: 600; margin-bottom: 24px; } .form-section { font-size: 11px; font-weight: 700; text-transform: uppercase; letter-spacing: .6px; color: var(--accent); margin: 22px 0 12px; } + details.adv { margin-top: 20px; border: 1px solid var(--border); border-radius: 9px; } + details.adv > summary { + list-style: none; cursor: pointer; padding: 11px 14px; + font-size: 12px; font-weight: 600; color: var(--muted); + display: flex; align-items: center; gap: 7px; user-select: none; + } + details.adv > summary::-webkit-details-marker { display: none; } + details.adv > summary::before { content: '▶'; font-size: 9px; transition: transform .15s; } + details.adv[open] > summary::before { transform: rotate(90deg); } + details.adv > summary:hover { color: var(--text); } + .adv-body { padding: 4px 14px 14px; } .form-row { display: grid; grid-template-columns: 1fr 1fr; gap: 12px; } .fg { margin-bottom: 14px; } .fg label { display: block; color: var(--muted); font-size: 12px; margin-bottom: 5px; } @@ -124,6 +135,9 @@ .modal-box h2 { font-size: 18px; font-weight: 600; margin-bottom: 24px; } .error-msg { color: var(--red); font-size: 12px; margin-top: 8px; text-align: center; } .error-msg.hidden { display: none; } + .remember-row { display: flex; align-items: center; gap: 8px; margin-top: 12px; } + .remember-row input[type=checkbox] { width: 15px; height: 15px; accent-color: var(--accent); cursor: pointer; } + .remember-row label { color: var(--muted); font-size: 13px; cursor: pointer; user-select: none; }
@@ -134,7 +148,11 @@Use {item_id} as a placeholder — it is replaced per item ID below.
Finds <a href> whose href contains this pattern. Use {item_id} as placeholder.
No configs yet. Create one to get started.
@@ -297,6 +370,96 @@