nfel
5bc7acd114
The worker needs Redis to connect and complete crawl runs. Removing --no-deps so compose starts dependencies for one-off runs. Co-authored-by: Qwen-Coder <qwen-coder@alibabacloud.com>
127 lines
4.7 KiB
Makefile
127 lines
4.7 KiB
Makefile
.DEFAULT_GOAL := help
|
|
SHELL := /bin/bash
|
|
UV := uv
|
|
PYTHON := $(UV) run python
|
|
DATA_DIR := data
|
|
|
|
# ── colours ──────────────────────────────────────────────────────────────────
|
|
BOLD := $(shell tput bold 2>/dev/null)
|
|
RESET := $(shell tput sgr0 2>/dev/null)
|
|
GREEN := $(shell tput setaf 2 2>/dev/null)
|
|
CYAN := $(shell tput setaf 6 2>/dev/null)
|
|
|
|
.PHONY: help install sync lock lint typecheck \
|
|
dev admin worker otp fresh batch \
|
|
build up down logs shell \
|
|
clean
|
|
|
|
help: ## Show this help
|
|
@awk 'BEGIN{FS=":.*##"} /^[a-zA-Z_-]+:.*##/ {printf " $(CYAN)%-18s$(RESET) %s\n",$$1,$$2}' $(MAKEFILE_LIST)
|
|
@echo ""
|
|
@echo " $(BOLD)Examples$(RESET)"
|
|
@echo " make install # first-time setup"
|
|
@echo " make admin # start admin panel → http://localhost:8000"
|
|
@echo " make otp PHONE=+989... # run OTP scenario for one phone"
|
|
@echo " make otp # run OTP scenario for all phones in .env"
|
|
@echo " make fresh # run fresh-session scenario"
|
|
@echo " make up # start everything in Docker"
|
|
|
|
# ── Local dev ─────────────────────────────────────────────────────────────────
|
|
|
|
install: ## Bootstrap: create venv, install all deps (+ dev)
|
|
$(UV) sync --all-extras
|
|
@mkdir -p $(DATA_DIR)
|
|
@echo "$(GREEN)Done.$(RESET) Copy .env.example → .env and fill in your values."
|
|
|
|
sync: ## Sync venv with lockfile (fast, no resolution)
|
|
$(UV) sync
|
|
|
|
lock: ## Re-resolve and update uv.lock
|
|
$(UV) lock
|
|
|
|
lint: typecheck ## Run all linters
|
|
|
|
typecheck: ## Run Pyright type-checker
|
|
$(UV) run pyright
|
|
|
|
# ── Crawler ───────────────────────────────────────────────────────────────────
|
|
|
|
patch-driver: ## Download + patch ChromeDriver once into drivers/ (run after Chrome updates)
|
|
@mkdir -p drivers
|
|
PYTHONPATH=. $(PYTHON) scripts/patch_driver.py
|
|
|
|
dev: ## Start admin panel + RQ worker together (Ctrl-C stops both)
|
|
@mkdir -p $(DATA_DIR)
|
|
@trap 'kill 0' INT TERM EXIT; \
|
|
$(UV) run rq worker --with-scheduler batch & \
|
|
$(PYTHON) main.py admin
|
|
|
|
admin: ## Start the admin panel (http://localhost:8000)
|
|
@mkdir -p $(DATA_DIR)
|
|
$(PYTHON) main.py admin
|
|
|
|
worker: ## Start RQ worker with scheduler (batch queue)
|
|
$(UV) run rq worker --with-scheduler batch
|
|
|
|
otp: ## Run Scenario 1 — SMS-OTP login (PHONE=+98... or all from .env)
|
|
@mkdir -p $(DATA_DIR)
|
|
ifdef PHONE
|
|
$(PYTHON) main.py otp --phone $(PHONE)
|
|
else
|
|
$(PYTHON) main.py otp
|
|
endif
|
|
|
|
fresh: ## Run Scenario 2 — fresh cookie session
|
|
@mkdir -p $(DATA_DIR)
|
|
$(PYTHON) main.py fresh
|
|
|
|
WORKERS ?= 3
|
|
RUNS ?= 10
|
|
STAGGER ?= 500
|
|
|
|
batch: ## Run many parallel runners (WORKERS=3 RUNS=10 STAGGER=500)
|
|
@mkdir -p $(DATA_DIR)
|
|
$(PYTHON) main.py batch --workers $(WORKERS) --runs $(RUNS) --stagger $(STAGGER)
|
|
|
|
# ── Docker ────────────────────────────────────────────────────────────────────
|
|
|
|
build: ## Build all Docker images (admin + worker)
|
|
docker compose build
|
|
|
|
worker-build: ## Rebuild worker with Playwright base image
|
|
docker compose build worker
|
|
|
|
up: ## Start all services in Docker (detached)
|
|
docker compose up -d
|
|
@echo "$(GREEN)Admin panel:$(RESET) http://localhost:8000"
|
|
|
|
down: ## Stop and remove containers
|
|
docker compose down
|
|
|
|
logs: ## Follow container logs
|
|
docker compose logs -f
|
|
|
|
shell: ## Open a shell in the running admin container
|
|
docker compose exec admin bash
|
|
|
|
# Worker one-off commands (override the default RQ worker command)
|
|
docker-otp: ## Run OTP scenario in Docker (PHONE=+98...)
|
|
ifdef PHONE
|
|
docker compose run --rm -e PHONE=$(PHONE) worker python main.py otp --phone $(PHONE)
|
|
else
|
|
docker compose run --rm worker python main.py otp
|
|
endif
|
|
|
|
docker-fresh: ## Run fresh-session scenario in Docker
|
|
docker compose run --rm worker python main.py fresh
|
|
|
|
docker-admin-up: ## Start only the admin panel in Docker (detached)
|
|
docker compose up -d admin
|
|
|
|
# ── Housekeeping ──────────────────────────────────────────────────────────────
|
|
|
|
clean: ## Remove venv, cache, and compiled files
|
|
rm -rf .venv __pycache__ .pytest_cache .mypy_cache
|
|
find . -type d -name __pycache__ -exec rm -rf {} + 2>/dev/null || true
|
|
find . -name "*.pyc" -delete 2>/dev/null || true
|