"""SQLite schema and DB helpers (aiosqlite).""" from __future__ import annotations import json import os import sqlite3 from pathlib import Path from typing import Any DB_PATH = os.getenv("DB_PATH", "data/tracker.db") CREATE_DDL = """ CREATE TABLE IF NOT EXISTS runs ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), scenario TEXT NOT NULL, identifier TEXT NOT NULL, success INTEGER NOT NULL, failure_reason TEXT, steps_json TEXT NOT NULL DEFAULT '[]', total_ms INTEGER NOT NULL DEFAULT 0 ); CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario); CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at); CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier); -- Dynamic flow configurations editable from the admin panel CREATE TABLE IF NOT EXISTS flow_configs ( id INTEGER PRIMARY KEY AUTOINCREMENT, created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')), name TEXT NOT NULL, is_active INTEGER NOT NULL DEFAULT 0, -- Search step search_texts_json TEXT NOT NULL DEFAULT '[]', -- JSON string[] search_box_selector TEXT NOT NULL DEFAULT '', search_box_selector_type TEXT NOT NULL DEFAULT 'css', -- 'css' | 'xpath' search_submit_selector TEXT NOT NULL DEFAULT '', -- optional — leave blank to use Enter key search_submit_selector_type TEXT NOT NULL DEFAULT 'css', search_results_selector TEXT NOT NULL DEFAULT '', -- container that appears after results load -- Scroll step scroll_container_selector TEXT NOT NULL DEFAULT '', -- element to scroll; blank = window max_scrolls INTEGER NOT NULL DEFAULT 30, scroll_pause_ms INTEGER NOT NULL DEFAULT 1200, no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000, -- bail if DOM height unchanged after this -- Item selection item_selector_template TEXT NOT NULL DEFAULT '', -- Template may contain {item_id} placeholder, e.g. "[data-id='{item_id}']" -- or a plain selector like ".result-card" (picks first match) item_selector_type TEXT NOT NULL DEFAULT 'css', target_item_ids_json TEXT NOT NULL DEFAULT '[]' -- JSON string[] — one click per id ); """ def init_db() -> None: """Create the DB file and tables (sync, called at startup).""" path = Path(DB_PATH) path.parent.mkdir(parents=True, exist_ok=True) conn = sqlite3.connect(str(path)) conn.executescript(CREATE_DDL) conn.commit() conn.close() # --------------------------------------------------------------------------- # Async helpers (used by FastAPI routes via aiosqlite) # --------------------------------------------------------------------------- async def insert_run( scenario: str, identifier: str, success: bool, failure_reason: str | None, steps: list[dict[str, object]], total_ms: int, ) -> int: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: cur = await db.execute( """INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms) VALUES (?, ?, ?, ?, ?, ?)""", (scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms), ) await db.commit() return int(cur.lastrowid or 0) # --------------------------------------------------------------------------- # flow_configs CRUD # --------------------------------------------------------------------------- FlowConfigRow = dict[str, Any] async def list_flow_configs() -> list[FlowConfigRow]: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: db.row_factory = aiosqlite.Row rows = await (await db.execute( "SELECT * FROM flow_configs ORDER BY id DESC" )).fetchall() return [_parse_flow_row(dict(r)) for r in rows] async def get_flow_config(cfg_id: int) -> FlowConfigRow | None: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: db.row_factory = aiosqlite.Row row = await (await db.execute( "SELECT * FROM flow_configs WHERE id=?", (cfg_id,) )).fetchone() return _parse_flow_row(dict(row)) if row else None async def get_active_flow_config() -> FlowConfigRow | None: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: db.row_factory = aiosqlite.Row row = await (await db.execute( "SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1" )).fetchone() return _parse_flow_row(dict(row)) if row else None async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int: import aiosqlite fields = [ "name", "is_active", "search_texts_json", "search_box_selector", "search_box_selector_type", "search_submit_selector", "search_submit_selector_type", "search_results_selector", "scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms", "item_selector_template", "item_selector_type", "target_item_ids_json", ] # Serialize list fields for key in ("search_texts_json", "target_item_ids_json"): if key in data and isinstance(data[key], list): data[key] = json.dumps(data[key]) async with aiosqlite.connect(DB_PATH) as db: if data.get("is_active"): await db.execute("UPDATE flow_configs SET is_active=0") if cfg_id is None: placeholders = ", ".join("?" for _ in fields) cols = ", ".join(fields) values = [data.get(f) for f in fields] cur = await db.execute( f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values ) row_id = int(cur.lastrowid or 0) else: set_clause = ", ".join(f"{f}=?" for f in fields) values_u = [data.get(f) for f in fields] + [cfg_id] await db.execute(f"UPDATE flow_configs SET {set_clause} WHERE id=?", values_u) row_id = cfg_id await db.commit() return row_id async def delete_flow_config(cfg_id: int) -> None: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,)) await db.commit() async def activate_flow_config(cfg_id: int) -> None: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: await db.execute("UPDATE flow_configs SET is_active=0") await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,)) await db.commit() def _parse_flow_row(row: dict[str, Any]) -> FlowConfigRow: for key in ("search_texts_json", "target_item_ids_json"): if key in row and isinstance(row[key], str): try: row[key] = json.loads(row[key]) except (json.JSONDecodeError, TypeError): row[key] = [] return row async def get_stats() -> dict[str, object]: import aiosqlite async with aiosqlite.connect(DB_PATH) as db: db.row_factory = aiosqlite.Row total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0] unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0] successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0] failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0] failure_reasons_raw = await ( await db.execute( """SELECT failure_reason, COUNT(*) as cnt FROM runs WHERE success=0 AND failure_reason IS NOT NULL GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10""" ) ).fetchall() failure_reasons = [{"reason": r["failure_reason"], "count": r["cnt"]} for r in failure_reasons_raw] scenario_breakdown_raw = await ( await db.execute( """SELECT scenario, COUNT(*) as total, SUM(success) as ok, COUNT(*) - SUM(success) as fail FROM runs GROUP BY scenario""" ) ).fetchall() scenario_breakdown = [ {"scenario": r["scenario"], "total": r["total"], "success": r["ok"], "failure": r["fail"]} for r in scenario_breakdown_raw ] recent_raw = await ( await db.execute( """SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms FROM runs ORDER BY id DESC LIMIT 50""" ) ).fetchall() recent = [dict(r) for r in recent_raw] return { "total_runs": total, "unique_identifiers": unique_ids, "successes": successes, "failures": failures, "success_rate": round(successes / total * 100, 1) if total else 0.0, "failure_reasons": failure_reasons, "scenario_breakdown": scenario_breakdown, "recent_runs": recent, }