242 lines
9.2 KiB
Python
242 lines
9.2 KiB
Python
"""SQLite schema and DB helpers (aiosqlite)."""
|
|
from __future__ import annotations
|
|
|
|
import json
|
|
import os
|
|
import sqlite3
|
|
from pathlib import Path
|
|
from typing import Any
|
|
|
|
DB_PATH = os.getenv("DB_PATH", "data/tracker.db")
|
|
|
|
CREATE_DDL = """
|
|
CREATE TABLE IF NOT EXISTS runs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
scenario TEXT NOT NULL,
|
|
identifier TEXT NOT NULL,
|
|
success INTEGER NOT NULL,
|
|
failure_reason TEXT,
|
|
steps_json TEXT NOT NULL DEFAULT '[]',
|
|
total_ms INTEGER NOT NULL DEFAULT 0
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at);
|
|
CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier);
|
|
|
|
-- Dynamic flow configurations editable from the admin panel
|
|
CREATE TABLE IF NOT EXISTS flow_configs (
|
|
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
|
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
|
name TEXT NOT NULL,
|
|
is_active INTEGER NOT NULL DEFAULT 0,
|
|
|
|
-- Search step
|
|
search_texts_json TEXT NOT NULL DEFAULT '[]', -- JSON string[]
|
|
search_box_selector TEXT NOT NULL DEFAULT '',
|
|
search_box_selector_type TEXT NOT NULL DEFAULT 'css', -- 'css' | 'xpath'
|
|
search_submit_selector TEXT NOT NULL DEFAULT '', -- optional — leave blank to use Enter key
|
|
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
|
|
search_results_selector TEXT NOT NULL DEFAULT '', -- container that appears after results load
|
|
|
|
-- Scroll step
|
|
scroll_container_selector TEXT NOT NULL DEFAULT '', -- element to scroll; blank = window
|
|
max_scrolls INTEGER NOT NULL DEFAULT 30,
|
|
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
|
|
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000, -- bail if DOM height unchanged after this
|
|
|
|
-- Item selection
|
|
item_selector_template TEXT NOT NULL DEFAULT '',
|
|
-- Template may contain {item_id} placeholder, e.g. "[data-id='{item_id}']"
|
|
-- or a plain selector like ".result-card" (picks first match)
|
|
item_selector_type TEXT NOT NULL DEFAULT 'css',
|
|
target_item_ids_json TEXT NOT NULL DEFAULT '[]' -- JSON string[] — one click per id
|
|
);
|
|
"""
|
|
|
|
|
|
def init_db() -> None:
|
|
"""Create the DB file and tables (sync, called at startup)."""
|
|
path = Path(DB_PATH)
|
|
path.parent.mkdir(parents=True, exist_ok=True)
|
|
conn = sqlite3.connect(str(path))
|
|
conn.executescript(CREATE_DDL)
|
|
conn.commit()
|
|
conn.close()
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# Async helpers (used by FastAPI routes via aiosqlite)
|
|
# ---------------------------------------------------------------------------
|
|
|
|
async def insert_run(
|
|
scenario: str,
|
|
identifier: str,
|
|
success: bool,
|
|
failure_reason: str | None,
|
|
steps: list[dict[str, object]],
|
|
total_ms: int,
|
|
) -> int:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
cur = await db.execute(
|
|
"""INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms)
|
|
VALUES (?, ?, ?, ?, ?, ?)""",
|
|
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
|
|
)
|
|
await db.commit()
|
|
return int(cur.lastrowid or 0)
|
|
|
|
|
|
# ---------------------------------------------------------------------------
|
|
# flow_configs CRUD
|
|
# ---------------------------------------------------------------------------
|
|
|
|
FlowConfigRow = dict[str, Any]
|
|
|
|
|
|
async def list_flow_configs() -> list[FlowConfigRow]:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
rows = await (await db.execute(
|
|
"SELECT * FROM flow_configs ORDER BY id DESC"
|
|
)).fetchall()
|
|
return [_parse_flow_row(dict(r)) for r in rows]
|
|
|
|
|
|
async def get_flow_config(cfg_id: int) -> FlowConfigRow | None:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
row = await (await db.execute(
|
|
"SELECT * FROM flow_configs WHERE id=?", (cfg_id,)
|
|
)).fetchone()
|
|
return _parse_flow_row(dict(row)) if row else None
|
|
|
|
|
|
async def get_active_flow_config() -> FlowConfigRow | None:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
row = await (await db.execute(
|
|
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1"
|
|
)).fetchone()
|
|
return _parse_flow_row(dict(row)) if row else None
|
|
|
|
|
|
async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int:
|
|
import aiosqlite
|
|
|
|
fields = [
|
|
"name", "is_active",
|
|
"search_texts_json", "search_box_selector", "search_box_selector_type",
|
|
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
|
|
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
|
"item_selector_template", "item_selector_type", "target_item_ids_json",
|
|
]
|
|
|
|
# Serialize list fields
|
|
for key in ("search_texts_json", "target_item_ids_json"):
|
|
if key in data and isinstance(data[key], list):
|
|
data[key] = json.dumps(data[key])
|
|
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
if data.get("is_active"):
|
|
await db.execute("UPDATE flow_configs SET is_active=0")
|
|
|
|
if cfg_id is None:
|
|
placeholders = ", ".join("?" for _ in fields)
|
|
cols = ", ".join(fields)
|
|
values = [data.get(f) for f in fields]
|
|
cur = await db.execute(
|
|
f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values
|
|
)
|
|
row_id = int(cur.lastrowid or 0)
|
|
else:
|
|
set_clause = ", ".join(f"{f}=?" for f in fields)
|
|
values_u = [data.get(f) for f in fields] + [cfg_id]
|
|
await db.execute(f"UPDATE flow_configs SET {set_clause} WHERE id=?", values_u)
|
|
row_id = cfg_id
|
|
|
|
await db.commit()
|
|
return row_id
|
|
|
|
|
|
async def delete_flow_config(cfg_id: int) -> None:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
|
|
await db.commit()
|
|
|
|
|
|
async def activate_flow_config(cfg_id: int) -> None:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
await db.execute("UPDATE flow_configs SET is_active=0")
|
|
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
|
|
await db.commit()
|
|
|
|
|
|
def _parse_flow_row(row: dict[str, Any]) -> FlowConfigRow:
|
|
for key in ("search_texts_json", "target_item_ids_json"):
|
|
if key in row and isinstance(row[key], str):
|
|
try:
|
|
row[key] = json.loads(row[key])
|
|
except (json.JSONDecodeError, TypeError):
|
|
row[key] = []
|
|
return row
|
|
|
|
|
|
async def get_stats() -> dict[str, object]:
|
|
import aiosqlite
|
|
async with aiosqlite.connect(DB_PATH) as db:
|
|
db.row_factory = aiosqlite.Row
|
|
|
|
total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0]
|
|
unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0]
|
|
successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0]
|
|
failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0]
|
|
|
|
failure_reasons_raw = await (
|
|
await db.execute(
|
|
"""SELECT failure_reason, COUNT(*) as cnt
|
|
FROM runs WHERE success=0 AND failure_reason IS NOT NULL
|
|
GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10"""
|
|
)
|
|
).fetchall()
|
|
failure_reasons = [{"reason": r["failure_reason"], "count": r["cnt"]} for r in failure_reasons_raw]
|
|
|
|
scenario_breakdown_raw = await (
|
|
await db.execute(
|
|
"""SELECT scenario, COUNT(*) as total,
|
|
SUM(success) as ok,
|
|
COUNT(*) - SUM(success) as fail
|
|
FROM runs GROUP BY scenario"""
|
|
)
|
|
).fetchall()
|
|
scenario_breakdown = [
|
|
{"scenario": r["scenario"], "total": r["total"], "success": r["ok"], "failure": r["fail"]}
|
|
for r in scenario_breakdown_raw
|
|
]
|
|
|
|
recent_raw = await (
|
|
await db.execute(
|
|
"""SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms
|
|
FROM runs ORDER BY id DESC LIMIT 50"""
|
|
)
|
|
).fetchall()
|
|
recent = [dict(r) for r in recent_raw]
|
|
|
|
return {
|
|
"total_runs": total,
|
|
"unique_identifiers": unique_ids,
|
|
"successes": successes,
|
|
"failures": failures,
|
|
"success_rate": round(successes / total * 100, 1) if total else 0.0,
|
|
"failure_reasons": failure_reasons,
|
|
"scenario_breakdown": scenario_breakdown,
|
|
"recent_runs": recent,
|
|
}
|