init
This commit is contained in:
+241
@@ -0,0 +1,241 @@
|
||||
"""SQLite schema and DB helpers (aiosqlite)."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import sqlite3
|
||||
from pathlib import Path
|
||||
from typing import Any
|
||||
|
||||
DB_PATH = os.getenv("DB_PATH", "data/tracker.db")
|
||||
|
||||
CREATE_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
scenario TEXT NOT NULL,
|
||||
identifier TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
failure_reason TEXT,
|
||||
steps_json TEXT NOT NULL DEFAULT '[]',
|
||||
total_ms INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier);
|
||||
|
||||
-- Dynamic flow configurations editable from the admin panel
|
||||
CREATE TABLE IF NOT EXISTS flow_configs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
name TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 0,
|
||||
|
||||
-- Search step
|
||||
search_texts_json TEXT NOT NULL DEFAULT '[]', -- JSON string[]
|
||||
search_box_selector TEXT NOT NULL DEFAULT '',
|
||||
search_box_selector_type TEXT NOT NULL DEFAULT 'css', -- 'css' | 'xpath'
|
||||
search_submit_selector TEXT NOT NULL DEFAULT '', -- optional — leave blank to use Enter key
|
||||
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_results_selector TEXT NOT NULL DEFAULT '', -- container that appears after results load
|
||||
|
||||
-- Scroll step
|
||||
scroll_container_selector TEXT NOT NULL DEFAULT '', -- element to scroll; blank = window
|
||||
max_scrolls INTEGER NOT NULL DEFAULT 30,
|
||||
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
|
||||
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000, -- bail if DOM height unchanged after this
|
||||
|
||||
-- Item selection
|
||||
item_selector_template TEXT NOT NULL DEFAULT '',
|
||||
-- Template may contain {item_id} placeholder, e.g. "[data-id='{item_id}']"
|
||||
-- or a plain selector like ".result-card" (picks first match)
|
||||
item_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
target_item_ids_json TEXT NOT NULL DEFAULT '[]' -- JSON string[] — one click per id
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
"""Create the DB file and tables (sync, called at startup)."""
|
||||
path = Path(DB_PATH)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.executescript(CREATE_DDL)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# Async helpers (used by FastAPI routes via aiosqlite)
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
async def insert_run(
|
||||
scenario: str,
|
||||
identifier: str,
|
||||
success: bool,
|
||||
failure_reason: str | None,
|
||||
steps: list[dict[str, object]],
|
||||
total_ms: int,
|
||||
) -> int:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
cur = await db.execute(
|
||||
"""INSERT INTO runs (scenario, identifier, success, failure_reason, steps_json, total_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
|
||||
)
|
||||
await db.commit()
|
||||
return int(cur.lastrowid or 0)
|
||||
|
||||
|
||||
# ---------------------------------------------------------------------------
|
||||
# flow_configs CRUD
|
||||
# ---------------------------------------------------------------------------
|
||||
|
||||
FlowConfigRow = dict[str, Any]
|
||||
|
||||
|
||||
async def list_flow_configs() -> list[FlowConfigRow]:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
rows = await (await db.execute(
|
||||
"SELECT * FROM flow_configs ORDER BY id DESC"
|
||||
)).fetchall()
|
||||
return [_parse_flow_row(dict(r)) for r in rows]
|
||||
|
||||
|
||||
async def get_flow_config(cfg_id: int) -> FlowConfigRow | None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE id=?", (cfg_id,)
|
||||
)).fetchone()
|
||||
return _parse_flow_row(dict(row)) if row else None
|
||||
|
||||
|
||||
async def get_active_flow_config() -> FlowConfigRow | None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1"
|
||||
)).fetchone()
|
||||
return _parse_flow_row(dict(row)) if row else None
|
||||
|
||||
|
||||
async def upsert_flow_config(data: dict[str, Any], cfg_id: int | None = None) -> int:
|
||||
import aiosqlite
|
||||
|
||||
fields = [
|
||||
"name", "is_active",
|
||||
"search_texts_json", "search_box_selector", "search_box_selector_type",
|
||||
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
|
||||
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
||||
"item_selector_template", "item_selector_type", "target_item_ids_json",
|
||||
]
|
||||
|
||||
# Serialize list fields
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if key in data and isinstance(data[key], list):
|
||||
data[key] = json.dumps(data[key])
|
||||
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
if data.get("is_active"):
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
|
||||
if cfg_id is None:
|
||||
placeholders = ", ".join("?" for _ in fields)
|
||||
cols = ", ".join(fields)
|
||||
values = [data.get(f) for f in fields]
|
||||
cur = await db.execute(
|
||||
f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values
|
||||
)
|
||||
row_id = int(cur.lastrowid or 0)
|
||||
else:
|
||||
set_clause = ", ".join(f"{f}=?" for f in fields)
|
||||
values_u = [data.get(f) for f in fields] + [cfg_id]
|
||||
await db.execute(f"UPDATE flow_configs SET {set_clause} WHERE id=?", values_u)
|
||||
row_id = cfg_id
|
||||
|
||||
await db.commit()
|
||||
return row_id
|
||||
|
||||
|
||||
async def delete_flow_config(cfg_id: int) -> None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
|
||||
async def activate_flow_config(cfg_id: int) -> None:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
|
||||
def _parse_flow_row(row: dict[str, Any]) -> FlowConfigRow:
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if key in row and isinstance(row[key], str):
|
||||
try:
|
||||
row[key] = json.loads(row[key])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
row[key] = []
|
||||
return row
|
||||
|
||||
|
||||
async def get_stats() -> dict[str, object]:
|
||||
import aiosqlite
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
|
||||
total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0]
|
||||
unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0]
|
||||
successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0]
|
||||
failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0]
|
||||
|
||||
failure_reasons_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT failure_reason, COUNT(*) as cnt
|
||||
FROM runs WHERE success=0 AND failure_reason IS NOT NULL
|
||||
GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10"""
|
||||
)
|
||||
).fetchall()
|
||||
failure_reasons = [{"reason": r["failure_reason"], "count": r["cnt"]} for r in failure_reasons_raw]
|
||||
|
||||
scenario_breakdown_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT scenario, COUNT(*) as total,
|
||||
SUM(success) as ok,
|
||||
COUNT(*) - SUM(success) as fail
|
||||
FROM runs GROUP BY scenario"""
|
||||
)
|
||||
).fetchall()
|
||||
scenario_breakdown = [
|
||||
{"scenario": r["scenario"], "total": r["total"], "success": r["ok"], "failure": r["fail"]}
|
||||
for r in scenario_breakdown_raw
|
||||
]
|
||||
|
||||
recent_raw = await (
|
||||
await db.execute(
|
||||
"""SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms
|
||||
FROM runs ORDER BY id DESC LIMIT 50"""
|
||||
)
|
||||
).fetchall()
|
||||
recent = [dict(r) for r in recent_raw]
|
||||
|
||||
return {
|
||||
"total_runs": total,
|
||||
"unique_identifiers": unique_ids,
|
||||
"successes": successes,
|
||||
"failures": failures,
|
||||
"success_rate": round(successes / total * 100, 1) if total else 0.0,
|
||||
"failure_reasons": failure_reasons,
|
||||
"scenario_breakdown": scenario_breakdown,
|
||||
"recent_runs": recent,
|
||||
}
|
||||
Reference in New Issue
Block a user