added db
This commit is contained in:
@@ -0,0 +1,4 @@
|
||||
from db.connection import DB_PATH, get_db, init_db
|
||||
from db.repositories import flow_config_repo, runs_repo
|
||||
|
||||
__all__ = ["DB_PATH", "get_db", "init_db", "runs_repo", "flow_config_repo"]
|
||||
@@ -0,0 +1,67 @@
|
||||
"""Database connection management and schema initialisation."""
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import sqlite3
|
||||
from contextlib import asynccontextmanager
|
||||
from pathlib import Path
|
||||
from typing import AsyncGenerator
|
||||
|
||||
import aiosqlite
|
||||
|
||||
DB_PATH: str = os.getenv("DB_PATH", "data/tracker.db")
|
||||
|
||||
_DDL = """
|
||||
CREATE TABLE IF NOT EXISTS runs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
scenario TEXT NOT NULL,
|
||||
identifier TEXT NOT NULL,
|
||||
success INTEGER NOT NULL,
|
||||
failure_reason TEXT,
|
||||
steps_json TEXT NOT NULL DEFAULT '[]',
|
||||
total_ms INTEGER NOT NULL DEFAULT 0
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_scenario ON runs(scenario);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_created_at ON runs(created_at);
|
||||
CREATE INDEX IF NOT EXISTS idx_runs_identifier ON runs(identifier);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS flow_configs (
|
||||
id INTEGER PRIMARY KEY AUTOINCREMENT,
|
||||
created_at TEXT NOT NULL DEFAULT (strftime('%Y-%m-%dT%H:%M:%SZ', 'now')),
|
||||
name TEXT NOT NULL,
|
||||
is_active INTEGER NOT NULL DEFAULT 0,
|
||||
search_texts_json TEXT NOT NULL DEFAULT '[]',
|
||||
search_box_selector TEXT NOT NULL DEFAULT '',
|
||||
search_box_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_submit_selector TEXT NOT NULL DEFAULT '',
|
||||
search_submit_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
search_results_selector TEXT NOT NULL DEFAULT '',
|
||||
scroll_container_selector TEXT NOT NULL DEFAULT '',
|
||||
max_scrolls INTEGER NOT NULL DEFAULT 30,
|
||||
scroll_pause_ms INTEGER NOT NULL DEFAULT 1200,
|
||||
no_new_content_timeout_ms INTEGER NOT NULL DEFAULT 3000,
|
||||
item_selector_template TEXT NOT NULL DEFAULT '',
|
||||
item_selector_type TEXT NOT NULL DEFAULT 'css',
|
||||
target_item_ids_json TEXT NOT NULL DEFAULT '[]'
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def init_db() -> None:
|
||||
"""Create the DB file and apply the schema (sync, called once at startup)."""
|
||||
path = Path(DB_PATH)
|
||||
path.parent.mkdir(parents=True, exist_ok=True)
|
||||
conn = sqlite3.connect(str(path))
|
||||
conn.executescript(_DDL)
|
||||
conn.commit()
|
||||
conn.close()
|
||||
|
||||
|
||||
@asynccontextmanager
|
||||
async def get_db() -> AsyncGenerator[aiosqlite.Connection, None]:
|
||||
"""Async context manager — yields an open, Row-factory-enabled connection."""
|
||||
async with aiosqlite.connect(DB_PATH) as db:
|
||||
db.row_factory = aiosqlite.Row
|
||||
yield db
|
||||
@@ -0,0 +1,7 @@
|
||||
from db.repositories.flow_configs import FlowConfigRepository, flow_config_repo
|
||||
from db.repositories.runs import RunsRepository, runs_repo
|
||||
|
||||
__all__ = [
|
||||
"RunsRepository", "runs_repo",
|
||||
"FlowConfigRepository", "flow_config_repo",
|
||||
]
|
||||
@@ -0,0 +1,98 @@
|
||||
"""Repository for the `flow_configs` table."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from db.connection import get_db
|
||||
|
||||
FlowConfigRow = dict[str, Any]
|
||||
|
||||
_FIELDS = [
|
||||
"name", "is_active",
|
||||
"search_texts_json", "search_box_selector", "search_box_selector_type",
|
||||
"search_submit_selector", "search_submit_selector_type", "search_results_selector",
|
||||
"scroll_container_selector", "max_scrolls", "scroll_pause_ms", "no_new_content_timeout_ms",
|
||||
"item_selector_template", "item_selector_type", "target_item_ids_json",
|
||||
]
|
||||
|
||||
|
||||
def _deserialise(row: dict[str, Any]) -> FlowConfigRow:
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if isinstance(row.get(key), str):
|
||||
try:
|
||||
row[key] = json.loads(row[key])
|
||||
except (json.JSONDecodeError, TypeError):
|
||||
row[key] = []
|
||||
return row
|
||||
|
||||
|
||||
def _serialise(data: dict[str, Any]) -> dict[str, Any]:
|
||||
out = dict(data)
|
||||
for key in ("search_texts_json", "target_item_ids_json"):
|
||||
if isinstance(out.get(key), list):
|
||||
out[key] = json.dumps(out[key])
|
||||
return out
|
||||
|
||||
|
||||
class FlowConfigRepository:
|
||||
async def list(self) -> list[FlowConfigRow]:
|
||||
async with get_db() as db:
|
||||
rows = await (await db.execute(
|
||||
"SELECT * FROM flow_configs ORDER BY id DESC"
|
||||
)).fetchall()
|
||||
return [_deserialise(dict(r)) for r in rows]
|
||||
|
||||
async def get(self, cfg_id: int) -> FlowConfigRow | None:
|
||||
async with get_db() as db:
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE id=?", (cfg_id,)
|
||||
)).fetchone()
|
||||
return _deserialise(dict(row)) if row else None
|
||||
|
||||
async def get_active(self) -> FlowConfigRow | None:
|
||||
async with get_db() as db:
|
||||
row = await (await db.execute(
|
||||
"SELECT * FROM flow_configs WHERE is_active=1 ORDER BY id DESC LIMIT 1"
|
||||
)).fetchone()
|
||||
return _deserialise(dict(row)) if row else None
|
||||
|
||||
async def create(self, data: dict[str, Any]) -> int:
|
||||
data = _serialise(data)
|
||||
async with get_db() as db:
|
||||
if data.get("is_active"):
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
placeholders = ", ".join("?" for _ in _FIELDS)
|
||||
cols = ", ".join(_FIELDS)
|
||||
values = [data.get(f) for f in _FIELDS]
|
||||
cur = await db.execute(
|
||||
f"INSERT INTO flow_configs ({cols}) VALUES ({placeholders})", values
|
||||
)
|
||||
await db.commit()
|
||||
return int(cur.lastrowid or 0)
|
||||
|
||||
async def update(self, cfg_id: int, data: dict[str, Any]) -> None:
|
||||
data = _serialise(data)
|
||||
async with get_db() as db:
|
||||
if data.get("is_active"):
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
set_clause = ", ".join(f"{f}=?" for f in _FIELDS)
|
||||
values = [data.get(f) for f in _FIELDS] + [cfg_id]
|
||||
await db.execute(
|
||||
f"UPDATE flow_configs SET {set_clause} WHERE id=?", values
|
||||
)
|
||||
await db.commit()
|
||||
|
||||
async def activate(self, cfg_id: int) -> None:
|
||||
async with get_db() as db:
|
||||
await db.execute("UPDATE flow_configs SET is_active=0")
|
||||
await db.execute("UPDATE flow_configs SET is_active=1 WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
async def delete(self, cfg_id: int) -> None:
|
||||
async with get_db() as db:
|
||||
await db.execute("DELETE FROM flow_configs WHERE id=?", (cfg_id,))
|
||||
await db.commit()
|
||||
|
||||
|
||||
flow_config_repo = FlowConfigRepository()
|
||||
@@ -0,0 +1,82 @@
|
||||
"""Repository for the `runs` table."""
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
from typing import Any
|
||||
|
||||
from db.connection import get_db
|
||||
|
||||
|
||||
class RunsRepository:
|
||||
async def insert(
|
||||
self,
|
||||
scenario: str,
|
||||
identifier: str,
|
||||
success: bool,
|
||||
failure_reason: str | None,
|
||||
steps: list[dict[str, Any]],
|
||||
total_ms: int,
|
||||
) -> int:
|
||||
async with get_db() as db:
|
||||
cur = await db.execute(
|
||||
"""INSERT INTO runs
|
||||
(scenario, identifier, success, failure_reason, steps_json, total_ms)
|
||||
VALUES (?, ?, ?, ?, ?, ?)""",
|
||||
(scenario, identifier, int(success), failure_reason, json.dumps(steps), total_ms),
|
||||
)
|
||||
await db.commit()
|
||||
return int(cur.lastrowid or 0)
|
||||
|
||||
async def stats(self) -> dict[str, Any]:
|
||||
async with get_db() as db:
|
||||
total = (await (await db.execute("SELECT COUNT(*) FROM runs")).fetchone())[0]
|
||||
unique_ids = (await (await db.execute("SELECT COUNT(DISTINCT identifier) FROM runs")).fetchone())[0]
|
||||
successes = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=1")).fetchone())[0]
|
||||
failures = (await (await db.execute("SELECT COUNT(*) FROM runs WHERE success=0")).fetchone())[0]
|
||||
|
||||
failure_reasons = [
|
||||
{"reason": r["failure_reason"], "count": r["cnt"]}
|
||||
for r in await (await db.execute(
|
||||
"""SELECT failure_reason, COUNT(*) as cnt
|
||||
FROM runs WHERE success=0 AND failure_reason IS NOT NULL
|
||||
GROUP BY failure_reason ORDER BY cnt DESC LIMIT 10"""
|
||||
)).fetchall()
|
||||
]
|
||||
|
||||
scenario_breakdown = [
|
||||
{
|
||||
"scenario": r["scenario"],
|
||||
"total": r["total"],
|
||||
"success": r["ok"],
|
||||
"failure": r["fail"],
|
||||
}
|
||||
for r in await (await db.execute(
|
||||
"""SELECT scenario,
|
||||
COUNT(*) as total,
|
||||
SUM(success) as ok,
|
||||
COUNT(*) - SUM(success) as fail
|
||||
FROM runs GROUP BY scenario"""
|
||||
)).fetchall()
|
||||
]
|
||||
|
||||
recent = [
|
||||
dict(r)
|
||||
for r in await (await db.execute(
|
||||
"""SELECT id, created_at, scenario, identifier, success, failure_reason, total_ms
|
||||
FROM runs ORDER BY id DESC LIMIT 50"""
|
||||
)).fetchall()
|
||||
]
|
||||
|
||||
return {
|
||||
"total_runs": total,
|
||||
"unique_identifiers": unique_ids,
|
||||
"successes": successes,
|
||||
"failures": failures,
|
||||
"success_rate": round(successes / total * 100, 1) if total else 0.0,
|
||||
"failure_reasons": failure_reasons,
|
||||
"scenario_breakdown": scenario_breakdown,
|
||||
"recent_runs": recent,
|
||||
}
|
||||
|
||||
|
||||
runs_repo = RunsRepository()
|
||||
Reference in New Issue
Block a user