wip

db/connection.py

@@ -37,14 +37,22 @@ CREATE TABLE IF NOT EXISTS flow_configs (
     search_box_selector_type    TEXT    NOT NULL DEFAULT 'css',
     search_submit_selector      TEXT    NOT NULL DEFAULT '',
     search_submit_selector_type TEXT    NOT NULL DEFAULT 'css',
-    search_results_selector     TEXT    NOT NULL DEFAULT '',
-    scroll_container_selector   TEXT    NOT NULL DEFAULT '',
+    search_results_selector      TEXT    NOT NULL DEFAULT '',
+    search_results_selector_type TEXT    NOT NULL DEFAULT 'css',
+    scroll_container_selector      TEXT    NOT NULL DEFAULT '',
+    scroll_container_selector_type TEXT    NOT NULL DEFAULT 'css',
     max_scrolls                 INTEGER NOT NULL DEFAULT 30,
     scroll_pause_ms             INTEGER NOT NULL DEFAULT 1200,
     no_new_content_timeout_ms   INTEGER NOT NULL DEFAULT 3000,
+    pagination_selector         TEXT    NOT NULL DEFAULT '',
+    pagination_selector_type    TEXT    NOT NULL DEFAULT 'css',
+    max_pages                   INTEGER NOT NULL DEFAULT 10,
+    pagination_wait_ms          INTEGER NOT NULL DEFAULT 1500,
     item_selector_template      TEXT    NOT NULL DEFAULT '',
     item_selector_type          TEXT    NOT NULL DEFAULT 'css',
-    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]'
+    item_url_template           TEXT    NOT NULL DEFAULT '',
+    target_item_ids_json        TEXT    NOT NULL DEFAULT '[]',
+    target_url                  TEXT    NOT NULL DEFAULT ''
 );
 """
 
@@ -55,10 +63,29 @@ def init_db() -> None:
     path.parent.mkdir(parents=True, exist_ok=True)
     conn = sqlite3.connect(str(path))
     conn.executescript(_DDL)
+    # Additive migrations for columns added after initial release
+    _migrate(conn)
     conn.commit()
     conn.close()
 
 
+def _migrate(conn: sqlite3.Connection) -> None:
+    existing = {row[1] for row in conn.execute("PRAGMA table_info(flow_configs)")}
+    additions = {
+        "search_results_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "item_url_template": "TEXT NOT NULL DEFAULT ''",
+        "scroll_container_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "pagination_selector": "TEXT NOT NULL DEFAULT ''",
+        "pagination_selector_type": "TEXT NOT NULL DEFAULT 'css'",
+        "max_pages": "INTEGER NOT NULL DEFAULT 10",
+        "pagination_wait_ms": "INTEGER NOT NULL DEFAULT 1500",
+        "target_url": "TEXT NOT NULL DEFAULT ''",
+    }
+    for col, definition in additions.items():
+        if col not in existing:
+            conn.execute(f"ALTER TABLE flow_configs ADD COLUMN {col} {definition}")
+
+
 @asynccontextmanager
 async def get_db() -> AsyncGenerator[aiosqlite.Connection, None]:
     """Async context manager — yields an open, Row-factory-enabled connection."""