updates :)

admin/main.py

@@ -1,7 +1,9 @@
 """FastAPI admin application."""
 from __future__ import annotations
 
-import threading
+import os
+import subprocess
+import sys
 from pathlib import Path
 
 from fastapi import FastAPI
@@ -18,24 +20,29 @@ app.include_router(router)
 _STATIC = Path(__file__).parent / "static"
 app.mount("/static", StaticFiles(directory=str(_STATIC)), name="static")
 
+_rq_worker: subprocess.Popen[bytes] | None = None
+
 
 @app.on_event("startup")
 def on_startup() -> None:
+    global _rq_worker
     init_db()
-    _start_huey_consumer()
+    redis_url = os.environ.get("REDIS_URL", "redis://localhost:6379/0")
+    _rq_worker = subprocess.Popen(
+        [sys.executable, "-m", "rq", "worker", "--url", redis_url, "batch"],
+        stdout=subprocess.DEVNULL,
+        stderr=subprocess.DEVNULL,
+    )
 
 
-def _start_huey_consumer() -> None:
-    from huey.consumer import Consumer
-    from crawler.tasks import huey
-
-    class _ThreadConsumer(Consumer):
-        def _set_signal_handlers(self) -> None:
-            pass  # signal.signal() only works on the main thread
-
-    consumer = _ThreadConsumer(huey, workers=1, periodic=False)
-    t = threading.Thread(target=consumer.run, daemon=True, name="huey-consumer")
-    t.start()
+@app.on_event("shutdown")
+def on_shutdown() -> None:
+    if _rq_worker is not None:
+        _rq_worker.terminate()
+        try:
+            _rq_worker.wait(timeout=5)
+        except subprocess.TimeoutExpired:
+            _rq_worker.kill()
 
 
 @app.get("/")