124 lines
4.4 KiB
Python
124 lines
4.4 KiB
Python
from types import SimpleNamespace
|
|
from typing import Any
|
|
|
|
import pytest
|
|
from pydantic import BaseModel
|
|
|
|
from resume_agent.llm import LLMError, OpenAILLM
|
|
|
|
|
|
class LoggedResult(BaseModel):
|
|
answer: str
|
|
|
|
|
|
def fake_client(parsed: BaseModel) -> Any:
|
|
class Completions:
|
|
def parse(self, **kwargs: Any) -> Any:
|
|
return SimpleNamespace(
|
|
choices=[SimpleNamespace(message=SimpleNamespace(parsed=parsed))]
|
|
)
|
|
|
|
return SimpleNamespace(
|
|
base_url="https://llm.example/v1/",
|
|
chat=SimpleNamespace(completions=Completions()),
|
|
)
|
|
|
|
|
|
def test_custom_endpoint_defaults_to_chat(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.setenv("RESUME_AGENT_MODEL", "example-model")
|
|
monkeypatch.delenv("RESUME_AGENT_API_STYLE", raising=False)
|
|
monkeypatch.delenv("RESUME_AGENT_LLM_TIMEOUT_SECONDS", raising=False)
|
|
|
|
llm = OpenAILLM()
|
|
|
|
assert llm.model == "example-model"
|
|
assert llm.api_style == "chat"
|
|
assert llm.timeout_seconds == 1800
|
|
assert str(llm.client.base_url) == "https://llm.example/v1/"
|
|
|
|
|
|
def test_custom_endpoint_allows_model_discovery(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.delenv("RESUME_AGENT_MODEL", raising=False)
|
|
|
|
llm = OpenAILLM()
|
|
|
|
assert llm.model is None
|
|
|
|
|
|
def test_parse_requires_model(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.delenv("RESUME_AGENT_MODEL", raising=False)
|
|
|
|
with pytest.raises(LLMError, match="RESUME_AGENT_MODEL"):
|
|
OpenAILLM().parse(dict, "instructions", "input") # type: ignore[type-var]
|
|
|
|
|
|
def test_custom_llm_timeout(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.setenv("RESUME_AGENT_MODEL", "example-model")
|
|
monkeypatch.setenv("RESUME_AGENT_LLM_TIMEOUT_SECONDS", "2700")
|
|
|
|
llm = OpenAILLM()
|
|
|
|
assert llm.timeout_seconds == 2700
|
|
assert llm.client.timeout.read == 2700
|
|
assert llm.client.timeout.connect == 30
|
|
|
|
|
|
def test_llm_timeout_must_be_positive(monkeypatch: pytest.MonkeyPatch) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.setenv("RESUME_AGENT_MODEL", "example-model")
|
|
monkeypatch.setenv("RESUME_AGENT_LLM_TIMEOUT_SECONDS", "0")
|
|
|
|
with pytest.raises(LLMError, match="greater than zero"):
|
|
OpenAILLM()
|
|
|
|
|
|
def test_model_queries_log_payloads_to_stdout(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.setenv("RESUME_AGENT_MODEL", "example-model")
|
|
monkeypatch.setenv("RESUME_AGENT_LOG_MODEL_PAYLOADS", "true")
|
|
llm = OpenAILLM()
|
|
llm.client = fake_client(LoggedResult(answer="done"))
|
|
|
|
with caplog.at_level("INFO", logger="resume_agent.model"):
|
|
llm.parse(LoggedResult, "System logging test", "Private resume input")
|
|
|
|
output = caplog.text
|
|
assert "event=start" in output
|
|
assert "System logging test" in output
|
|
assert "Private resume input" in output
|
|
assert '"answer": "done"' in output
|
|
assert "test-token" not in output
|
|
|
|
|
|
def test_model_payload_logging_can_be_disabled(
|
|
monkeypatch: pytest.MonkeyPatch,
|
|
caplog: pytest.LogCaptureFixture,
|
|
) -> None:
|
|
monkeypatch.setenv("RESUME_AGENT_API_KEY", "test-token")
|
|
monkeypatch.setenv("RESUME_AGENT_BASE_URL", "https://llm.example/v1")
|
|
monkeypatch.setenv("RESUME_AGENT_MODEL", "example-model")
|
|
monkeypatch.setenv("RESUME_AGENT_LOG_MODEL_PAYLOADS", "false")
|
|
llm = OpenAILLM()
|
|
llm.client = fake_client(LoggedResult(answer="done"))
|
|
|
|
with caplog.at_level("INFO", logger="resume_agent.model"):
|
|
llm.parse(LoggedResult, "Hidden system prompt", "Hidden resume input")
|
|
|
|
output = caplog.text
|
|
assert "event=start" in output
|
|
assert "Hidden system prompt" not in output
|
|
assert "Hidden resume input" not in output
|