Add configurable evidence guard for resume tailoring
This commit is contained in:
@@ -140,6 +140,34 @@ def test_tailoring_strength_rejects_out_of_range_values() -> None:
|
||||
tailor_resume(FakeLLM([]), profile(), "long job post", tailoring_strength=101)
|
||||
|
||||
|
||||
def test_profile_evidence_mode_allows_claims_without_fact_ids() -> None:
|
||||
flexible = package()
|
||||
for item in flexible.resume.summary:
|
||||
item.evidence_ids = []
|
||||
for section in flexible.resume.sections:
|
||||
for item in section.items:
|
||||
item.evidence_ids = []
|
||||
llm = FakeLLM([flexible, flexible])
|
||||
|
||||
result = tailor_resume(
|
||||
llm,
|
||||
profile(),
|
||||
"long job post",
|
||||
evidence_mode="profile",
|
||||
)
|
||||
|
||||
assert result.resume.summary[0].evidence_ids == []
|
||||
assert "Per-claim evidence IDs are optional" in llm.calls[1][0]
|
||||
|
||||
|
||||
def test_strict_evidence_mode_still_rejects_missing_fact_ids() -> None:
|
||||
flexible = package()
|
||||
flexible.resume.summary[0].evidence_ids = []
|
||||
|
||||
with pytest.raises(ValueError, match="no evidence"):
|
||||
tailor_resume(FakeLLM([flexible]), profile(), "long job post")
|
||||
|
||||
|
||||
def test_revision_chat_rewrites_and_audits_current_package() -> None:
|
||||
current = package()
|
||||
revised = package()
|
||||
|
||||
@@ -217,6 +217,77 @@ def test_background_tailoring_returns_pollable_task(tmp_path: Path) -> None:
|
||||
assert (output_dir / "resume-ohmycv.md").is_file()
|
||||
|
||||
|
||||
def test_profile_based_tailoring_allows_empty_evidence_ids(tmp_path: Path) -> None:
|
||||
profile_path = tmp_path / "profile.json"
|
||||
output_dir = tmp_path / "output"
|
||||
save_json(sample_profile(), profile_path)
|
||||
flexible = sample_package()
|
||||
for item in flexible.resume.summary:
|
||||
item.evidence_ids = []
|
||||
for section in flexible.resume.sections:
|
||||
for item in section.items:
|
||||
item.evidence_ids = []
|
||||
app = create_app(
|
||||
profile_path=profile_path,
|
||||
output_dir=output_dir,
|
||||
llm_factory=lambda: FakeLLM([flexible, flexible]), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
response = client.post(
|
||||
"/api/tailor",
|
||||
json={
|
||||
"job_url": None,
|
||||
"job_text": "Backend engineer role requiring Python and API performance. " * 3,
|
||||
"evidence_mode": "profile",
|
||||
},
|
||||
)
|
||||
|
||||
assert response.status_code == 200
|
||||
assert response.json()["resume"]["summary"][0]["evidence_ids"] == []
|
||||
assert (output_dir / "evidence-mode.txt").read_text() == "profile"
|
||||
assert "<!-- evidence:" not in (output_dir / "resume-audited.md").read_text()
|
||||
|
||||
|
||||
def test_app_wide_guard_off_forces_profile_mode_and_persists(tmp_path: Path) -> None:
|
||||
profile_path = tmp_path / "profile.json"
|
||||
output_dir = tmp_path / "output"
|
||||
settings_path = tmp_path / "app-settings.json"
|
||||
save_json(sample_profile(), profile_path)
|
||||
flexible = sample_package()
|
||||
for item in flexible.resume.summary:
|
||||
item.evidence_ids = []
|
||||
for section in flexible.resume.sections:
|
||||
for item in section.items:
|
||||
item.evidence_ids = []
|
||||
app = create_app(
|
||||
profile_path=profile_path,
|
||||
output_dir=output_dir,
|
||||
settings_path=settings_path,
|
||||
llm_factory=lambda: FakeLLM([flexible, flexible]), # type: ignore[arg-type]
|
||||
)
|
||||
|
||||
with TestClient(app) as client:
|
||||
updated = client.put("/api/settings", json={"evidence_guard": False})
|
||||
status = client.get("/api/status")
|
||||
response = client.post(
|
||||
"/api/tailor",
|
||||
json={
|
||||
"job_url": None,
|
||||
"job_text": "Backend engineer role requiring Python and API performance. " * 3,
|
||||
"evidence_mode": "strict",
|
||||
},
|
||||
)
|
||||
|
||||
assert updated.status_code == 200
|
||||
assert updated.json() == {"evidence_guard": False}
|
||||
assert status.json()["evidence_guard"] is False
|
||||
assert response.status_code == 200
|
||||
assert response.json()["resume"]["summary"][0]["evidence_ids"] == []
|
||||
assert (output_dir / "evidence-mode.txt").read_text() == "profile"
|
||||
assert '"evidence_guard": false' in settings_path.read_text()
|
||||
|
||||
|
||||
def test_tailor_requires_exactly_one_job_source(tmp_path: Path) -> None:
|
||||
profile_path = tmp_path / "profile.json"
|
||||
save_json(sample_profile(), profile_path)
|
||||
|
||||
Reference in New Issue
Block a user