fare-policy-assistant/tests/test_report.py at main · ghzhost/fare-policy-assistant · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
from evals.report import generate_markdown

SUMMARY = {
    "run_at": "2026-06-12T01:00:00+00:00",
    "mode": "full",
    "offline": True,
    "judges_ran": False,
    "answer_model": "mock",
    "judge_model": "mock",
    "prompt_versions": {"system": "v1 2026-06-11"},
    "duration_seconds": 1.0,
    "suites": {"groundedness": {"passed": 1, "total": 2, "pass_rate": 50.0}},
    "total": {"passed": 1, "total": 2},
}

RECORDS = [
    {
        "case_id": "ground-001",
        "suite": "groundedness",
        "mirror_of": None,
        "passed": True,
        "question": "ok?",
        "rationale": "r",
        "answer": "fine [doc:mst-fares]",
        "kind": "answered",
        "passages": [],
        "checks": [],
        "judges": [],
    },
    {
        "case_id": "ground-002",
        "suite": "groundedness",
        "mirror_of": None,
        "passed": False,
        "question": "how much is the pass?",
        "rationale": "fare table",
        "answer": "no idea",
        "kind": "answered",
        "passages": [
            {"chunk_id": "mst-fares#1", "section": "Fares", "score": 9.1, "text": "x" * 300}
        ],
        "checks": [{"name": "citation_present_and_resolvable", "passed": False, "detail": "none"}],
        "judges": [],
    },
]


def test_scoreboard_and_failures_present():
    md = generate_markdown(SUMMARY, RECORDS)
    assert "| groundedness | 1 | 2 | 50.0% |" in md
    assert "ground-002" in md
    assert "citation_present_and_resolvable: none" in md
    # Passing cases are not dumped as failures.
    assert "### ground-001" not in md


def test_offline_run_is_labeled():
    md = generate_markdown(SUMMARY, RECORDS)
    assert "deterministic checks only" in md
    assert "skipped, not passed" in md


def test_variance_section_always_documents_the_tooling():
    md = generate_markdown(SUMMARY, RECORDS)
    assert "## Measuring variance" in md


def test_failure_trace_shows_passage_provenance():
    """Issue #142: an outside reader checking a dated claim in a failure trace
    needs the same source/fetch-date provenance the answer model and judge
    were shown, not just chunk id and score."""
    records = [
        {
            "case_id": "fresh-999",
            "suite": "groundedness",
            "mirror_of": None,
            "passed": False,
            "question": "how much is the pass?",
            "rationale": "fare table",
            "answer": "no idea",
            "kind": "answered",
            "passages": [
                {
                    "chunk_id": "mst-fares#1",
                    "doc_id": "mst-fares",
                    "agency": "MST",
                    "doc_title": "Fares",
                    "url": "https://mst.org/fares/",
                    "fetch_date": "2026-06-12",
                    "section": "Fares",
                    "score": 9.1,
                    "text": "x" * 300,
                }
            ],
            "checks": [],
            "judges": [],
        }
    ]
    md = generate_markdown(SUMMARY, records)
    assert "Fares — Fares, score 9.1, fetched 2026-06-12" in md
    assert "--replicates" in md
    assert "evals.compare" in md


def test_scoreboard_renders_wilson_interval_when_replicated():
    summary = {
        **SUMMARY,
        "replicates": 3,
        "suites": {
            "groundedness": {
                "passed": 1,
                "total": 2,
                "pass_rate": 50.0,
                "ci_low": 23.7,
                "ci_high": 76.3,
                "replicates": 3,
            }
        },
    }
    md = generate_markdown(summary, RECORDS)
    assert "| groundedness | 1 | 2 | 50.0% (23.7–76.3) |" in md
    assert "mean over 3 replicate runs" in md