coverage.py

"""Eval coverage map: which fare provisions are actually tested, and which are
corpus blind spots.

A scoreboard says how many cases pass; it does not say whether the cases *reach*
every provision the corpus publishes. This builds an agency x reduced-fare-
program matrix, counts the eval cases that touch each cell, and — crucially —
flags cells the **corpus covers but no case tests** (a real blind spot) apart
from cells the corpus simply does not have (not applicable).

    python -m evals.coverage            # print the matrix + blind spots
    python -m evals.coverage --write    # also regenerate docs/eval-coverage.md

Detection is keyword-based and deliberately conservative: it can under-count a
case that tests a program without naming it, so the matrix is a floor on
coverage, not a ceiling. That is the safe direction for a blind-spot report.
"""

from __future__ import annotations

import re
import sys

from assistant import config, domain
from assistant.ingest import load_chunks
from assistant.retrieve import detect_agencies
from evals.runner import load_suites

# Reduced-fare programs / rider classes to track, with the vocabulary the
# corpus and the questions use for each (domain-specific — see
# template/MANIFEST.yaml, re-state these for a new domain).
PROGRAMS: dict[str, re.Pattern[str]] = {
    "senior": re.compile(r"\bsenior|adulto mayor|nakatatanda|6[025]\+|age 6[025]\b", re.I),
    "disabled": re.compile(r"\bdisab|discapac|mobility pass|kapansanan\b", re.I),
    "medicare": re.compile(r"\bmedicare\b", re.I),
    "veteran": re.compile(r"\bveteran|dd[ -]?214|courtesy card\b", re.I),
    "youth/student": re.compile(
        r"\byouth|student|tk[- ]?12|k-12|aggie|ucsb|sbcc|estudiante\b", re.I
    ),
    "child free": re.compile(r"\bchild|toddler|infant|under \d+ inch|0-18 ride free\b", re.I),
    "regular": re.compile(r"\bregular|adult|basic|single ride|one-way\b", re.I),
}


def agencies() -> list[str]:
    return sorted(set(domain.get_profile().aliases.values()))


def _case_text(case: dict) -> str:
    parts = [
        case.get("question") or "",
        " ".join(case.get("turns") or []),
        " ".join(case.get("required_facts") or []),
        case.get("rationale") or "",
        case.get("boundary") or "",
    ]
    return " ".join(parts)


def case_agencies(case: dict) -> list[str]:
    scope = case.get("agency_scope")
    if scope:
        return [scope]
    return detect_agencies(_case_text(case))


def case_programs(case: dict) -> set[str]:
    text = _case_text(case)
    return {name for name, pat in PROGRAMS.items() if pat.search(text)}


def corpus_programs_by_agency() -> dict[str, set[str]]:
    """For each agency, the set of programs its corpus text actually mentions —
    the ground truth a blind spot is measured against."""
    text_by_agency: dict[str, list[str]] = {}
    for chunk in load_chunks():
        text_by_agency.setdefault(chunk.agency, []).append(f"{chunk.section} {chunk.text}")
    out: dict[str, set[str]] = {}
    for agency, texts in text_by_agency.items():
        blob = " ".join(texts)
        out[agency] = {name for name, pat in PROGRAMS.items() if pat.search(blob)}
    return out


def build_matrix() -> dict[tuple[str, str], int]:
    """(agency, program) -> number of eval cases touching it."""
    matrix: dict[tuple[str, str], int] = {}
    for suite in load_suites():
        for case in suite["cases"]:
            if case.get("draft"):
                continue
            for agency in case_agencies(case):
                for program in case_programs(case):
                    matrix[(agency, program)] = matrix.get((agency, program), 0) + 1
    return matrix


def blind_spots(
    matrix: dict[tuple[str, str], int], corpus: dict[str, set[str]]
) -> list[tuple[str, str]]:
    """Cells the corpus covers but no case tests."""
    spots = []
    for agency in agencies():
        for program in corpus.get(agency, set()):
            if matrix.get((agency, program), 0) == 0:
                spots.append((agency, program))
    return spots


def render_markdown() -> str:
    matrix = build_matrix()
    corpus = corpus_programs_by_agency()
    progs = list(PROGRAMS)
    lines = ["# Eval coverage map", "", "Generated by `make coverage` (`evals/coverage.py`)."]
    lines.append("")
    lines.append(
        "Cells: number of eval cases touching that agency x program. `-` means "
        "the corpus does not publish that program for the agency; **0** (bold) "
        "means the corpus covers it but no case tests it — a blind spot."
    )
    lines.append("")
    lines.append("| Agency | " + " | ".join(progs) + " |")
    lines.append("|---|" + "|".join(["---"] * len(progs)) + "|")
    for agency in agencies():
        cells = []
        for program in progs:
            n = matrix.get((agency, program), 0)
            if program not in corpus.get(agency, set()):
                cells.append("-")
            elif n == 0:
                cells.append("**0**")
            else:
                cells.append(str(n))
        lines.append(f"| {agency} | " + " | ".join(cells) + " |")
    spots = blind_spots(matrix, corpus)
    lines.append("")
    if spots:
        lines.append(f"## Blind spots ({len(spots)})")
        lines.append("")
        lines.append("Corpus-published provisions with no eval case touching them:")
        lines.append("")
        for agency, program in spots:
            lines.append(f"- **{agency}** / {program}")
    else:
        lines.append("## Blind spots")
        lines.append("")
        lines.append("None: every corpus-published program above is touched by at least one case.")
    lines.append("")
    return "\n".join(lines)


def main() -> int:
    write = "--write" in sys.argv
    md = render_markdown()
    print(md)
    if write:
        out = config.REPO_ROOT / "docs" / "eval-coverage.md"
        out.write_text(md + "\n", encoding="utf-8")
        print(f"\nwrote {out}")
    return 0


if __name__ == "__main__":
    sys.exit(main())