test_report.py

from __future__ import annotations

import json
from pathlib import Path

from constituent_reconciler.decisions import band_pairs
from constituent_reconciler.evaluate import CalibrationReport, EvalReport, evaluate
from constituent_reconciler.report import render_eval_markdown


def _eval_report() -> EvalReport:
    banded = band_pairs(
        [("a", "b", 0.99), ("x", "y", 0.85)],
        auto_threshold=0.97,
        review_threshold=0.80,
    )
    return evaluate(banded, [["a", "b"], ["x", "y"]], n_records=4)


def test_eval_markdown_shows_kappa_section_and_pass() -> None:
    calibration = CalibrationReport(n_labels=20, kappa=0.80, threshold=0.60, passed=True)
    markdown = render_eval_markdown(_eval_report(), dataset="demo", calibration=calibration)
    assert "## Calibration (LLM field judge)" in markdown
    assert "Cohen's kappa: **0.80** over 20 labels." in markdown
    assert "Kappa gate at 0.60: **PASS**" in markdown


def test_eval_markdown_shows_kappa_fail() -> None:
    calibration = CalibrationReport(n_labels=20, kappa=0.20, threshold=0.60, passed=False)
    markdown = render_eval_markdown(_eval_report(), dataset="demo", calibration=calibration)
    assert "Kappa gate at 0.60: **FAIL** (observed 0.20)." in markdown


def test_eval_markdown_without_labels_is_fail_closed() -> None:
    markdown = render_eval_markdown(_eval_report(), dataset="demo", calibration=None)
    assert "## Calibration (LLM field judge)" in markdown
    assert "fail-closed" in markdown
    assert "Kappa gate at 0.60: **FAIL** (no labels)." in markdown


def test_eval_markdown_renders_disaggregated_risk_classes() -> None:
    banded = band_pairs(
        [("a", "b", 0.85)],
        auto_threshold=0.97,
        review_threshold=0.80,
    )
    report = evaluate(
        banded,
        [["a", "b"]],
        n_records=2,
        segments={"transliterated name": [["a", "b"]]},
    )
    markdown = render_eval_markdown(report, dataset="bias-demo")

    assert "## Disaggregated error by documented risk class" in markdown
    assert "| transliterated name | 1 | 1 | 0 | 100.0% | 0 |" in markdown


def test_provenance_defaults_to_the_fixture_sentence() -> None:
    """With no provenance declared, the report still describes the committed fixtures."""
    markdown = render_eval_markdown(_eval_report(), dataset="demo")
    assert "seeded synthetic fixtures" in markdown
    assert "no real personal data in the fixtures" in markdown


def test_declared_provenance_replaces_the_synthetic_claim() -> None:
    """A real dataset must not produce a report asserting it is synthetic.

    The sentence was printed unconditionally, so a real-data eval generated a
    report stating there was no real personal data in it -- a false provenance
    claim, produced automatically, on a tool whose discipline is provenance.
    """
    declared = "Ground truth is NCVR ncid across two statewide snapshots; real records."
    markdown = render_eval_markdown(_eval_report(), dataset="ncvr", provenance=declared)
    assert declared in markdown
    assert "seeded synthetic fixtures" not in markdown
    assert "no real personal data" not in markdown


def test_blank_provenance_falls_back_rather_than_leaving_a_gap() -> None:
    markdown = render_eval_markdown(_eval_report(), dataset="demo", provenance="   ")
    assert "seeded synthetic fixtures" in markdown


def test_a_truth_note_is_not_promoted_to_provenance(tmp_path: Path) -> None:
    """`note` must not stand in for `provenance`, or every committed report changes.

    The first version of this fix read the truth file's `note`. Every fixture
    truth file has one describing how its ground truth was planted, so both
    committed reports were rewritten and each lost the "no real personal data in
    the fixtures" assurance. CI caught it. The two keys answer different
    questions: how truth was built, versus where the records came from.
    """
    truth = tmp_path / "ground_truth.json"
    truth.write_text(
        json.dumps({"note": "Planted clusters for the demo fixtures.", "clusters": []}),
        encoding="utf-8",
    )
    loaded = json.loads(truth.read_text(encoding="utf-8"))

    markdown = render_eval_markdown(
        _eval_report(), dataset="demo", provenance=loaded.get("provenance")
    )
    assert "seeded synthetic fixtures" in markdown
    assert "Planted clusters for the demo fixtures." not in markdown


def test_eval_markdown_names_the_generator_that_produced_it() -> None:
    """A report must not credit a command that was not run and would not reproduce it."""

    markdown = render_eval_markdown(
        _eval_report(), dataset="febrl4", generator="make eval-benchmark"
    )
    assert "Generated by `make eval-benchmark`" in markdown
    assert "Generated by `reconcile eval`" not in markdown


def test_eval_markdown_defaults_to_the_cli_generator() -> None:
    assert "Generated by `reconcile eval`" in render_eval_markdown(
        _eval_report(), dataset="intake-demo"
    )


def test_eval_markdown_reports_f1_alongside_precision_and_recall() -> None:
    markdown = render_eval_markdown(_eval_report(), dataset="intake-demo")
    assert "| F1, auto |" in markdown
    assert "| F1, auto+review coverage |" in markdown
    assert "is not a gate" in markdown


def test_calibration_section_says_not_applicable_when_no_field_judge_ran() -> None:
    """A kappa FAIL for a component that never executed is a false coverage claim."""

    markdown = render_eval_markdown(_eval_report(), dataset="febrl4", field_judge_ran=False)
    assert "Not applicable to this run" in markdown
    assert "**FAIL** (no labels)" not in markdown


def test_calibration_section_still_fails_closed_when_the_judge_was_in_the_path() -> None:
    assert "**FAIL** (no labels)" in render_eval_markdown(_eval_report(), dataset="intake-demo")