tests.yml

# The repository's own gate. It runs on every push to main and every pull
# request, and it is the same suite a contributor runs locally:
#
#   PYTHONPATH=src:tests python3 -m unittest discover -s tests
#
# Nothing here needs a key, a network, or a third-party package: the tests
# bind a local HTTP server on the loopback interface for the adapter and
# model-judge paths, and everything else is standard library.
#
# Every step below has to be able to fail. A step that cannot go red is a
# badge, and this repository exists to argue against those.

name: tests

on:
  push:
    branches: [main]
  pull_request:

permissions:
  contents: read

jobs:
  tests:
    runs-on: ubuntu-latest
    strategy:
      fail-fast: false
      matrix:
        python-version: ["3.11", "3.12", "3.13", "3.14"]
    steps:
      - uses: actions/checkout@11d5960a326750d5838078e36cf38b85af677262 # v4.4.0
        with:
          persist-credentials: false

      - uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
        with:
          python-version: ${{ matrix.python-version }}

      - name: Run the suite
        run: PYTHONPATH=src:tests python3 -m unittest discover -s tests

      # The audit of the bundled demo is committed. If a change to the harness
      # moves a score, a hash, or a byte of the report, this step fails — which
      # is the reproducibility claim, enforced.
      #
      # `git status --porcelain` and not `git diff --exit-code` alone: the run
      # id is the output directory's name, so anything that moves the run id
      # writes a *new, untracked* directory and leaves the committed one
      # untouched. A diff sees nothing and reports success, which is this
      # repository's own favourite failure — a check that passed because it did
      # not run. Untracked output is a failure here.
      - name: The committed demo audit must still reproduce byte for byte
        run: |
          PYTHONPATH=src python3 -m plumbline gate \
            --config examples/riverbend.toml --out audits
          git diff -- audits baselines
          dirty=$(git status --porcelain -- audits baselines)
          if [ -n "$dirty" ]; then
            echo "the committed audit is not what this code produces:" >&2
            echo "$dirty" >&2
            exit 1
          fi

      # The committed report must still match its own seal.
      - name: The committed report has not been edited since it was produced
        run: PYTHONPATH=src python3 -m plumbline verify audits/*/report.json

      # The tamper drill, as documented in the README. Exit codes are captured
      # explicitly rather than leaned on through `&&`: a drill that asserts
      # only "did not exit 0" would be satisfied by the harness crashing, which
      # is the opposite of what it is trying to prove.
      - name: Tamper drill — integrity refusal, then a caught fabrication
        run: |
          set -u
          run_gate() {
            set +e
            PYTHONPATH=src python3 -m plumbline gate \
              --config examples/riverbend.toml --out /tmp/tamper
            code=$?
            set -e
            echo "$code"
          }
          python3 - <<'PY'
          import pathlib
          p = pathlib.Path("datasets/riverbend-demo/responses.jsonl")
          p.write_text(p.read_text().replace("850 dollars", "900 dollars"))
          PY

          code=$(run_gate | tail -1)
          if [ "$code" != "3" ]; then
            echo "expected exit 3 (integrity refusal), got $code" >&2
            exit 1
          fi

          PYTHONPATH=src python3 -m plumbline seal datasets/riverbend-demo
          code=$(run_gate | tail -1)
          if [ "$code" != "1" ]; then
            echo "expected exit 1 (the fabrication is caught and scored), got $code" >&2
            exit 1
          fi
          git checkout -- datasets/riverbend-demo