ci.yml

# Cairn CI.
#
# Four jobs, in deliberate order of dependency:
#
#   core       install, lint, test. Standard library only, no network needed,
#              and no auditor: this is the path a contributor runs, and it
#              must not depend on anything outside this repository.
#   interface  the accessibility behaviours that need a real browser.
#   audit      the merge gate. Resolves the pinned auditor, grades the
#              recorded evidence, and then holds the result against the
#              committed baseline — because a floor is a minimum and a score
#              can decay a long way above one without breaching it.
#   live       the same questions, asked over HTTP against a running server,
#              and compared to the evidence the gate graded. Separate from
#              `audit` on purpose: the gate must stay offline and
#              deterministic, and nothing in this job can make it pass.
#
# `audit` is the job that must be marked required in branch protection, and it
# is NOT. Until somebody with admin rights applies the ruleset committed at
# .github/rulesets/main.json, this job is advisory: it runs, it writes a
# verdict, and nothing stops a merge while that verdict is red. It is not
# enough for a gate to exist. A gate nobody has made blocking is a report, and
# this comment is here so nobody reads the job name and assumes otherwise.
#
# ---------------------------------------------------------------------------
# Why this gate fails instead of skipping
# ---------------------------------------------------------------------------
# The obvious way to write an audit job is to skip it when the harness cannot
# be fetched — the network is flaky, the run is on a fork, someone is offline.
# Every one of those is a real inconvenience and none of them is a reason to
# report green.
#
# A skipped gate and a passed gate look identical on a pull request: one green
# check. The difference only exists in a log nobody opens. So the moment the
# harness is unreachable is exactly the moment the gate has told you nothing,
# and the honest report of "told you nothing" is a red check.
#
# So: no `continue-on-error`, no `if:` that lets this job be skipped, and
# never `PLUMBLINE_SRC`. If the harness cannot be resolved,
# ./plumbline-gate.sh exits 4 and this job fails. A gate that could not run is
# not a gate that passed.
#
# ---------------------------------------------------------------------------
# Why every gate invocation says `env -u PLUMBLINE_SRC`
# ---------------------------------------------------------------------------
# The runner honours `PLUMBLINE_SRC` as a develop-the-harness escape hatch: set
# it and resolution is skipped entirely, the pin is not consulted, and the run
# is graded by whatever is in that directory. It warns loudly on stderr and
# then does it anyway — which is the right call for the harness's own
# developers and the wrong one for a merge gate, because a warning in a log
# nobody opens next to a green check is a green check.
#
# The runner is vendored from Plumbline byte for byte (a step below proves it),
# so the fix does not belong in the file: patching it here would fork the one
# implementation of resolution this repository has, which is the thing the
# vendoring exists to prevent. It belongs at the call site. `env -u` removes
# the variable from the gate's environment whatever put it there — a workflow
# edit, a repository-level variable, a self-hosted runner's profile — so the
# only harness this job can be graded by is the pinned one.
# ---------------------------------------------------------------------------

name: ci

on:
  push:
    branches: [main]
  pull_request:

permissions:
  contents: read

jobs:
  core:
    name: core (install, lint, test — auditor unreachable)
    runs-on: ubuntu-latest
    strategy:
      matrix:
        python: ["3.11", "3.12"]
    steps:
      # Nothing in this workflow pushes, so no job has any use for a
      # credential left behind in .git/config where every later step and
      # every uploaded artifact can reach it.
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: ${{ matrix.python }}

      - name: Install
        run: pip install -e ".[dev]"

      - name: Lint
        run: ruff check .

      - name: Test
        run: python3 -m unittest discover -s tests

      - name: The demo path runs with no install at all
        run: |
          python3 -m cairn index
          python3 -m cairn ask "How much is the monthly grocery allowance for one person?"
          python3 -m cairn ask --explain "What vaccinations does my dog need?"

      - name: Nothing above resolved the auditor
        run: |
          if [ -d .plumbline-cache ]; then
            echo "the core path fetched the auditor; it must not need it" >&2
            exit 1
          fi
          if grep -rn "plumbline" --include="*.py" cairn/ | grep -v "plumbline-bundle\|plumbline-checksums\|plumbline-contrast\|plumbline.pin\|plumbline/questions.toml\|plumbline/bundle"; then
            echo "the engine imports the auditor; it must not" >&2
            exit 1
          fi

      - name: Fail-closed drill — the gate fails when the harness cannot be resolved
        run: |
          # Same runner, same pin format, a repository that cannot be fetched.
          # The gate must exit 4 rather than skipping or reporting success. If
          # this step ever passes silently, the audit job below is decorative.
          sed 's#^repo = .*#repo = file:///nonexistent/harness.git#' plumbline.pin > /tmp/unreachable.pin
          set +e
          env -u PLUMBLINE_SRC PLUMBLINE_PIN_FILE=/tmp/unreachable.pin PLUMBLINE_CACHE_DIR=/tmp/empty-cache ./plumbline-gate.sh
          code=$?
          set -e
          echo "gate exited $code with an unreachable harness"
          test "$code" -eq 4

  interface:
    name: interface (accessibility behaviours in Chromium)
    runs-on: ubuntu-latest
    steps:
      # Nothing in this workflow pushes, so no job has any use for a
      # credential left behind in .git/config where every later step and
      # every uploaded artifact can reach it.
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"
      - uses: actions/setup-node@v4
        with:
          node-version: "20"

      # `npm ci`, never `npm install`. The lock file is committed and the
      # manifest names exact versions; `ci` installs precisely that and exits
      # non-zero if the two disagree, where `install` would quietly resolve
      # something newer and rewrite the lock. The rule set grading this page
      # is axe-core, and a floating rule set means "62/62 passed" is a
      # statement about whatever npm picked this morning. a11y.mjs then checks
      # the version that actually reached the browser, because an install can
      # be right and a stale node_modules still wrong.
      - name: Install browser checks
        working-directory: tests/browser
        run: |
          npm ci
          npx playwright install --with-deps chromium

      - name: Index
        run: python3 -m cairn index

      - name: Run the behaviour checks
        working-directory: tests/browser
        run: npm run check

  audit:
    name: audit (merge gate — must be required in branch protection)
    runs-on: ubuntu-latest
    needs: [core]
    steps:
      # Nothing in this workflow pushes, so no job has any use for a
      # credential left behind in .git/config where every later step and
      # every uploaded artifact can reach it.
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      # Keyed on the pin file, so bumping the pin fetches the new harness and
      # every other run reuses the old one. The key is the pin, never a
      # branch name: the cache must not be able to serve a different commit
      # than the one this repository asked for.
      - name: Cache the pinned harness
        uses: actions/cache@v4
        with:
          path: .plumbline-cache
          key: plumbline-${{ hashFiles('plumbline.pin') }}

      - name: Re-record the evidence from the current engine
        run: |
          python3 -m cairn index
          python3 -m cairn record

      - name: The committed evidence matches what the engine produces now
        run: |
          # `cairn record` is deterministic. If this diff is non-empty the
          # engine's behaviour changed without the bundle being updated, and
          # the audit below would be grading yesterday's answers.
          git diff --exit-code -- plumbline/bundle

      - name: Run the pinned audit
        run: env -u PLUMBLINE_SRC ./plumbline-gate.sh --summary-file "$GITHUB_STEP_SUMMARY"

      # The reason the bypass above is fixed at the call site rather than in
      # the runner. If this diff is ever non-empty, "vendored verbatim" — said
      # in DESIGN.md, in plumbline-live.sh, and in the comment at the top of
      # this file — has stopped being true, and the argument for not patching
      # the runner locally has stopped holding with it. It runs here because
      # this is the job that has a resolved harness to compare against; the
      # step above is what put it there.
      - name: The runner is the one the pinned harness ships, byte for byte
        run: |
          ref=$(sed -n 's/^[[:space:]]*ref[[:space:]]*=[[:space:]]*\([0-9a-f]\{40\}\).*/\1/p' plumbline.pin | head -n 1)
          test -n "$ref"
          diff -u ".plumbline-cache/$ref/gate/plumbline-gate.sh" plumbline-gate.sh

      # The gate checks floors. This checks what a floor cannot: a score that
      # moved without breaching one — in either direction, because an
      # improvement nobody adopts leaves the recorded bar below what the
      # system does, and the whole gap becomes decay nothing will notice — and
      # a suite switched off without saying so. It reads the report the step
      # above just wrote, so it runs after it and never instead of it. No
      # `continue-on-error`, for the same reason the gate has none.
      # The one test in the suite that needs a resolved harness, run in the
      # one job that has one. It holds plumbline/target.toml's floors and its
      # suite list against the defaults parsed out of the pinned harness's own
      # source, and it skips itself when there is no checkout to read — which,
      # in `core`, is always, because `core` fails if the cache exists at all.
      # So it ran on a laptop where somebody had happened to run the gate, and
      # nowhere else, while its docstring said it ran here.
      - name: The floors and the suite list, against the harness that defines them
        run: python3 -m unittest tests.test_audit_guard -v

      - name: The committed baseline still matches, and no gap is undeclared
        run: python3 audit_guard.py --summary-file "$GITHUB_STEP_SUMMARY"

      - name: Keep the report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: audit-report
          path: plumbline/audits/
          if-no-files-found: warn

  # -------------------------------------------------------------------------
  # The gate grades a recording. This grades the thing the recording is of.
  #
  # A bundle is bytes on disk; the server is code that changes. They agreed on
  # the day `cairn record` wrote the bundle, and every audit report this
  # repository publishes is a statement about the recording. Whether it is
  # also a statement about the interface a person meets was an assumption
  # until this job existed.
  #
  # It is deliberately NOT the gate, and the separation is structural rather
  # than a convention:
  #
  #   - `plumbline.pin` names plumbline/target.toml. Nothing in the gate's
  #     path reads plumbline/live.toml, so the gate cannot acquire a socket by
  #     configuration.
  #   - ./plumbline-live.sh cannot resolve the harness. It requires a checkout
  #     the gate already fetched and verified, so a run against a live server
  #     is never the act that installs the thing grading it.
  #   - the `audit` job does not need this job and does not call this script;
  #     tests/test_live.py fails if it ever does.
  #
  # The drift check itself does not need this job either — tests/test_live.py
  # posts every committed question to a real loopback server in the core path,
  # offline, with no harness. What this job adds is the harness in the loop:
  # it is the harness's own HTTP recorder that seals the evidence, so what is
  # graded is what the harness saw, not what Cairn says the harness would have
  # seen.
  # -------------------------------------------------------------------------
  live:
    name: live (the served interface, graded by the pinned harness)
    runs-on: ubuntu-latest
    needs: [core]
    steps:
      # Nothing in this workflow pushes, so no job has any use for a
      # credential left behind in .git/config where every later step and
      # every uploaded artifact can reach it.
      - uses: actions/checkout@v4
        with:
          persist-credentials: false
      - uses: actions/setup-python@v5
        with:
          python-version: "3.12"

      - name: Cache the pinned harness
        uses: actions/cache@v4
        with:
          path: .plumbline-cache
          key: plumbline-${{ hashFiles('plumbline.pin') }}

      # ./plumbline-gate.sh is the only thing in this repository that fetches
      # the harness, and running it here is not a duplicate of the audit job's
      # work by accident — it is the order the comparison needs. Asking
      # whether the server matches the graded evidence is only a question
      # worth answering while that evidence is passing.
      - name: Resolve the harness, by running the gate that owns resolution
        run: env -u PLUMBLINE_SRC ./plumbline-gate.sh

      - name: Grade the running server and compare it to the recording
        run: ./plumbline-live.sh --summary-file "$GITHUB_STEP_SUMMARY"

      - name: Keep the live report
        if: always()
        uses: actions/upload-artifact@v4
        with:
          name: live-report
          path: .plumbline-live/audits/
          if-no-files-found: warn