accessibility.yml

name: accessibility

# The test suite proves what a static checker can prove without a browser: contrast ratios,
# landmarks, heading order, table semantics, and that no meaning is carried by colour alone. That
# is most of it and it runs on every push in `make verify`.
#
# This job is the part that needs a real rendering engine. Lighthouse loads the pages in Chrome
# and scores them, and the bar is 100 on accessibility with no exceptions. A 99 here is a
# regression, not a rounding difference: every point below 100 is a specific element a specific
# person cannot use.
#
# lighthouse-budget.json is NOT the other half, and was described here for weeks as though it
# were. `--budget-path` never makes Lighthouse exit non-zero, the scoring step below reads only
# the accessibility category, and lighthouse@12 (12.8.2) emits no budget audit at all: a run
# against a budget file with every line set to zero exited 0, scored accessibility 1, and its
# report contained no audit whose key mentions "budget". Five of the six audits here ask for
# --only-categories=accessibility, which does not even collect the resource summary.
#
# The resource-count budget the README actually argues for -- no scripts, no external
# stylesheets, no fonts, no images, no third-party requests, because a page about undisclosed
# information should not be quietly shipping a tracker -- is enforced statically in
# `make verify`, by tests/test_accessibility.py. The flag is left below because the budget file
# is still the readable statement of intent and lighthouse may enforce it again one day; it is
# not counted as a gate anywhere.
#
# That paragraph used to say the budget was enforced "over every one of the 616 generated
# pages" (the site's total at the time), and it was not. TestTheResourceBudget runs over a
# six-page fixture: one page of each
# kind, from a two-institution report with no implausible finding in it, so the markup the home
# page and the institution pages render around a finding was never parsed by anything. A
# tracker added to that branch shipped past the suite. TestTheResourceBudgetOverThePublishedSite
# now renders the committed report and the committed national artifact -- the same bytes
# pages.yml uploads, all 617 pages -- and parses every one, and that test checks the count above
# against the build rather than leaving it as a number in a comment.

on:
  push:
    branches: [main, master]
  pull_request:

permissions:
  contents: read

jobs:
  lighthouse:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "20"

      - name: Install
        run: python -m pip install -e .

      # Built from the committed capture and the committed national artifact, so this job needs
      # no API key and no network beyond npm. It audits the same bytes a visitor would get.
      - name: Build the site
        run: |
          python -m disclosed.cli grade --source data/sample.json --out /tmp/report.json
          python -m disclosed.cli site \
            --report /tmp/report.json --national data/national.json \
            --scorecard-census data/scorecard-census.json \
            --out site --generated "${GITHUB_SHA}"

      - name: Serve it
        run: |
          python -m http.server 8000 --directory site &
          for _ in $(seq 1 30); do
            curl -sf http://127.0.0.1:8000/ >/dev/null && break
            sleep 1
          done

      # One page of each kind. They share a shell and a stylesheet, so auditing all 617 would be
      # 617 audits of the same five templates.
      - name: Audit
        run: |
          # The URL goes FIRST. Lighthouse takes it as a positional argument, and with it trailing
          # behind the flags the CLI exited "Please provide a url" without auditing anything.
          npx --yes lighthouse@12 http://127.0.0.1:8000/ \
            --output=json --output-path=/tmp/home.json --quiet --chrome-flags="--headless" \
            --budget-path=lighthouse-budget.json \
            --only-categories=accessibility,best-practices,seo,performance
          for page in methodology national census state/CA institution/110468; do
            npx --yes lighthouse@12 "http://127.0.0.1:8000/${page}/" \
              --output=json --output-path="/tmp/$(basename "$page").json" --quiet \
              --chrome-flags="--headless" --budget-path=lighthouse-budget.json \
              --only-categories=accessibility
          done

      - name: Require 100 on accessibility
        run: |
          # Named explicitly rather than globbed. `/tmp/*.json` swept up a file that was not a
          # lighthouse report and the scorer died on KeyError: 'categories', after all five real
          # pages had already scored 1. The glob was also the more dangerous shape: had lighthouse
          # written nothing, the loop body would never run and `exit "${fail}"` would exit 0, so a
          # gate that measured zero pages would have reported success.
          reports="/tmp/home.json /tmp/methodology.json /tmp/national.json /tmp/census.json /tmp/CA.json /tmp/110468.json"
          expected=6
          found=0
          fail=0
          for report in ${reports}; do
            if [ ! -f "${report}" ]; then
              echo "::error title=Missing report::${report} was never written; lighthouse did not audit that page."
              fail=1
              continue
            fi
            found=$((found + 1))
            score=$(python -c "
          import json,sys
          data = json.load(open(sys.argv[1]))
          category = data['categories'].get('accessibility')
          # A missing category is a failed audit, not a pass. Defaulting it to 1.0 would turn a
          # broken run into a green check, which is this project's own failure mode wearing a
          # CI badge.
          print('missing' if category is None or category.get('score') is None
                else category['score'])
          " "$report")
            echo "${report}: accessibility ${score}"
            [ "${score}" = "1" ] || fail=1
          done
          if [ "${found}" -ne "${expected}" ]; then
            echo "::error title=Audited ${found} of ${expected} pages::A pass over a partial set is not a pass."
            fail=1
          fi
          exit "${fail}"