accessibility.yml

name: accessibility

# The test suite proves what a static checker can prove without a browser: contrast ratios,
# landmarks, heading order, table semantics, and that no meaning is carried by colour alone. That
# is most of it and it runs on every push in `make verify`.
#
# This job is the part that needs a real rendering engine. Lighthouse loads the pages in Chrome
# and scores them, and the bar is 100 on accessibility with no exceptions. A 99 here is a
# regression, not a rounding difference: every point below 100 is a specific element a specific
# person cannot use.
#
# The budgets in lighthouse-budget.json are the other half. Every resource type except the
# document itself is budgeted at zero, which is not an aspiration: the site genuinely has no
# scripts, no external stylesheets, no fonts, no images and no third-party requests, and the
# budget exists so that adding one is a build failure rather than a decision nobody noticed. A
# page about undisclosed information should not be quietly shipping a tracker.

on:
  push:
    branches: [main, master]
  pull_request:

permissions:
  contents: read

jobs:
  lighthouse:
    runs-on: ubuntu-latest
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"
      - uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
        with:
          node-version: "20"

      - name: Install
        run: python -m pip install -e .

      # Built from the committed capture and the committed national artifact, so this job needs
      # no API key and no network beyond npm. It audits the same bytes a visitor would get.
      - name: Build the site
        run: |
          python -m disclosed.cli grade --source data/sample.json --out /tmp/report.json
          python -m disclosed.cli site \
            --report /tmp/report.json --national data/national.json \
            --out site --generated "${GITHUB_SHA}"

      - name: Serve it
        run: |
          python -m http.server 8000 --directory site &
          for _ in $(seq 1 30); do
            curl -sf http://127.0.0.1:8000/ >/dev/null && break
            sleep 1
          done

      # One page of each kind. They share a shell and a stylesheet, so auditing all 616 would be
      # 616 audits of the same four templates.
      - name: Audit
        run: |
          # The URL goes FIRST. Lighthouse takes it as a positional argument, and with it trailing
          # behind the flags the CLI exited "Please provide a url" without auditing anything.
          npx --yes lighthouse@12 http://127.0.0.1:8000/ \
            --output=json --output-path=/tmp/home.json --quiet --chrome-flags="--headless" \
            --budget-path=lighthouse-budget.json \
            --only-categories=accessibility,best-practices,seo,performance
          for page in methodology national state/CA institution/110468; do
            npx --yes lighthouse@12 "http://127.0.0.1:8000/${page}/" \
              --output=json --output-path="/tmp/$(basename "$page").json" --quiet \
              --chrome-flags="--headless" --budget-path=lighthouse-budget.json \
              --only-categories=accessibility
          done

      - name: Require 100 on accessibility
        run: |
          # Named explicitly rather than globbed. `/tmp/*.json` swept up a file that was not a
          # lighthouse report and the scorer died on KeyError: 'categories', after all five real
          # pages had already scored 1. The glob was also the more dangerous shape: had lighthouse
          # written nothing, the loop body would never run and `exit "${fail}"` would exit 0, so a
          # gate that measured zero pages would have reported success.
          reports="/tmp/home.json /tmp/methodology.json /tmp/national.json /tmp/CA.json /tmp/110468.json"
          expected=5
          found=0
          fail=0
          for report in ${reports}; do
            if [ ! -f "${report}" ]; then
              echo "::error title=Missing report::${report} was never written; lighthouse did not audit that page."
              fail=1
              continue
            fi
            found=$((found + 1))
            score=$(python -c "
          import json,sys
          data = json.load(open(sys.argv[1]))
          category = data['categories'].get('accessibility')
          # A missing category is a failed audit, not a pass. Defaulting it to 1.0 would turn a
          # broken run into a green check, which is this project's own failure mode wearing a
          # CI badge.
          print('missing' if category is None or category.get('score') is None
                else category['score'])
          " "$report")
            echo "${report}: accessibility ${score}"
            [ "${score}" = "1" ] || fail=1
          done
          if [ "${found}" -ne "${expected}" ]; then
            echo "::error title=Audited ${found} of ${expected} pages::A pass over a partial set is not a pass."
            fail=1
          fi
          exit "${fail}"