perimeter/tests/test_a11y_gate.py at main · ghzhost/perimeter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
"""The WCAG gate has to be able to fail, so this runs it against inputs that should.

``tools/a11y.mjs`` is the check behind README.md's claim that the built pages are held to
axe-core's WCAG A and AA rule sets in CI. Until 2026-08-18 it was the only gate in this
repository with no failure evidence of any kind, while ``tools/determinism.sh`` had
thirteen cases. ADR 0004 says a check is not adopted here until it has been seen to fail
on the input it exists to catch, so these are that evidence.

The vacuous pass this pins down is the one the gate had. axe returns four buckets, and a
rule it ran but could not decide lands in ``incomplete``, not in ``violations``. The gate
read ``violations`` only. Measured against a page carrying a focusable link inside
``aria-hidden="true"``, a real 4.1.2 problem, it printed ``ok`` and exited 0, because
axe had filed the rule as undecided. "Could not check" was rendering as "clean", in the
gate belonging to the repository whose whole subject is not doing that.

So: an undecided rule now fails unless it is declared in ``UNDECIDABLE_HERE`` with a
reason and with where it is checked instead, and what was not decided is printed on every
run, passing or failing.
"""

from __future__ import annotations

import os
import shutil
import subprocess
from pathlib import Path

import pytest

ROOT = Path(__file__).resolve().parents[1]
GATE = ROOT / "tools" / "a11y.mjs"

CLEAN = 0
FAILED = 1
UNUSABLE = 2

# The gate needs node and the two packages `make node-sync` installs. `make verify` runs
# node-sync before test for exactly this reason. If the toolchain is genuinely absent
# these tests skip locally and say so, but never on CI, where a silently skipped gate
# test is the same lie the gate itself used to tell.
NODE = shutil.which("node")
HAVE_TOOLCHAIN = (
    NODE is not None
    and (ROOT / "node_modules" / "axe-core").is_dir()
    and (ROOT / "node_modules" / "jsdom").is_dir()
)

if not HAVE_TOOLCHAIN and os.environ.get("CI"):  # pragma: no cover - CI-only guard
    raise RuntimeError(
        "node, axe-core or jsdom is missing and CI is set: the WCAG gate's own tests "
        "cannot run, and a skipped gate test must not read as a passing one. Run "
        "`make node-sync` before `make test`."
    )

pytestmark = pytest.mark.skipif(
    not HAVE_TOOLCHAIN,
    reason="node with axe-core and jsdom is required; run `make node-sync`",
)


def run(*args: Path | str) -> subprocess.CompletedProcess[str]:
    # Fixed argv, absolute interpreter and script, no shell: nothing here is interpolated
    # from anything but the test's own tmp_path.
    return subprocess.run(  # noqa: S603
        [NODE or "node", str(GATE), *map(str, args)],
        capture_output=True,
        text=True,
        check=False,
        cwd=ROOT,
    )


def page(body: str, *, lang: str = ' lang="en"') -> str:
    """A minimal conformant page, with the body under test dropped into its main."""
    return (
        f"<!doctype html><html{lang}><head><meta charset='utf-8'>"
        "<title>Gate test page</title></head><body>"
        "<nav aria-label='test'><a href='#content'>Skip to content</a></nav>"
        f"<main id='content'><h1>Gate test page</h1>{body}</main>"
        "<footer><p>Test material generated by tests/test_a11y_gate.py.</p></footer>"
        "</body></html>"
    )


def build(root: Path, pages: dict[str, str]) -> Path:
    root.mkdir(parents=True, exist_ok=True)
    for name, html in pages.items():
        (root / name).write_text(html, encoding="utf-8")
    return root


# --------------------------------------------------------------------------------------
# Inputs the gate cannot examine. ADR 0004: those are failures, not clean runs.
# --------------------------------------------------------------------------------------


def test_the_gate_exists() -> None:
    assert GATE.is_file()


def test_no_directory_argument_is_unusable() -> None:
    result = run()
    assert result.returncode == UNUSABLE, result.stderr
    assert "usage:" in result.stderr


def test_a_missing_directory_is_unusable(tmp_path: Path) -> None:
    result = run(tmp_path / "was-never-built")
    assert result.returncode == UNUSABLE, result.stderr
    assert "cannot read" in result.stderr


def test_a_directory_holding_no_pages_is_unusable(tmp_path: Path) -> None:
    """Nothing to check is not the same as nothing wrong."""
    result = run(build(tmp_path / "empty", {}))
    assert result.returncode == UNUSABLE, result.stderr
    assert "no .html files" in result.stderr


# --------------------------------------------------------------------------------------
# Violations. The half of the job the gate always did.
# --------------------------------------------------------------------------------------


def test_an_image_with_no_alt_text_fails(tmp_path: Path) -> None:
    root = build(tmp_path / "img", {"page.html": page("<img src='x.png'>")})
    result = run(root)
    assert result.returncode == FAILED, result.stdout
    assert "image-alt" in result.stderr
    assert "1 accessibility violation(s)" in result.stderr


def test_a_page_with_no_language_fails(tmp_path: Path) -> None:
    root = build(tmp_path / "lang", {"page.html": page("<p>Text.</p>", lang="")})
    result = run(root)
    assert result.returncode == FAILED, result.stdout
    assert "html-has-lang" in result.stderr


def test_one_bad_page_fails_the_whole_run(tmp_path: Path) -> None:
    root = build(
        tmp_path / "mixed",
        {"good.html": page("<p>Text.</p>"), "bad.html": page("<img src='x.png'>")},
    )
    result = run(root)
    assert result.returncode == FAILED, result.stdout
    assert "FAIL bad.html" in result.stderr
    assert "ok   good.html" in result.stdout


# --------------------------------------------------------------------------------------
# Undecided rules. The half the gate used to report as a pass.
# --------------------------------------------------------------------------------------


def test_a_focusable_link_inside_aria_hidden_fails(tmp_path: Path) -> None:
    """The measured regression case.

    axe cannot decide `aria-hidden-focus` without layout, so it files it as incomplete.
    The gate reported this page as `ok` and exited 0 until 2026-08-18.
    """
    root = build(
        tmp_path / "hidden",
        {"page.html": page("<div aria-hidden='true'><a href='#x'>Link</a></div>")},
    )
    result = run(root)
    assert result.returncode == FAILED, result.stdout
    assert "aria-hidden-focus" in result.stderr
    assert "[undecided]" in result.stderr
    assert "1 undeclared undecided rule(s)" in result.stderr
    assert "ok   page.html" not in result.stdout


def test_an_untested_frame_fails(tmp_path: Path) -> None:
    """A frame axe could not reach into is an unchecked region, not a checked one."""
    root = build(
        tmp_path / "frame",
        {"page.html": page("<iframe title='embedded' src='about:blank'></iframe>")},
    )
    result = run(root)
    assert result.returncode == FAILED, result.stdout
    assert "frame-tested" in result.stderr
    assert "1 undeclared undecided rule(s)" in result.stderr


def test_the_failure_message_says_what_to_do_about_an_undecided_rule(
    tmp_path: Path,
) -> None:
    root = build(
        tmp_path / "hidden2",
        {"page.html": page("<div aria-hidden='true'><a href='#x'>Link</a></div>")},
    )
    result = run(root)
    assert "could not decide it. That is not a pass" in result.stderr
    assert "UNDECIDABLE_HERE" in result.stderr


# --------------------------------------------------------------------------------------
# A clean run. What it must say, not only that it exits 0.
# --------------------------------------------------------------------------------------


def test_a_clean_page_passes(tmp_path: Path) -> None:
    root = build(tmp_path / "clean", {"page.html": page("<p>Text.</p>")})
    result = run(root)
    assert result.returncode == CLEAN, result.stderr
    assert "ok   page.html" in result.stdout


def test_a_clean_run_names_every_rule_it_could_not_decide(tmp_path: Path) -> None:
    """The disclosure this gate exists to make.

    A reader of CI output is told what was checked and what was not, in the same place.
    A number of pages "clean against 6 rule sets", with three rules per page quietly
    undecided, is the shape this repository refuses everywhere else.
    """
    root = build(tmp_path / "clean2", {"page.html": page("<p>Text.</p>")})
    result = run(root)
    assert result.returncode == CLEAN, result.stderr
    assert "undecided (" in result.stdout
    assert "not decided here, declared in tools/a11y.mjs:" in result.stderr
    for rule in ("color-contrast", "landmark-one-main", "page-has-heading-one"):
        assert rule in result.stderr, rule
    assert "covered by:" in result.stderr


def test_the_disclosure_also_prints_on_a_failing_run(tmp_path: Path) -> None:
    root = build(tmp_path / "clean3", {"page.html": page("<img src='x.png'>")})
    result = run(root)
    assert result.returncode == FAILED
    assert "not decided here, declared in tools/a11y.mjs:" in result.stderr


def test_a_declared_rule_that_never_fires_is_marked_as_such(tmp_path: Path) -> None:
    """`target-size` is declared, and axe reports it on none of these pages.

    README.md called it one of two rules "suppressed for want of a renderer". Measured,
    it lands in neither `violations` nor `incomplete`: it is not suppressed, it simply
    never fires, which is a different and weaker statement about what was checked.
    """
    root = build(tmp_path / "clean4", {"page.html": page("<p>Text.</p>")})
    result = run(root)
    assert result.returncode == CLEAN, result.stderr
    assert "target-size  [not reported by axe on these pages]" in result.stderr