forked from ChelseaKR/sprout
-
Notifications
You must be signed in to change notification settings - Fork 0
311 lines (294 loc) · 13.8 KB
/
Copy pathci.yml
File metadata and controls
311 lines (294 loc) · 13.8 KB
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
254
255
256
257
258
259
260
261
262
263
264
265
266
267
268
269
270
271
272
273
274
275
276
277
278
279
280
281
282
283
284
285
286
287
288
289
290
291
292
293
294
295
296
297
298
299
300
301
302
303
304
305
306
307
308
309
310
311
name: ci
on:
push:
branches: [main] # feature-branch validation comes from the pull_request run only (CI-CD-STANDARD §11i)
pull_request:
# Least-privilege by default; jobs escalate only what they need.
permissions:
contents: read
concurrency:
group: ci-${{ github.ref }}
cancel-in-progress: ${{ github.event_name == 'pull_request' }}
jobs:
test:
runs-on: ubuntu-latest
strategy:
fail-fast: false
matrix:
python: ${{ github.event_name == 'pull_request' && fromJSON('["3.12"]') || fromJSON('["3.12","3.13"]') }}
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev + serve) on ${{ matrix.python }}
run: uv sync --locked --extra serve --python ${{ matrix.python }}
- name: Ruff format check
run: uv run ruff format --check src tests
- name: Ruff lint
run: uv run ruff check src tests
- name: Mypy (strict)
run: uv run mypy
- name: Pytest (branch coverage gate >=90% from pyproject)
run: uv run pytest
- name: Build sdist + wheel (packaging regression check — CQ-10)
run: uv build
security:
runs-on: ubuntu-latest
permissions:
contents: read
# gitleaks-action resolves a pull_request's commit range through
# `GET /repos/{owner}/{repo}/pulls/{n}/commits`, which 403s on a private repo
# without this scope. Least privilege: read on contents + pull requests, nothing
# else — no write anywhere.
pull-requests: read
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
# The PR-event scan is `git log <first-commit>^..<last-commit>`; those commits
# are not in a depth-1 checkout of the merge ref, so gitleaks needs real history.
fetch-depth: 0
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev)
run: uv sync --locked
- name: Semgrep (SAST)
run: uvx --with 'setuptools<81' semgrep scan --config p/python --error src
- name: pip-audit (blocking — not muted)
run: uv run pip-audit
# Runs on pull_request too, as of 2026-08-01. It used to carry
# `if: github.event_name != 'pull_request'`, which meant a secret introduced in a
# PR was not caught until it had already been merged to main — the PR's own
# "security" check went green having scanned nothing for secrets. Blocking, no
# `|| true`, no continue-on-error.
- name: gitleaks (secret scan)
uses: gitleaks/gitleaks-action@e0c47f4f8be36e29cdc102c57e68cb5cbf0e8d1e # v3
env:
GITHUB_TOKEN: ${{ secrets.GITHUB_TOKEN }}
# On a PR finding the action would otherwise try to comment, which needs write
# scope. Turn the comment off rather than grant it — the failing step is the
# signal. (The same stance the author's other repos take.)
GITLEAKS_ENABLE_COMMENTS: "false"
eval-a11y:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev + serve)
run: uv sync --locked --extra serve
- name: Ingest the bundled corpus
run: uv run sprout ingest
- name: Eval suites (offline deterministic judge, merge-blocking)
run: uv run sprout eval --out docs/audits
- name: Structural a11y gate (chat UI + HTML report)
run: |
uv run sprout a11y-check web/dist/index.html
uv run sprout a11y-check docs/audits/eval-report.html
- name: Claims-integrity gate (docs vs code/config source of truth)
run: uv run sprout claims-check
- name: Judge calibration (gated — deterministic judge, 66 probes; see docs/ROADMAP.md)
run: uv run sprout calibrate eval/judge_probes.yaml --out docs/audits --gate
- name: Gate inventory (FIX-02 — every ledger AUTO row must map to a real mechanism)
run: uv run sprout gate-inventory --out docs/audits
- name: Tier-A SLO + burn-rate-alert schema gate
run: uv run sprout slo-check
- name: Corpus report (report-only — EXP-12 heuristics advisory until tuned)
run: uv run sprout corpus-report --out docs/audits
- name: Corpus proposal review (E5 — every submitted proposal, merge-blocking on errors)
run: uv run sprout propose check
smoke:
# Phase 1: a dedicated CI smoke suite of corpus-derived questions, distinct from the
# hand-authored Phase 2 eval harness (job `eval-a11y` above). Every case here is
# templated mechanically from the ingested corpus's own species/topic taxonomy, so
# coverage tracks the corpus automatically — see src/sprout/smoke.py and
# docs/ROADMAP.md Phase 1.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev + serve)
run: uv sync --locked --extra serve
- name: Ingest the bundled corpus
run: uv run sprout ingest
- name: Corpus-derived smoke suite (merge-blocking)
run: uv run sprout smoke --out docs/audits
tuning-scope:
# ROADMAP Phase 3: tune retrieval/prompts only against committed eval failures, never a
# held-out set. Mechanical enforcement — see AIEV-30 / docs/ROADMAP.md.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
fetch-depth: 0 # full history + all remote branch refs; the gate diffs against origin/<base>
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev)
run: uv sync --locked
- name: Tuning-scope gate
env:
BASE_REF: ${{ github.event.pull_request.base.ref || 'main' }}
run: uv run sprout check-tuning-scope --base "origin/${BASE_REF}"
web-static:
# EXP-08 (docs/ideation/03-expansions.md): the TypeScript port's cross-language
# conformance test — the deliverable's spine. Regenerates the fixtures from the
# *current* Python pipeline on every run, so drift fails here, not later.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "20"
cache: npm
cache-dependency-path: web-static/package-lock.json
- name: Install (dev + serve)
run: uv sync --locked --extra serve
- name: Ingest the bundled corpus
run: uv run sprout ingest
- name: Export the static bundle (index.json + config.json)
run: uv run python scripts/export_web_bundle.py
- name: Generate cross-language conformance fixtures from the Python pipeline
run: uv run python scripts/generate_conformance_fixtures.py
- name: Install web-static deps
run: cd web-static && npm ci
- name: Typecheck
run: cd web-static && npm run typecheck
- name: Conformance test (142 eval-suite cases vs. the Python pipeline)
run: cd web-static && npm test
- name: Build the deployable static site
run: cd web-static && npm run build:site
- name: Structural accessibility check on the published surface
run: uv run sprout a11y-check web-static/public/index.html
pa11y:
# Browser cross-check (axe + htmlcs runners, see web/.pa11yci.json) against the live-rendered
# chat UI. Merge-blocking: was advisory-only (continue-on-error + `|| true`); promoted per
# docs/ROADMAP.md gap tracking once the one real finding it caught (empty reminders table
# tripping axe's th-has-data-cells) was fixed rather than papered over.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "20"
- name: Install + ingest
run: uv sync --locked --extra serve && uv run sprout ingest
- name: Serve and pa11y the UI (axe + htmlcs runners, merge-blocking)
run: |
uv run sprout serve &
sleep 5
npx --yes pa11y-ci --config web/.pa11yci.json
lighthouse:
# Lighthouse's accessibility category, scoped with --only-categories so this stays a
# focused a11y gate (not a performance/SEO budget). Covers the chat UI and the committed
# HTML eval report, per the WCAG 2.2 AA conformance row in docs/ROADMAP.md. Was previously
# not wired into CI at all.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- uses: actions/setup-node@820762786026740c76f36085b0efc47a31fe5020 # v7.0.0
with:
node-version: "20"
- name: Install + ingest
run: uv sync --locked --extra serve && uv run sprout ingest
- name: Regenerate the HTML eval report
run: uv run sprout eval --out docs/audits
- name: Serve + Lighthouse accessibility audit (chat UI + eval report, merge-blocking, threshold 0.95)
run: |
uv run sprout serve &
python3 -m http.server 4173 --directory docs/audits &
sleep 5
npx --yes lighthouse http://127.0.0.1:8000/ \
--only-categories=accessibility --output=json --output-path=lh-chat-ui.json \
--chrome-flags="--headless=new --no-sandbox" --quiet
npx --yes lighthouse http://127.0.0.1:4173/eval-report.html \
--only-categories=accessibility --output=json --output-path=lh-eval-report.json \
--chrome-flags="--headless=new --no-sandbox" --quiet
python3 -c "
import json, sys
failed = False
for label, path in [('chat UI', 'lh-chat-ui.json'), ('eval report', 'lh-eval-report.json')]:
score = json.load(open(path))['categories']['accessibility']['score']
print(f'{label} Lighthouse accessibility score: {score}')
if score is None or score < 0.95:
failed = True
sys.exit(1 if failed else 0)
"
- name: Upload Lighthouse reports
if: always()
uses: actions/upload-artifact@043fb46d1a93c77aae656e7c1c64a875d1fc6a0a # v7.0.1
with:
name: lighthouse-reports
path: |
lh-chat-ui.json
lh-eval-report.json
retention-days: 14
docs:
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (docs)
run: uv sync --locked --group docs
- name: MkDocs strict build
run: uv run mkdocs build --strict
zizmor:
# Workflow-SAST review, RTA §F claim (CICD-19): any PR touching workflows is scanned.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: zizmor (workflow SAST)
run: uvx zizmor --offline --min-severity high .github/workflows/
ci-parity:
# Mechanically proves `make verify` covers the same commands as this file's required jobs
# (CONTRIBUTING.md "CI parity" note; ROADMAP.md ci-parity-no-mechanical-diff). Fails if this
# workflow or the Makefile changes on one side without the other.
runs-on: ubuntu-latest
steps:
- uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7
with:
persist-credentials: false
- uses: astral-sh/setup-uv@20cfd1bf945f4377ade1205e4dbc17946fc9a30d # v10.0.1
- name: Install (dev)
run: uv sync --locked
- name: make verify vs. ci-gate invocation diff
run: uv run sprout ci-parity-check
ci-gate:
# The ONLY required status check for branch protection. axe, pa11y, and Lighthouse
# accessibility are all merge-blocking (see docs/ROADMAP.md).
if: always()
needs: [test, security, eval-a11y, smoke, tuning-scope, web-static, pa11y, lighthouse, docs, zizmor, ci-parity]
runs-on: ubuntu-latest
steps:
- name: Require all gates to pass (skipped path-filtered jobs are OK)
# Interpolating join(needs.*.result) straight into the script body reads as a
# template-injection-adjacent pattern to workflow-SAST tools (zizmor flags it);
# the values are GitHub-controlled enums (success/failure/skipped/cancelled) so it
# was never exploitable, but routing through env: satisfies the rule directly
# instead of waiving it.
env:
GATE_RESULTS: ${{ join(needs.*.result, ' ') }}
run: |
echo "gate results: $GATE_RESULTS"
for r in $GATE_RESULTS; do
if [ "$r" != "success" ] && [ "$r" != "skipped" ]; then
echo "::error::a required gate did not pass: $r"
exit 1
fi
done