snapshot.yml

name: daily-snapshot

# The whole point of the project is the comparison between runs. A single snapshot cannot tell a
# field that was never collected from a field that was collected until last Tuesday, so the
# history has to accrue somewhere, and it accrues in git: a snapshot is per-field counts, a few
# hundred bytes, and the record of what stopped being published belongs in a diff that anyone can
# read rather than in a bucket they have to trust.
#
# The full graded report is deliberately NOT committed. It is several megabytes per run and every
# byte of it is reproducible from the snapshot plus the source, so committing it would trade the
# readability of the history for nothing.

on:
  schedule:
    - cron: "41 9 * * *" # daily, off the hour to avoid the top-of-hour scheduling crush
  workflow_dispatch:

permissions:
  contents: read

concurrency:
  group: daily-snapshot
  cancel-in-progress: false

jobs:
  snapshot:
    runs-on: ubuntu-latest
    permissions:
      contents: write # commits data/snapshots/<date>.json
    steps:
      - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
        with:
          fetch-depth: 0 # the previous snapshot is the entire point; a shallow clone has none

      - uses: actions/setup-python@5fda3b95a4ea91299a34e894583c3862153e4b97 # v7.0.0
        with:
          python-version: "3.12"

      - name: Install
        run: python -m pip install -e .

      # Without a key the adapter silently falls back to DEMO_KEY, which allows about three pages
      # an hour. The run would fetch ~300 of ~6,300 institutions and then rate-limit. Better to
      # refuse at the top with the remedy stated than to fail obscurely 40 pages in.
      - name: Require an API key
        env:
          DATA_GOV_API_KEY: ${{ secrets.DATA_GOV_API_KEY }}
        run: |
          if [ -z "${DATA_GOV_API_KEY}" ]; then
            echo "::error title=Missing DATA_GOV_API_KEY::The repository secret DATA_GOV_API_KEY \
          is not set. A full national run is ~63 pages; DEMO_KEY allows about three an hour, so \
          this job would rate-limit partway through and a partial fetch would look exactly like a \
          nationwide collapse in reporting. Get a free key at https://api.data.gov/signup/ and \
          add it under Settings > Secrets and variables > Actions."
            exit 1
          fi

      - name: Grade every institution
        env:
          DATA_GOV_API_KEY: ${{ secrets.DATA_GOV_API_KEY }}
        run: python -m disclosed.cli grade --out report.json

      # Into data/snapshots/scorecard/, kept apart from data/snapshots/ipeds/. The two are
      # different populations with no field in common, and a comparison across them would skip
      # every field and report "no change in per-field disclosure", which is the most reassuring
      # possible way of saying nothing at all. `drift` refuses such a pair outright; this keeps
      # the glob below from ever assembling one.
      - name: Reduce to a committable snapshot
        id: snap
        run: |
          taken="$(date -u +%F)"
          echo "taken=${taken}" >> "$GITHUB_OUTPUT"
          mkdir -p data/snapshots/scorecard
          python -m disclosed.cli snapshot \
            --report report.json \
            --taken "${taken}" \
            --out "data/snapshots/scorecard/${taken}.json"

      # Drift against the most recent earlier snapshot from this same source. Reported into the
      # job summary so a systemic change is visible without cloning anything.
      - name: Report drift since the previous snapshot
        run: |
          taken="${{ steps.snap.outputs.taken }}"
          previous="$(ls data/snapshots/scorecard/*.json 2>/dev/null \
            | grep -v "/${taken}.json$" | tail -n 1 || true)"
          if [ -z "${previous}" ]; then
            echo "First Scorecard snapshot; nothing to compare against yet." \
              >> "$GITHUB_STEP_SUMMARY"
            exit 0
          fi
          {
            echo "## Disclosure drift: $(basename "${previous}" .json) to ${taken}"
            echo '```'
            python -m disclosed.cli drift \
              "${previous}" "data/snapshots/scorecard/${taken}.json"
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"

      # Staged BEFORE the comparison, because `git diff` compares the index and the working tree
      # and an untracked file is in neither. Every daily snapshot is a new path, so the old shape
      # of this step -- `git diff --quiet -- data/snapshots/` ahead of the `git add` -- was asking
      # git a question that could only ever be answered "unchanged". It answered that ten days
      # running while the run above graded 6,273 institutions each time, and the record this
      # project's entire drift argument depends on was written to a runner and thrown away. The
      # comparison a new file can fail is the index against HEAD, which is what `--cached` is.
      - name: Commit the snapshot
        env:
          TAKEN: ${{ steps.snap.outputs.taken }}
        run: |
          git add data/snapshots/
          if git diff --cached --quiet --exit-code -- data/snapshots/; then
            echo "Snapshot for ${TAKEN} is byte-identical to the committed one; nothing to commit."
            exit 0
          fi
          git config user.name "github-actions[bot]"
          git config user.email "41898282+github-actions[bot]@users.noreply.github.com"
          git commit -m "data: per-field disclosure snapshot ${TAKEN} [skip ci]"
          git push

      # The post-condition, checked rather than assumed. A step that reports success without
      # having done anything is the failure this repository exists to name, and the previous
      # version of the step above was exactly that for ten consecutive runs. This one asserts the
      # effect: today's snapshot is in the history, not merely on the runner's disk.
      - name: Fail if the snapshot never reached the history
        env:
          TAKEN: ${{ steps.snap.outputs.taken }}
        run: |
          path="data/snapshots/scorecard/${TAKEN}.json"
          if ! git ls-files --error-unmatch -- "${path}" >/dev/null 2>&1; then
            echo "::error title=Snapshot not committed::${path} was written by this run but is \
          not tracked. The daily series is the only evidence behind every drift claim this \
          project makes, so a run that grades the country and commits nothing is a failed run, \
          not a quiet one."
            exit 1
          fi
          echo "${path} is committed."