publish-vector-data.yml

# Builds the vector half of the map in CI and publishes it to R2.
#
# "Vector half" means the trail lines and the POIs - everything except the USGS
# topo raster background. That split is what makes this workflow small enough to
# be worth having: export_trails.py and export_poi.py read only the ArcGIS and
# opentrail GeoJSON that fetch_all.py pulls (export_trails.py:409-416,
# export_poi.py:105-108), and never touch the 14 GB of topo quads. So this runs
# in minutes on a hosted runner with no special disk handling, where the raster
# stage needs a 51-way matrix and its own workflow.
#
# What a hiker gets from this alone: the trail drawn by blaze colour, shelters,
# water, campsites and resupply, local search, and their mile along the trail -
# on a blank background instead of topo. Genuinely useful, and honestly not the
# finished thing.
#
# Nothing here runs on a push. Publishing overwrites what hikers download, so it
# is a deliberate act: dispatch it, and tick `publish` only when you mean it.
# Left unticked it builds and gates and uploads nothing, which is the right way
# to find out whether upstream data still parses.
#
# WHICH ENVIRONMENT IT PUBLISHES TO defaults to UA rather than production, and
# that default is the same argument migrate.yml already makes about schema
# changes: UA first, always, so that the production run has a precedent instead
# of being the first attempt. features/DATA_ENVIRONMENTS.md is the design;
# publish.py refuses to run at all without a value, so the choice is made in the
# dispatch form rather than inherited from whatever ran last.

name: Publish vector data

on:
  workflow_dispatch:
    inputs:
      publish:
        description: "Upload to R2. Leave unticked for a dry run that builds and checks only."
        type: boolean
        default: false
      data_environment:
        description: "Which environment's data this becomes. UA first; production is the promotion."
        type: choice
        options: [ua, production, dev]
        default: ua
      include_elevation:
        description: "Also build the elevation profile. Adds ~25 min; no client code reads it yet."
        type: boolean
        default: false
      include_photos:
        description: "Fetch POI photos before exporting. Untick only to publish a release without them."
        type: boolean
        default: true
      include_osm_water:
        description: "Fetch OSM water points (Geofabrik extracts, ~3.5 GB / ~15 min). Untick to publish with opentrail's water alone."
        type: boolean
        default: true
      include_trail_water:
        description: "Derive trail crossings and reachable site water (adds ~5.7 GB of USGS subregions / ~30 min). Untick to reuse the last run's."
        type: boolean
        default: false

# One publish at a time. Two concurrent runs would race on latest.json and could
# interleave uploads, leaving the manifest describing a mixture of two builds.
concurrency:
  group: publish-data
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  build-and-publish:
    runs-on: ubuntu-latest
    # Building and publishing share a job on purpose. The export scripts record
    # absolute paths in their manifests (export_trails.py:387) and publish.py
    # reads those paths back, so the two only agree on one filesystem. Splitting
    # them across jobs would work only as long as the runner's checkout path
    # never changed - a trap rather than a design.
    environment: production
    # Raised from 60 when photos were added, because the old ceiling no longer
    # fit and the failure mode is bad: a timeout kills a release mid-run, and
    # the first thing publish.py uploads is photos, so a half-finished publish
    # is the one state the ordering was designed to avoid.
    #
    # Measured on the first cold run (2026-08-09): ATC photos 17m47s, and that
    # is with an empty cache, which is the normal case rather than the unlucky
    # one - see the cache step. Commons adds ~10-15 min cold on top, so photos
    # alone are ~30 min of a 60-minute budget. Ticking include_elevation as
    # well (+~25 min by its own note) would have run right at the ceiling.
    # Nobody had hit it because photos did not exist until now.
    timeout-minutes: 120
    defaults:
      run:
        working-directory: pipeline

    env:
      # One home for the cached-path list, read by both the restore and the
      # save step below (#542). An env var rather than a YAML anchor because
      # GitHub Actions does not support anchors, and two hand-kept copies of
      # a list whose whole job is completeness is the drift this repository
      # keeps finding. Paths are workspace-relative, unlike `run:` steps,
      # which default to pipeline/.
      FETCH_OUTPUTS: |
        pipeline/data/raw/*.geojson
        pipeline/data/raw/trail_water.json
        pipeline/data/raw/epqs_elevations.json
        pipeline/data/raw/manifest.json
        pipeline/data/raw/opentrail_state.json
        pipeline/data/raw/receipts
        pipeline/data/raw/poi_photos
        pipeline/data/raw/poi_images.json
        pipeline/data/raw/poi_images_atc.json
        pipeline/data/raw/elevation

    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.14"
          cache: pip
          cache-dependency-path: pipeline/requirements.txt

      # Runtime deps only - this builds real data rather than running the suite.
      - name: Install dependencies
        run: pip install -r requirements.txt

      # EVERY fetcher's durable output, carried between runs (#542).
      #
      # Each fetcher already writes its output atomically and each is
      # change-aware, so none of them needed fixing. What was missing is that
      # a GitHub runner is empty every time, so all that carefulness had
      # nothing to be careful ABOUT: fetch_all.py's skip compares against a
      # manifest.json that did not survive, and re-fetched all thirteen
      # layers from cold on every single run.
      #
      # That is what made run 31592776758 expensive. It died on a USGS 504 at
      # step 13 and discarded twelve completed steps - and the photos were
      # the only thing that came back, because the photo cache this step
      # grew out of was the only cache in the job.
      #
      # THE PATHS MUST COVER EVERY OUTPUT ANY RECEIPT NAMES. Receipts are
      # restored with the files they describe, and check_output_quality.py
      # re-hashes those files against the receipt - so a cache that restored a
      # receipt without its output would fail the gate on its own contents,
      # reporting drift where there was only a gap in this list.
      #
      # That is not a hypothetical either: the first version of this list had
      # elevation's tile index under data/processed/, where it does not live.
      # tests/test_fetch_cache_paths.py now checks every entry against the
      # constants the fetchers actually write to, because a hand-kept list
      # whose whole job is completeness is the drift this repository keeps
      # finding.
      #
      # Split restore/save rather than `actions/cache`, and `always()` on the
      # save, because the bundled post-step is skipped when the job FAILS -
      # so a plain cache would discard the work on precisely the runs this
      # exists to rescue. That is not hypothetical: it is what happened on
      # both failed runs of 2026-08-09, and what #549 found the elevation
      # catalogue cache doing before #550 removed it.
      #
      # Not conditional on include_photos, unlike the photo-only cache it
      # replaces: the ArcGIS layers matter to every run. Changing the key
      # orphans the old poi-photos- entries, so the first run after this
      # merges is cold - once.
      - name: Restore fetched data
        id: fetch-cache
        uses: actions/cache/restore@v6
        with:
          path: ${{ env.FETCH_OUTPUTS }}
          key: fetch-outputs-${{ github.run_id }}
          restore-keys: fetch-outputs-

      # fetch_all.py covers the ArcGIS sources only - it drives lib/arcgis.py and
      # iterates sources.json. opentrail.org is a different API with its own
      # script, and export_poi.py reads its output directly
      # (export_poi.py:108), so both fetches are required before any export.
      - name: Fetch ArcGIS sources
        run: python fetch_all.py

      - name: Fetch opentrail.org POIs
        run: python fetch_opentrail.py

      # Read the POI sources and clip them BEFORE any of the fetching below,
      # and fail here if they are not exportable. Writes nothing.
      #
      # This step exists because the ordering underneath it is unavoidable
      # and expensive: `export_poi.py` attaches photos, so it has to run
      # after the fetches, which means every defect in the DATA got
      # discovered an hour into a run. It happened twice in one release - a
      # row ATC left with no geometry, and before that a dead photo link -
      # each costing a full run and the photo cache with it.
      #
      # `--check` runs the export's own reading code, not a copy of it, so a
      # source this passes is a source the export can read. What it cannot
      # speak for is the enrichment, which needs the fetches; the same gate
      # therefore runs again inside the export proper.
      - name: Check POI sources are exportable
        run: python export_poi.py --check

      # Photos live outside the repository - pipeline/.gitignore excludes
      # data/ entirely - so a fresh runner has neither the outcome files nor
      # the image bytes, and without this step export_poi.py takes its
      # "exporting without photos" path and publish.py finds nothing to
      # upload. That is a green run that silently ships POIs with no photo,
      # which is how this went unnoticed when the ATC source was added.
      #
      # Their bytes come back from the "Restore fetched data" step at the top
      # of this job, which covers the photo store along with every other
      # fetcher's output (#542). The key is deliberately coarse - one cache,
      # restored by prefix - because the fetches themselves decide what is
      # stale, and a cache miss costs time rather than correctness.
      #
      # Do not expect it to hit. GitHub evicts cache entries not accessed for
      # 7 days, and DATA_RELEASES.md puts releases on a monthly-ish cadence,
      # so the normal publish finds no cache and pays the full ~18 minutes
      # (measured cold, 2026-08-09) plus Commons. It earns its keep on runs
      # close together - a dry run followed by the real publish, or a re-run
      # after a failure - which is exactly when re-downloading 75 MB from ATC
      # would be most gratuitous.
      #
      # The cold cost is not pure waste, which is why it is tolerated rather
      # than engineered around: ATC is still adding photos to these layers
      # (one dated October 2025), and a cold fetch is how those get found.
      # If it ever needs to be cheap, the fix is not a longer-lived cache -
      # it is teaching the fetch that bytes already in R2 need not be
      # downloaded again to be published.
      #
      # ATC's own facility photographs: 453 of the 522 photos a hiker sees,
      # and the only source covering shelters and campsites at all
      # (features/POI_PHOTOS.md). Allowed to fail the release, unlike the
      # Commons fetch below: this reads the same ATC data the export is built
      # from, so if it is unreachable the release has bigger problems than
      # missing photos.
      - name: Fetch ATC facility photos
        if: inputs.include_photos
        run: python fetch_atc_photos.py

      # Commons fills water and resupply only, and it is a third-party API
      # this project has no relationship with - so a Commons outage must not
      # be able to block a trail-data release. continue-on-error is the whole
      # point of running it separately: the export ships cleanly without it,
      # carrying whatever the last cached run found.
      - name: Fetch Commons photos
        if: inputs.include_photos
        continue-on-error: true
        run: python fetch_poi_images.py

      # OSM water points (#529): the fourteen Geofabrik state extracts, the
      # same files the basemap build downloads, scanned for the four
      # point-source tags. The extracts are NOT in FETCH_OUTPUTS - caching
      # ~3.5 GB that Geofabrik republishes daily would evict the caches that
      # earn their keep - but the scan's small output geojson is, via the
      # *.geojson glob, so an unticked run still exports whatever water the
      # last ticked run found, the same way photos ride the cache between
      # runs. Untick to skip the download; export_poi.py ships opentrail's
      # points alone, as every release before this source did.
      - name: Fetch OSM water points
        if: inputs.include_osm_water
        run: python fetch_osm_water.py

      # Where the trail meets water, and which sites have water a hiker can
      # reach (#529). Off by default, unlike the OSM points: it downloads
      # ~5.7 GB of USGS subregions on top of reading the extracts, and its
      # answer barely moves - NHD is a frozen snapshot and the trail does not
      # often change where it fords a creek. The output rides FETCH_OUTPUTS,
      # so an unticked run exports the last derivation's crossings rather
      # than none, exactly as the photo fetches behave. Its elevation cache
      # rides along too, so a re-run after a threshold change pays for the
      # hydrography and not for several hundred more EPQS round trips.
      - name: Derive trail water
        if: inputs.include_trail_water
        run: python fetch_trail_water.py

      - name: Export trail lines
        run: python export_trails.py

      - name: Export POIs
        run: python export_poi.py

      # AFTER the POIs, and that ordering is load-bearing rather than tidy.
      # export_spurs.py publishes the id of the POI each spur leads to, and
      # that id has to be the one the client already holds - so it resolves
      # against export_poi.py's published records, not against the raw ATC
      # points they were built from. Run first, it would publish links to POIs
      # no device has ever heard of.
      - name: Export spur destinations
        run: python export_spurs.py

      # Elevation has its own fetch too, and it is the slow one - it streams
      # USGS 3DEP tiles, which is why the whole leg is opt-in.
      #
      # There is no catalogue cache to restore any more (#550). This step used
      # to hold `actions/cache@v4` over the per-cell TNM answers, and it went
      # with the API it cached: the tile list is now computed from the
      # corridor's bounding box, so this step makes no requests that could 504
      # and has nothing worth remembering between runs.
      #
      # It is worth noting what that cache was actually doing, since it was
      # added for #536 and is being deleted rather than fixed: `actions/cache`
      # skips its post-run save when the job FAILS, so the cache built to
      # rescue a run that died mid-fetch was discarded on exactly those runs.
      # Computing the list removes the need for either the cache or the
      # restore/save split that would have fixed it.
      - name: Fetch elevation tiles
        if: inputs.include_elevation
        run: python fetch_elevation.py

      - name: Export elevation profile
        if: inputs.include_elevation
        run: python export_elevation.py

      # Saved even when an earlier step failed, which is the whole reason the
      # cache is split into restore/save rather than left as `actions/cache`:
      # the bundled post-step only runs on success, so both failed runs of
      # 2026-08-09 threw away the ~55 minutes of fetching they had already
      # done and the retry started from cold.
      #
      # AFTER the last fetch, so one save covers all five. The photo-only
      # version of this step sat above the exports and therefore could not
      # carry elevation's tile index at all. What is on disk is correct
      # whatever the exports make of it - the fetch outputs are gated and
      # atomic, and the photos are content-addressed - so there is nothing to
      # be gained by discarding them because a later step disagreed with the
      # data.
      #
      # `always()` rather than `failure()` so the successful path saves
      # through the same line, and unconditional on the photo/elevation
      # inputs because a run that fetched neither still refreshed thirteen
      # ArcGIS layers worth keeping.
      - name: Save fetched data
        if: always()
        uses: actions/cache/save@v6
        with:
          path: ${{ env.FETCH_OUTPUTS }}
          key: fetch-outputs-${{ github.run_id }}

      # Record what every upstream said at the moment this build fetched, and
      # publish it beside the artifacts as build_state.json. That file is the
      # entire input to the scheduled freshness check
      # (.github/workflows/check-upstream-freshness.yml), which reads it over
      # the public URL and therefore needs no R2 credentials of its own.
      #
      # After every fetch, never before: the state has to describe the data
      # this build actually used. It captures only the sources this run
      # touched - a run without include_elevation records no elevation
      # markers, and the check reports that source as unchecked rather than
      # inventing a verdict for a leg that never ran.
      - name: Capture upstream freshness markers
        run: python check_freshness.py --capture data/processed/build_state.json

      # Runs before publish, never after. On a hosted runner there is no
      # data/quality_baseline.json from a previous run, so the drop-vs-baseline
      # check reports SKIPPED rather than comparing - the completeness and
      # corridor checks still do real work, and a SKIPPED check says so out loud
      # instead of quietly passing.
      # --optional elevation unless it was built: this run deliberately produces
      # a subset, which publish.py supports, and a gate that insisted on the
      # full set would contradict the thing it gates. It only excuses an ABSENT
      # artifact - one that exists and fails still fails.
      #
      # --fetched names the CONDITIONAL fetchers this run asked for, so their
      # receipts are required too (#542). fetch_all and fetch_opentrail need
      # no flag - no export can run without either, so that is a fact about
      # the code rather than about this run. Note the asymmetry with
      # --optional one line up, and it is deliberate: --optional excuses an
      # artifact that was never built, while --fetched ADDS a requirement.
      # A missing receipt is never excused, because it is the finding.
      #
      # fetch_poi_images is deliberately not named even when photos were
      # asked for. Its step carries continue-on-error, so requiring its
      # receipt would fail releases the workflow has already decided to
      # allow. It is reported with its age either way, so a run that shipped
      # without Commons photos still says so out loud.
      - name: Check output quality
        run: |
          fetched=""
          if [ "${{ inputs.include_photos }}" = "true" ]; then
            fetched="--fetched fetch_atc_photos"
          fi
          if [ "${{ inputs.include_osm_water }}" = "true" ]; then
            fetched="$fetched --fetched fetch_osm_water"
          fi
          if [ "${{ inputs.include_trail_water }}" = "true" ]; then
            fetched="$fetched --fetched fetch_trail_water"
          fi
          if [ "${{ inputs.include_elevation }}" = "true" ]; then
            python check_output_quality.py $fetched --fetched fetch_elevation
          else
            python check_output_quality.py $fetched --optional elevation
          fi

      - name: What would be published
        run: |
          python - <<'PY'
          from pathlib import Path

          from publish import collect_artifacts

          artifacts = collect_artifacts()
          total = 0
          for name, entry in sorted(artifacts.items()):
              size = Path(entry["path"]).stat().st_size
              total += size
              print(f"{size / 1e6:9.2f} MB  {name}")
          print(f"{total / 1e6:9.2f} MB  TOTAL across {len(artifacts)} artifacts")
          PY

      # Photos are outside the manifest by design (publish.py's collect_photos
      # explains why), so the artifact listing above cannot show them - and a
      # dry run that says nothing about photos is exactly how a release ships
      # without them and nobody notices until a card is blank.
      - name: How many photos would be published
        run: |
          python - <<'PY'
          import json
          from pathlib import Path

          from lib.poi_schema import POI_TYPES
          from publish import collect_photos

          photos = collect_photos()
          size = sum(Path(p).stat().st_size for p in photos.values())
          print(f"{len(photos)} photo object(s), {size / 1e6:.1f} MB")

          for name in ("poi_images_atc.json", "poi_images.json"):
              path = Path("data/raw") / name
              if not path.exists():
                  print(f"  {name}: absent")
                  continue
              records = json.loads(path.read_text())["pois"]
              found = sum(1 for r in records.values() if r.get("status") == "found")
              print(f"  {name}: {found} found of {len(records)} checked")

          # Every published type, read from the schema rather than listed
          # here: this block used to name four, which was every type that
          # could carry a photo when it was written. The day vistas, parking
          # and privies started publishing - three categories ATC has its own
          # photographs for - they would have been silently outside both the
          # count and the missing-bytes check below.
          referenced = set()
          for poi_type in POI_TYPES:
              exported = Path("data/processed/poi") / f"{poi_type}.geojson"
              if not exported.exists():
                  continue
              features = json.loads(exported.read_text())["features"]
              with_photo = [f for f in features if f["properties"].get("photo_key")]
              referenced.update(f["properties"]["photo_key"] for f in with_photo)
              print(f"  {poi_type}: {len(with_photo)}/{len(features)} features carry a photo")

          # The one failure that would reach a hiker as a broken card: an
          # exported feature pointing at an object this run is not uploading.
          # publish.py already orders photos before artifacts, so this catches
          # the other case - bytes that were never fetched at all.
          missing = referenced - set(photos)
          if missing:
              raise SystemExit(f"{len(missing)} exported photo_key(s) have no local image to upload, e.g. {sorted(missing)[:3]}")
          PY

      # Kept short-lived: these are inspectable evidence for a dry run, not a
      # distribution channel. R2 is the distribution channel.
      - name: Upload built artifacts for inspection
        uses: actions/upload-artifact@v4
        with:
          name: vector-data
          path: |
            pipeline/data/processed/trails.geojson
            pipeline/data/processed/trails_manifest.json
            pipeline/data/processed/poi/
            pipeline/data/processed/elevation_manifest.json
          retention-days: 3
          if-no-files-found: error

      - name: Publish to R2
        if: inputs.publish
        env:
          # The gate publish.py checks. Set only on this step, so every other
          # step in this workflow is structurally incapable of writing to the
          # bucket - see publish.py's writes_enabled().
          R2_WRITE_ENABLED: "true"
          # The second gate, and the one that decides *where*. publish.py scopes
          # every key it writes by this, so a UA run cannot reach production's
          # keys rather than merely being expected not to
          # (features/DATA_ENVIRONMENTS.md).
          OURHIKE_DATA_ENV: ${{ inputs.data_environment }}
          R2_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
          R2_BUCKET: ${{ secrets.R2_BUCKET }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
        run: python publish.py

      - name: Say what happened
        if: always()
        run: |
          if [ "${{ inputs.publish }}" != "true" ]; then
            echo "Dry run - nothing was uploaded. Re-run with 'publish' ticked to upload."
          elif [ "${{ inputs.data_environment }}" = "production" ]; then
            echo "Published to production. The client needs DATA_BASE_URL set and a Pages redeploy to see it."
          elif [ "${{ inputs.data_environment }}" = "ua" ]; then
            echo "Published to ua, under environments/ua/. Production's keys were not touched."
            echo "UA reads this automatically once ua.yml next rebuilds - see features/DATA_ENVIRONMENTS.md."
          else
            echo "Published to ${{ inputs.data_environment }}, under environments/${{ inputs.data_environment }}/."
            echo "Production's keys were not touched."
          fi