build-basemap.yml

# Builds the offline vector basemap and cuts the AT package from it - the
# proving run for pipeline/BASEMAP.md's build-once-extract-many design
# (issues #184/#185).
#
# Fits the FREE runner on purpose, and the pre-clip is what makes that true:
# osmium cuts each state extract down to the corridor shape before Planetiler
# ever runs, so Planetiler's RAM and temp-disk needs (0.5x / 5-10x its input)
# scale with the corridor ribbon, not with fourteen whole states. The
# fetched state PBFs (~3-4 GB) plus intermediates land well inside the 88 GB
# a hosted runner actually has free - measured by the step below, which is
# why that step exists and why no number here is guessed. When coverage
# outgrows this shape - a statewide network, a whole region - the answer is
# MORE free runners rather than a bigger one: every Geofabrik sub-region
# fits this machine, so a continental build shards across a matrix and
# splits at z9. Never a rebuild per trail. See BASEMAP.md and #194.
#
# Publishing is opt-in and twice-guarded, the same posture as
# publish-vector-data.yml and build-raster.yml: the `publish` input defaults
# to false, only the publish job holds R2 credentials, and it refuses any
# run that is not the full corridor build - a states-subset shakedown or a
# shallower pyramid stays a spike, and a spike that can accidentally ship is
# not a spike. What publishes is the AT package cut, never the source build.
# This is today's flat-key publish mechanism (publish.py + latest.json);
# DATA_RELEASES.md's immutable release folders remain the destination once
# that machinery exists, exactly as for every other artifact.
#
# The Planetiler jar is pinned by input rather than hardcoded, because its
# releases move and a bad pin should be fixable from the dispatch form, not a
# commit. Its default profile emits the OpenMapTiles schema the client's
# liveTopo.ts already renders - that schema match is the entire reason this
# build exists (see BASEMAP.md's "why not Protomaps/Shortbread" section).

name: Build vector basemap

on:
  workflow_dispatch:
    inputs:
      states:
        description: "Space-separated Geofabrik state names. Empty = the 14 AT states."
        type: string
        default: ""
      max_zoom:
        description: "Tile pyramid depth (14 = OpenMapTiles convention)"
        type: string
        default: "14"
      planetiler_jar_url:
        description: "Planetiler release jar to build with"
        type: string
        default: "https://github.com/onthegomap/planetiler/releases/latest/download/planetiler.jar"
      publish:
        description: "Upload the AT package to R2. Only full corridor builds (all 14 states, z14) are allowed through."
        type: boolean
        default: false
      data_environment:
        description: "Which environment's data this becomes. UA first; production is the promotion."
        type: choice
        options: [ua, production, dev]
        default: ua

concurrency:
  group: build-basemap
  cancel-in-progress: false

permissions:
  contents: read

jobs:
  build:
    runs-on: ubuntu-latest
    timeout-minutes: 180
    defaults:
      run:
        working-directory: pipeline
    steps:
      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.14"
          cache: pip
          cache-dependency-path: pipeline/requirements.txt

      - uses: actions/setup-java@v5
        with:
          distribution: temurin
          java-version: "21"

      # Reported rather than assumed, same as build-raster.yml: free disk is
      # the number the pre-clip design turns on, so it gets printed per run.
      - name: Runner capacity
        run: |
          echo "disk:"; df -h / /mnt 2>/dev/null || df -h /
          echo "cpus: $(nproc)"
          echo "memory:"; free -g

      - name: Install dependencies
        run: |
          pip install -r requirements.txt
          sudo apt-get update -qq && sudo apt-get install -y -qq osmium-tool
          osmium --version | head -1

      - name: Warm the DuckDB spatial extension
        run: python -c "import duckdb; duckdb.connect().execute('INSTALL spatial; LOAD spatial;')"

      # centerline.geojson, which the corridor is built from. fetch_all.py is
      # the only producer - same step build-raster.yml runs.
      - name: Fetch ArcGIS sources
        run: python fetch_all.py

      - name: Fetch the Planetiler jar
        run: |
          curl -sSL -o planetiler.jar "${{ inputs.planetiler_jar_url }}"
          ls -la planetiler.jar

      - name: Build the basemap
        run: |
          python export_basemap.py --planetiler-jar planetiler.jar \
            --max-zoom "${{ inputs.max_zoom }}" \
            ${{ inputs.states != '' && format('--states {0}', inputs.states) || '' }}

      - name: Cut the AT package from the build
        run: |
          python extract_package.py data/processed/basemap.pmtiles \
            --region data/processed/basemap_region.geojson \
            --out data/processed/at_basemap_package.pmtiles \
            --name "OurHike AT corridor basemap"

      # The Standard level (#276): the same cut capped at z13, its own
      # artifact so its advertised size and published hash describe exactly
      # the bytes a hiker gets. z14 is 66% of the full package's bytes, so
      # this is the ~180 MB option.
      - name: Cut the z13 package from the build
        run: |
          python extract_package.py data/processed/basemap.pmtiles \
            --region data/processed/basemap_region.geojson \
            --max-zoom 13 \
            --out data/processed/at_basemap_package_z13.pmtiles \
            --name "OurHike AT corridor basemap (z13)"

      # The whole point of the spike: real numbers where estimates were. To
      # the log AND the summary - a number that exists only in a rendered
      # summary cannot be quoted by anything reading the logs later.
      - name: Measure
        # `shell: bash` for `-o pipefail`: `| tee` below would otherwise take
        # this step's exit status from `tee`, so a crashed measurement would
        # report success on a build that produced nothing (#514).
        shell: bash
        run: |
          python -c "
          from pathlib import Path
          from export_basemap import report_archive
          report_archive(Path('data/processed/basemap.pmtiles'))
          report_archive(Path('data/processed/at_basemap_package.pmtiles'))
          report_archive(Path('data/processed/at_basemap_package_z13.pmtiles'))
          " | tee measure.txt
          {
            echo "## Basemap build"
            echo '```'
            cat measure.txt
            echo '```'
          } >> "$GITHUB_STEP_SUMMARY"
          df -h /

      - uses: actions/upload-artifact@v4
        with:
          name: basemap-archives
          path: |
            pipeline/data/processed/basemap.pmtiles
            pipeline/data/processed/basemap_region.geojson
          retention-days: 7
          if-no-files-found: error

      # The packages ride separately so the publish job downloads the ~0.7 GB
      # it ships and not the source build beside it.
      - uses: actions/upload-artifact@v4
        with:
          name: basemap-package
          path: |
            pipeline/data/processed/at_basemap_package.pmtiles
            pipeline/data/processed/at_basemap_package_z13.pmtiles
          retention-days: 7
          if-no-files-found: error

  publish:
    needs: build
    if: inputs.publish
    runs-on: ubuntu-latest
    environment: production
    # Every writer of latest.json shares this group: publish() does a
    # read-modify-write of the manifest, and two concurrent publishes could
    # interleave and clobber each other's entries.
    concurrency:
      group: publish-data
      cancel-in-progress: false
    timeout-minutes: 30
    defaults:
      run:
        working-directory: pipeline
    steps:
      # The client's basemap package declares one shape (the 14-state
      # corridor build's z0-14 cut, measured size). A states-subset build
      # would publish a package with most of the trail missing from it -
      # refused loudly rather than quietly skipped.
      - name: Refuse a non-canonical shape
        if: inputs.states != '' || inputs.max_zoom != '14'
        working-directory: .
        run: |
          echo "::error::publish is only for the full corridor build (states empty, z0-14); this run built states='${{ inputs.states }}' at z0-${{ inputs.max_zoom }}"
          exit 1

      - uses: actions/checkout@v4

      - uses: actions/setup-python@v5
        with:
          python-version: "3.14"
          cache: pip
          cache-dependency-path: pipeline/requirements.txt

      - name: Install dependencies
        run: pip install -r requirements.txt

      # A separate job from build on purpose, mirroring build-raster.yml:
      # publish.py hashes archives from PROCESSED_DIR at publish time rather
      # than reading a baked manifest, so nothing depends on this being the
      # runner that produced them - and the build job never holds a secret.
      - uses: actions/download-artifact@v4
        with:
          name: basemap-package
          path: pipeline/data/processed/

      - name: What will be published
        run: ls -la data/processed/

      - name: Publish to R2
        env:
          # The gate publish.py checks. Set only on this step, so every other
          # step in this workflow is structurally incapable of writing to the
          # bucket - see publish.py's writes_enabled().
          R2_WRITE_ENABLED: "true"
          # Which environment's data this becomes. publish.py scopes every
          # key by it and refuses to run without one, so a build cannot reach
          # production's keys by omission (features/DATA_ENVIRONMENTS.md).
          OURHIKE_DATA_ENV: ${{ inputs.data_environment }}
          R2_ENDPOINT_URL: ${{ secrets.R2_ENDPOINT_URL }}
          R2_BUCKET: ${{ secrets.R2_BUCKET }}
          R2_ACCESS_KEY_ID: ${{ secrets.R2_ACCESS_KEY_ID }}
          R2_SECRET_ACCESS_KEY: ${{ secrets.R2_SECRET_ACCESS_KEY }}
        run: python publish.py