gcp_llm_gateway.yml

name: Deploy LLM Gateway to GKE

on:
  workflow_dispatch:
    inputs:
      environment:
        description: 'Select the environment to deploy to'
        required: true
        default: 'development'
        type: choice
        options:
          - development
          - prod
      branch:
        description: 'Branch to deploy from'
        required: true
        default: 'main'
      release_sha:
        description: 'Production only: exact main SHA with a successful first-attempt Release Eligibility proof'
        required: false
        default: ''
      # Break-glass: the eligibility proof requires a *successful*
      # first-attempt Release Eligibility push-run on this exact SHA. A flaky
      # or red check therefore poisons that SHA permanently -- rerunning does
      # not retroactively make the original run succeed, so the only other
      # recovery is pushing a new commit to main and waiting for a full green
      # run. That is unacceptable for a hotfix, so this hatch exists.
      #
      # It does NOT relax the property that matters: the SHA must still be an
      # ancestor of main, so unreviewed code can never reach production.
      skip_eligibility_proof:
        description: 'Break-glass: deploy without a Release Eligibility proof (still requires a merged main SHA)'
        required: false
        default: false
        type: boolean
      break_glass_confirm:
        description: 'Type deploy-without-proof when skip_eligibility_proof is set'
        required: false
        default: ''
      break_glass_reason:
        description: 'Why the eligibility proof is unavailable (required with skip_eligibility_proof)'
        required: false
        default: ''

# LLM Gateway deployment reapplies shared backend-secrets, so it joins the
# full backend mutation domain for the selected environment.
concurrency:
  group: deploy-backend-stack-${{ github.event.inputs.environment }}
  cancel-in-progress: false

env:
  SERVICE: llm-gateway
  REGION: us-central1

jobs:
  deploy:
    environment: ${{ github.event.inputs.environment == 'prod' && 'prod' || 'development' }}
    permissions:
      actions: 'read'
      contents: 'read'
      id-token: 'write'

    runs-on: ubuntu-latest
    outputs:
      break_glass: ${{ steps.admitted_source.outputs.break_glass }}
      admitted_sha: ${{ steps.admitted_source.outputs.admitted_sha }}
    steps:
      - name: Validate Environment Input
        run: |
          if [[ "${{ github.event.inputs.environment }}" != "development" && "${{ github.event.inputs.environment }}" != "prod" ]]; then
            echo "Invalid environment: ${{ github.event.inputs.environment }}. Must be 'development' or 'prod'."
            exit 1
          fi
      - name: Delete huge unnecessary tools folder
        run: rm -rf /opt/hostedtoolcache

      - name: Checkout
        uses: actions/checkout@v7
        with:
          # Production starts from fresh main only to validate the requested
          # immutable SHA; development retains its caller-selected branch.
          ref: ${{ github.event.inputs.environment == 'prod' && 'main' || github.event.inputs.branch }}
          fetch-depth: 0

      - name: Admit production source and image identity
        id: admitted_source
        env:
          DEPLOY_SHA: ${{ github.event.inputs.release_sha }}
          DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }}
          SKIP_PROOF: ${{ github.event.inputs.skip_eligibility_proof }}
          BREAK_GLASS_CONFIRM: ${{ github.event.inputs.break_glass_confirm }}
          BREAK_GLASS_REASON: ${{ github.event.inputs.break_glass_reason }}
          GH_TOKEN: ${{ github.token }}
        run: |
          set -euo pipefail
          sha_pattern='^[0-9a-f]{40}$'
          if [[ "$DEPLOY_ENVIRONMENT" == prod ]]; then
            if [[ ! "$DEPLOY_SHA" =~ $sha_pattern || "$DEPLOY_SHA" == "0000000000000000000000000000000000000000" ]]; then
              echo "ERROR: release_sha must be one non-zero, lowercase, full commit SHA." >&2
              exit 1
            fi
            git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
            main_sha="$(git rev-parse --verify 'origin/main^{commit}')"
            git cat-file -e "${DEPLOY_SHA}^{commit}"

            # Always enforced, break-glass or not: only merged main code deploys.
            if ! git merge-base --is-ancestor "$DEPLOY_SHA" "$main_sha"; then
              echo "ERROR: release_sha is not an ancestor of fresh origin/main; only merged code may deploy." >&2
              exit 1
            fi

            if [[ "${SKIP_PROOF:-false}" == "true" ]]; then
              if [[ "$BREAK_GLASS_CONFIRM" != "deploy-without-proof" ]]; then
                echo "ERROR: skip_eligibility_proof requires break_glass_confirm=deploy-without-proof." >&2
                exit 1
              fi
              if [[ -z "${BREAK_GLASS_REASON// }" ]]; then
                echo "ERROR: skip_eligibility_proof requires a non-empty break_glass_reason." >&2
                exit 1
              fi
              echo "::warning::Release Eligibility proof skipped for ${DEPLOY_SHA}: ${BREAK_GLASS_REASON}"
              printf 'break_glass=true\n' >> "$GITHUB_OUTPUT"
            else
              proof_path="${RUNNER_TEMP}/release-eligibility-${DEPLOY_SHA}.json"
              gh api -H "Accept: application/vnd.github+json" \
                "repos/${GITHUB_REPOSITORY}/actions/workflows/release-eligibility.yml/runs?event=push&branch=main&status=completed&head_sha=${DEPLOY_SHA}&per_page=100" \
                > "$proof_path"
              python3 .github/scripts/verify_backend_release_admission.py \
                --sha "$DEPLOY_SHA" \
                --repository "$GITHUB_REPOSITORY" \
                --workflow-runs "$proof_path" \
                --require-first-attempt
              printf 'break_glass=false\n' >> "$GITHUB_OUTPUT"
            fi
            git checkout --detach "$DEPLOY_SHA"
          fi
          CHECKED_OUT_SHA=$(git rev-parse HEAD)
          if [[ "$DEPLOY_ENVIRONMENT" == prod && "$CHECKED_OUT_SHA" != "$DEPLOY_SHA" ]]; then
            echo "ERROR: checked-out HEAD $CHECKED_OUT_SHA does not match admitted release_sha $DEPLOY_SHA." >&2
            exit 1
          fi
          if [[ "$DEPLOY_ENVIRONMENT" == prod ]] && ! git merge-base --is-ancestor "$CHECKED_OUT_SHA" origin/main; then
            echo "ERROR: checked-out HEAD $CHECKED_OUT_SHA is not an ancestor of fresh origin/main $(git rev-parse origin/main)." >&2
            exit 1
          fi
          IMAGE_TAG=$(git rev-parse --short=7 "$CHECKED_OUT_SHA")
          echo "Admitted deployment source: $CHECKED_OUT_SHA"
          echo "CHECKED_OUT_SHA=$CHECKED_OUT_SHA" >> "$GITHUB_ENV"
          echo "IMAGE_TAG=$IMAGE_TAG" >> "$GITHUB_ENV"
          if [[ "$DEPLOY_ENVIRONMENT" == prod ]]; then
            printf 'admitted_sha=%s\n' "$DEPLOY_SHA" >> "$GITHUB_OUTPUT"
          fi

      - name: Google Auth
        id: auth
        uses: 'google-github-actions/auth@v3'
        with:
          credentials_json: ${{ secrets.GCP_CREDENTIALS }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v3

      - name: Login to GCR
        run: gcloud auth configure-docker

      - name: Set up Docker Buildx
        uses: docker/setup-buildx-action@v4

      - name: Install Helm
        uses: azure/setup-helm@v5

      - name: Verify production gateway service token
        if: github.event.inputs.environment == 'prod'
        run: |
          gcloud secrets describe OMI_LLM_GATEWAY_SERVICE_TOKEN \
            --project="${{ vars.GCP_PROJECT_ID }}" >/dev/null

      - name: Validate gateway env wiring
        run: |
          python backend/scripts/validate-llm-gateway-env.py \
            backend/charts/backend-listen/${{ vars.ENV }}_omi_backend_listen_values.yaml \
            backend/charts/llm-gateway/${{ vars.ENV }}_omi_llm_gateway_values.yaml

      - name: Require LLM Gateway Workload Identity binding
        env:
          LLM_GATEWAY_GSA: ${{ vars.LLM_GATEWAY_GSA }}
        run: test -n "$LLM_GATEWAY_GSA"

      - name: Build and Push Docker image
        run: |
          docker build -t gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG} -f backend/Dockerfile .
          python3 backend/scripts/runtime_image_contracts.py smoke \
            --dockerfile backend/Dockerfile \
            --image gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG}
          docker push gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG}

      - name: Get GKE credentials
        uses: google-github-actions/get-gke-credentials@v3
        with:
          cluster_name: ${{ vars.GKE_CLUSTER }}
          location: ${{ env.REGION }}
          project_id: ${{ vars.GCP_PROJECT_ID }}

      - name: Deploy LLM Gateway to GKE cluster using Helm
        env:
          LLM_GATEWAY_GSA: ${{ vars.LLM_GATEWAY_GSA }}
          ENVIRONMENT: ${{ vars.ENV }}
          GCP_PROJECT_ID: ${{ vars.GCP_PROJECT_ID }}
          GKE_CLUSTER: ${{ vars.GKE_CLUSTER }}
          REGION: ${{ env.REGION }}
        run: |
          IMAGE_TAG="${IMAGE_TAG}" backend/scripts/deploy-llm-gateway.sh

      - name: Verify LLM Gateway serving data plane
        id: gateway-serving
        run: |
          python3 backend/scripts/verify-llm-gateway-serving.py \
            --environment="${{ vars.ENV }}" \
            --project="${{ vars.GCP_PROJECT_ID }}" \
            --region="${{ env.REGION }}" \
            --github-output "$GITHUB_OUTPUT"

      - name: Probe LLM Gateway from the Cloud Run VPC
        run: |
          bash backend/scripts/probe-llm-gateway-from-cloud-run.sh \
            --project "${{ vars.GCP_PROJECT_ID }}" \
            --region "${{ env.REGION }}" \
            --image "gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG}" \
            --gateway-url "${{ steps.gateway-serving.outputs.gateway_url }}" \
            --network "${{ vars.CLOUD_RUN_VPC_NETWORK }}" \
            --subnet "${{ vars.CLOUD_RUN_VPC_SUBNET }}" \
            --vpc-egress private-ranges-only \
            --name-suffix "${GITHUB_RUN_ID}-${GITHUB_RUN_ATTEMPT}"

      - name: Smoke LLM Gateway
        run: |
          NAMESPACE="${{ vars.ENV }}-omi-backend"
          TOKEN="$(kubectl -n "$NAMESPACE" get secret ${{ vars.ENV }}-omi-backend-secrets -o jsonpath='{.data.OMI_LLM_GATEWAY_SERVICE_TOKEN}' | base64 -d)"
          METRICS_TOKEN="$(kubectl -n "$NAMESPACE" get secret ${{ vars.ENV }}-omi-backend-secrets -o jsonpath='{.data.METRICS_SECRET}' | base64 -d)"
          kubectl -n "$NAMESPACE" run "llm-gateway-smoke-${GITHUB_RUN_ID}" \
            --rm -i --restart=Never \
            --image "gcr.io/${{ vars.GCP_PROJECT_ID }}/${{ env.SERVICE }}:${IMAGE_TAG}" \
            --env "SMOKE_URL=http://${{ vars.ENV }}-omi-llm-gateway.${{ vars.ENV }}-omi-backend.svc.cluster.local:8080" \
            --env "SMOKE_TOKEN=$TOKEN" \
            --env "METRICS_SECRET=$METRICS_TOKEN" \
            --command -- sh -c 'python scripts/smoke-llm-gateway.py --url "$SMOKE_URL" --token "$SMOKE_TOKEN" --check-metrics'

      - name: Show Output
        run: echo "LLM Gateway deployed to ${{ github.event.inputs.environment }}"

  record_break_glass:
    needs: deploy
    if: ${{ needs.deploy.outputs.break_glass == 'true' }}
    permissions:
      contents: 'read'
      issues: 'write'
    runs-on: ubuntu-latest
    steps:
      - name: Record that the eligibility proof was bypassed
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          DEPLOY_SHA: ${{ needs.deploy.outputs.admitted_sha }}
          DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }}
          BREAK_GLASS_REASON: ${{ github.event.inputs.break_glass_reason }}
        run: |
          set -euo pipefail
          # A missing release-gate-failure label made gh issue create fail hard,
          # so the audit issue was never filed (see #10301 / #10389). Ensure the
          # label exists first — idempotent, so a concurrent run cannot race it.
          # Only tolerate the explicit "already exists" case; fail loudly on
          # any other label-creation error (permissions, rate limits, etc.).
          label_output="$(gh label create release-gate-failure \
              --repo "$REPO" \
              --description "Release Eligibility gate was bypassed (break-glass audit trail)" \
              --color "b60205" 2>&1)" || {
            if ! grep -qi 'already exists' <<< "$label_output"; then
              echo "::error::Failed to ensure release-gate-failure label: $label_output" >&2
              exit 1
            fi
          }
          gh issue create --repo "$REPO" \
            --title "Release Eligibility bypassed for ${DEPLOY_SHA:0:10} (${DEPLOY_ENVIRONMENT})" \
            --label release-gate-failure \
            --body "$(printf '%s\n' \
              "LLM Gateway deploy ran without a Release Eligibility proof." \
              "" \
              "- SHA: \`${DEPLOY_SHA}\` (verified ancestor of main)" \
              "- Environment: ${DEPLOY_ENVIRONMENT}" \
              "- Operator: @${GITHUB_ACTOR}" \
              "- Reason: ${BREAK_GLASS_REASON}" \
              "- Run: ${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}" \
              "" \
              "The gate being unusable is the defect. Fix the gate, then close this.")"