gcp_backend.yml

name: Deploy Backend to Cloud RUN

on:
  workflow_dispatch:
    inputs:
      environment:
        description: 'Select the environment to deploy to'
        required: true
        default: 'development'
        type: choice
        options:
          - development
          - prod
      release_sha:
        description: 'Exact main SHA with a successful Release Eligibility proof (deploy mode only)'
        required: false
        default: ''
      release_version:
        description: 'Release version (optional)'
        required: false
        default: ''
      mode:
        description: 'Workflow mode'
        required: true
        default: 'deploy'
        type: choice
        options:
          - deploy
          - repair-traffic-only
      deploy_targets:
        description: 'Deploy all backend surfaces or Cloud Run only'
        required: true
        default: 'cloud-run-only'
        type: choice
        options:
          - all
          - cloud-run-only
      deploy_gateway:
        description: 'Development only: deploy the LLM gateway binary with the backend stack (requires deploy_targets=all)'
        required: true
        default: false
        type: boolean
      # Break-glass: the eligibility proof requires a *successful* Release
      # Eligibility push-run on this exact SHA. A flaky or red check therefore
      # poisons that SHA permanently -- rerunning does not retroactively make
      # the original run succeed, so the only other recovery is pushing a new
      # commit to main and waiting for a full green run. That is unacceptable
      # for a hotfix, so this hatch exists.
      #
      # It does NOT relax the property that matters: the SHA must still be an
      # ancestor of main, so unreviewed code can never reach production.
      skip_eligibility_proof:
        description: 'Break-glass: deploy without a Release Eligibility proof (still requires a merged main SHA)'
        required: false
        default: false
        type: boolean
      break_glass_confirm:
        description: 'Type deploy-without-proof when skip_eligibility_proof is set'
        required: false
        default: ''
      break_glass_reason:
        description: 'Why the eligibility proof is unavailable (required with skip_eligibility_proof)'
        required: false
        default: ''

# Cloud Run revision promotion/traffic repair and the shared backend GKE resources
# are one environment-scoped mutation domain across the backend deploy workflows.
concurrency:
  group: deploy-backend-stack-${{ github.event.inputs.environment }}
  cancel-in-progress: false

env:
  SERVICE: backend
  REGION: us-central1
  # Development keeps its existing tagged no-traffic candidate probe. Production
  # instead validates the exact serving release after its rollback-safe shift.
  TRANSCRIPTION_CANDIDATE_TAG: stt-gate-${{ github.run_id }}-${{ github.run_attempt }}
  CLOUD_RUN_VPC_NETWORK: ${{ vars.CLOUD_RUN_VPC_NETWORK }}
  CLOUD_RUN_VPC_SUBNET: ${{ vars.CLOUD_RUN_VPC_SUBNET }}
  CONVERSATION_SUMMARIZED_APP_IDS: ${{ vars.CONVERSATION_SUMMARIZED_APP_IDS }}
  GKE_CLUSTER: ${{ vars.GKE_CLUSTER }}
  GOOGLE_CLIENT_ID: ${{ vars.GOOGLE_CLIENT_ID }}
  LLM_GATEWAY_GSA: ${{ vars.LLM_GATEWAY_GSA }}
  MCP_AUTHORIZATION_SERVER_URL: ${{ vars.MCP_AUTHORIZATION_SERVER_URL }}
  MCP_OAUTH_CHATGPT_CLIENT_ID: ${{ vars.MCP_OAUTH_CHATGPT_CLIENT_ID }}
  MCP_OAUTH_CHATGPT_REDIRECT_URIS: ${{ vars.MCP_OAUTH_CHATGPT_REDIRECT_URIS }}
  MCP_OAUTH_CLAUDE_CLIENT_ID: ${{ vars.MCP_OAUTH_CLAUDE_CLIENT_ID }}
  MCP_OAUTH_CLAUDE_CLIENT_NAME: ${{ vars.MCP_OAUTH_CLAUDE_CLIENT_NAME }}
  MCP_OAUTH_CLAUDE_REDIRECT_URIS: ${{ vars.MCP_OAUTH_CLAUDE_REDIRECT_URIS }}
  MCP_OAUTH_PUBLIC_CLIENT_ID: ${{ vars.MCP_OAUTH_PUBLIC_CLIENT_ID }}
  MCP_OAUTH_PUBLIC_REDIRECT_URIS: ${{ vars.MCP_OAUTH_PUBLIC_REDIRECT_URIS }}
  MCP_RESOURCE_URL: ${{ vars.MCP_RESOURCE_URL }}
  PUBLIC_SHARED_CONVERSATION_CHAT_FRONTEND_AUDIENCE: ${{ vars.PUBLIC_SHARED_CONVERSATION_CHAT_FRONTEND_AUDIENCE }}
  PUBLIC_SHARED_CONVERSATION_CHAT_FRONTEND_INVOKER_SA: ${{ vars.PUBLIC_SHARED_CONVERSATION_CHAT_FRONTEND_INVOKER_SA }}
  RAPID_API_HOST: ${{ vars.RAPID_API_HOST }}
  REDIS_DB_HOST: ${{ vars.REDIS_DB_HOST }}
  STT_PRERECORDED_MODEL: ${{ vars.STT_PRERECORDED_MODEL }}
  STT_SERVICE_MODELS: ${{ vars.STT_SERVICE_MODELS }}
  SYNC_LEDGER_FENCE_MODE: ${{ vars.SYNC_LEDGER_FENCE_MODE || 'legacy' }}
  TYPESENSE_HOST: ${{ vars.TYPESENSE_HOST }}
  TWILIO_ACCOUNT_SID: ${{ vars.TWILIO_ACCOUNT_SID }}
  TWILIO_API_KEY_SID: ${{ vars.TWILIO_API_KEY_SID }}
  TWILIO_TWIML_APP_SID: ${{ vars.TWILIO_TWIML_APP_SID }}
  X_OAUTH_CLIENT_ID: ${{ vars.X_OAUTH_CLIENT_ID }}
  X_OAUTH_REDIRECT_URI: ${{ vars.X_OAUTH_REDIRECT_URI }}

jobs:
  validate-production-boundary:
    if: >-
      github.event.inputs.mode == 'deploy'
    runs-on: ubuntu-latest
    steps:
      - name: Reject unsupported production deployment boundary before side effects
        env:
          DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }}
          DEPLOY_TARGETS: ${{ github.event.inputs.deploy_targets }}
        run: |
          set -euo pipefail
          if [[ "$DEPLOY_ENVIRONMENT" != "development" && "$DEPLOY_ENVIRONMENT" != "prod" ]]; then
            echo "Invalid environment: $DEPLOY_ENVIRONMENT. Must be 'development' or 'prod'." >&2
            exit 1
          fi
          if [[ "$DEPLOY_TARGETS" != "all" && "$DEPLOY_TARGETS" != "cloud-run-only" ]]; then
            echo "Invalid deploy_targets: $DEPLOY_TARGETS. Must be 'all' or 'cloud-run-only'." >&2
            exit 1
          fi
          if [[ "$DEPLOY_ENVIRONMENT" == "prod" && "$DEPLOY_TARGETS" == "all" ]]; then
            echo "environment=prod, deploy_targets=all is unsupported; use cloud-run-only." >&2
            exit 1
          fi
          if [[ "${{ github.event.inputs.deploy_gateway }}" == "true" && "$DEPLOY_ENVIRONMENT" == "prod" ]]; then
            echo "environment=prod, deploy_gateway=true is unsupported; use the standalone manual LLM gateway workflow." >&2
            exit 1
          fi
          if [[ "${{ github.event.inputs.deploy_gateway }}" == "true" && "$DEPLOY_TARGETS" != "all" ]]; then
            echo "deploy_gateway=true requires deploy_targets=all." >&2
            exit 1
          fi

  repair-traffic:
    if: >-
      github.ref == 'refs/heads/main' &&
      github.event.inputs.mode == 'repair-traffic-only'
    environment: ${{ github.event.inputs.environment == 'prod' && 'prod' || 'development' }}
    permissions:
      contents: 'read'
      id-token: 'write'
    runs-on: ubuntu-latest
    steps:
      - name: Validate Environment Input
        run: |
          if [[ "${{ github.event.inputs.environment }}" != "development" && "${{ github.event.inputs.environment }}" != "prod" ]]; then
            echo "Invalid environment: ${{ github.event.inputs.environment }}. Must be 'development' or 'prod'."
            exit 1
          fi

      - name: Checkout
        uses: actions/checkout@v7
        with:
          # Traffic repair changes no source-derived runtime state. It stays a
          # separate recovery path and does not consume a release proof.
          ref: main

      - name: Google Auth
        id: auth
        uses: 'google-github-actions/auth@v3'
        with:
          credentials_json: ${{ secrets.GCP_CREDENTIALS }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v3

      - name: Restore Cloud Run traffic spec to currently serving revisions
        run: |
          python3 backend/scripts/repair_cloud_run_traffic.py \
            --project=${{ vars.GCP_PROJECT_ID }} \
            --region=${{ env.REGION }} \
            --repair

      - name: Cloud Run deploy status report
        if: always()
        run: |
          python3 backend/scripts/deploy_status_report.py \
            --env ${{ vars.ENV }} \
            --project ${{ vars.GCP_PROJECT_ID }} \
            --region ${{ env.REGION }} \
            --include-cloud-run \
            --cloud-run-service backend \
            --cloud-run-service backend-sync \
            --cloud-run-service backend-sync-backfill \
            --cloud-run-service backend-integration || true

  firestore_readiness:
    needs: validate-production-boundary
    if: >-
      github.ref == 'refs/heads/main' &&
      github.event.inputs.mode == 'deploy'
    environment: ${{ github.event.inputs.environment == 'prod' && 'prod' || 'development' }}
    permissions:
      actions: 'read'
      contents: 'read'
    runs-on: ubuntu-latest
    outputs:
      admitted_sha: ${{ steps.admitted_source.outputs.admitted_sha }}
      break_glass: ${{ steps.admitted_source.outputs.break_glass }}
    steps:
      - name: Validate Firestore readiness inputs
        env:
          DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }}
        run: |
          set -euo pipefail
          if [[ "$DEPLOY_ENVIRONMENT" != "development" && "$DEPLOY_ENVIRONMENT" != "prod" ]]; then
            echo "Invalid environment: $DEPLOY_ENVIRONMENT. Must be 'development' or 'prod'."
            exit 1
          fi
      - name: Require read-only Firestore credentials
        env:
          GCP_FIRESTORE_READONLY_CREDENTIALS: ${{ secrets.GCP_FIRESTORE_READONLY_CREDENTIALS }}
        run: |
          if [ -z "$GCP_FIRESTORE_READONLY_CREDENTIALS" ]; then
            echo "::error title=Missing Firestore read-only credentials::Set GCP_FIRESTORE_READONLY_CREDENTIALS in the selected environment before deploying."
            exit 1
          fi

      - name: Checkout current main for source admission
        uses: actions/checkout@v7
        with:
          ref: main
          fetch-depth: 0

      - name: Verify exact admitted main source
        id: admitted_source
        env:
          DEPLOY_SHA: ${{ github.event.inputs.release_sha }}
          GH_TOKEN: ${{ github.token }}
          SKIP_PROOF: ${{ github.event.inputs.skip_eligibility_proof }}
          BREAK_GLASS_CONFIRM: ${{ github.event.inputs.break_glass_confirm }}
          BREAK_GLASS_REASON: ${{ github.event.inputs.break_glass_reason }}
        run: |
          set -euo pipefail
          if [[ ! "$DEPLOY_SHA" =~ ^[0-9a-f]{40}$ || "$DEPLOY_SHA" == "0000000000000000000000000000000000000000" ]]; then
            echo "ERROR: release_sha must be one non-zero, lowercase, full commit SHA."
            exit 1
          fi
          git fetch --no-tags origin +refs/heads/main:refs/remotes/origin/main
          main_sha="$(git rev-parse --verify 'origin/main^{commit}')"
          git cat-file -e "${DEPLOY_SHA}^{commit}"

          # Always enforced, break-glass or not: only merged main code deploys.
          if ! git merge-base --is-ancestor "$DEPLOY_SHA" "$main_sha"; then
            echo "ERROR: release_sha is not an ancestor of origin/main; only merged code may deploy."
            exit 1
          fi

          if [[ "${SKIP_PROOF:-false}" == "true" ]]; then
            if [[ "$BREAK_GLASS_CONFIRM" != "deploy-without-proof" ]]; then
              echo "ERROR: skip_eligibility_proof requires break_glass_confirm=deploy-without-proof." >&2
              exit 1
            fi
            if [[ -z "${BREAK_GLASS_REASON// }" ]]; then
              echo "ERROR: skip_eligibility_proof requires a non-empty break_glass_reason." >&2
              exit 1
            fi
            echo "::warning::Release Eligibility proof skipped for ${DEPLOY_SHA}: ${BREAK_GLASS_REASON}"
            printf 'break_glass=true\n' >> "$GITHUB_OUTPUT"
          else
            proof_path="${RUNNER_TEMP}/release-eligibility-${DEPLOY_SHA}.json"
            gh api -H "Accept: application/vnd.github+json" \
              "repos/${GITHUB_REPOSITORY}/actions/workflows/release-eligibility.yml/runs?event=push&branch=main&status=completed&head_sha=${DEPLOY_SHA}&per_page=100" \
              > "$proof_path"
            python3 .github/scripts/verify_backend_release_admission.py \
              --sha "$DEPLOY_SHA" \
              --repository "$GITHUB_REPOSITORY" \
              --workflow-runs "$proof_path"
            printf 'break_glass=false\n' >> "$GITHUB_OUTPUT"
          fi
          printf 'admitted_sha=%s\n' "$DEPLOY_SHA" >> "$GITHUB_OUTPUT"

      - name: Checkout admitted Firestore source
        uses: actions/checkout@v7
        with:
          ref: ${{ steps.admitted_source.outputs.admitted_sha }}

      - name: Google Auth for read-only Firestore inventory
        uses: 'google-github-actions/auth@v3'
        with:
          credentials_json: ${{ secrets.GCP_FIRESTORE_READONLY_CREDENTIALS }}

      - name: Set up gcloud
        uses: google-github-actions/setup-gcloud@v3

      - name: Verify serving Firestore indexes
        id: firestore_readiness
        env:
          FIRESTORE_PROPOSAL_PATH: ${{ runner.temp }}/firestore-schema-proposal-${{ github.run_id }}-${{ github.run_attempt }}.json
          FIRESTORE_SOURCE_COMMIT: ${{ steps.admitted_source.outputs.admitted_sha }}
        run: |
          python3 backend/scripts/reconcile_firestore_indexes.py \
            --project "${{ vars.RUNTIME_GCP_PROJECT_ID }}" \
            --check-only \
            --proposal-output "$FIRESTORE_PROPOSAL_PATH" \
            --source-commit "$FIRESTORE_SOURCE_COMMIT" \
            --proposal-ttl-seconds 3600

      - name: Validate blocked Firestore schema proposal
        id: validate_firestore_proposal
        if: ${{ failure() && steps.firestore_readiness.outcome == 'failure' }}
        env:
          FIRESTORE_PROPOSAL_PATH: ${{ runner.temp }}/firestore-schema-proposal-${{ github.run_id }}-${{ github.run_attempt }}.json
          FIRESTORE_SOURCE_COMMIT: ${{ steps.admitted_source.outputs.admitted_sha }}
        run: |
          python3 backend/scripts/reconcile_firestore_indexes.py \
            --project "${{ vars.RUNTIME_GCP_PROJECT_ID }}" \
            --validate-proposal "$FIRESTORE_PROPOSAL_PATH" \
            --source-commit "$FIRESTORE_SOURCE_COMMIT" \
            --proposal-ttl-seconds 3600

      - name: Preserve blocked Firestore schema proposal
        if: ${{ failure() && steps.firestore_readiness.outcome == 'failure' && steps.validate_firestore_proposal.outcome == 'success' }}
        uses: actions/upload-artifact@v7
        env:
          FIRESTORE_PROPOSAL_PATH: ${{ runner.temp }}/firestore-schema-proposal-${{ github.run_id }}-${{ github.run_attempt }}.json
        with:
          name: firestore-schema-proposal-${{ github.run_id }}-${{ github.run_attempt }}
          path: ${{ env.FIRESTORE_PROPOSAL_PATH }}
          if-no-files-found: error
          retention-days: 1

  # Kept out of firestore_readiness on purpose: that job is a least-privilege
  # release-proof boundary limited to actions:read/contents:read, so the audit
  # side effect gets its own job rather than widening it.
  record_break_glass:
    needs: firestore_readiness
    if: ${{ needs.firestore_readiness.outputs.break_glass == 'true' }}
    permissions:
      contents: 'read'
      issues: 'write'
    runs-on: ubuntu-latest
    steps:
      - name: Record that the eligibility proof was bypassed
        env:
          GH_TOKEN: ${{ github.token }}
          REPO: ${{ github.repository }}
          DEPLOY_SHA: ${{ needs.firestore_readiness.outputs.admitted_sha }}
          DEPLOY_ENVIRONMENT: ${{ github.event.inputs.environment }}
          BREAK_GLASS_REASON: ${{ github.event.inputs.break_glass_reason }}
        run: |
          set -euo pipefail
          # A missing release-gate-failure label made gh issue create fail hard,
          # so the audit issue was never filed (see #10301 / #10389). Ensure the
          # label exists first — idempotent, so a concurrent run cannot race it.
          # Only tolerate the explicit "already exists" case; fail loudly on
          # any other label-creation error (permissions, rate limits, etc.).
          label_output="$(gh label create release-gate-failure \
              --repo "$REPO" \
              --description "Release Eligibility gate was bypassed (break-glass audit trail)" \
              --color "b60205" 2>&1)" || {
            if ! grep -qi 'already exists' <<< "$label_output"; then
              echo "::error::Failed to ensure release-gate-failure label: $label_output" >&2
              exit 1
            fi
          }
          gh issue create --repo "$REPO" \
            --title "Release Eligibility bypassed for ${DEPLOY_SHA:0:10} (${DEPLOY_ENVIRONMENT})" \
            --label release-gate-failure \
            --body "$(printf '%s\n' \
              "Backend deploy ran without a Release Eligibility proof." \
              "" \
              "- SHA: \`${DEPLOY_SHA}\` (verified ancestor of main)" \
              "- Environment: ${DEPLOY_ENVIRONMENT}" \
              "- Operator: @${GITHUB_ACTOR}" \
              "- Reason: ${BREAK_GLASS_REASON}" \
              "- Run: ${GITHUB_SERVER_URL}/${REPO}/actions/runs/${GITHUB_RUN_ID}" \
              "" \
              "The gate being unusable is the defect. Fix the gate, then close this.")"

  deploy:
    needs: [validate-production-boundary, firestore_readiness, record_break_glass]
    if: >-
      always() &&
      github.ref == 'refs/heads/main' &&
      github.event.inputs.mode == 'deploy' &&
      needs.validate-production-boundary.result == 'success' &&
      needs.firestore_readiness.result == 'success' &&
      (needs.record_break_glass.result == 'success' || needs.record_break_glass.result == 'skipped')
    environment: ${{ github.event.inputs.environment == 'prod' && 'prod' || 'development' }}
    permissions:
      contents: 'read'
      id-token: 'write'

    runs-on: ubuntu-latest
    steps:
      - name: Checkout workflow dispatch surface for composite action
        uses: actions/checkout@v7

      - name: Deploy backend stack
        uses: ./.github/actions/deploy-backend-stack
        with:
          deploy_profile: manual
          admitted_sha: ${{ needs.firestore_readiness.outputs.admitted_sha }}
          environment: ${{ github.event.inputs.environment }}
          deploy_targets: ${{ github.event.inputs.deploy_targets }}
          deploy_gateway: ${{ github.event.inputs.deploy_gateway }}
          release_version: ${{ github.event.inputs.release_version }}
          gcp_credentials: ${{ secrets.GCP_CREDENTIALS }}
          project_id: ${{ vars.GCP_PROJECT_ID }}
          runtime_gcp_project_id: ${{ vars.RUNTIME_GCP_PROJECT_ID }}
          # Unset by default, which keeps the existing signing behaviour.
          firebase_probe_signer_service_account: ${{ vars.FIREBASE_PROBE_SIGNER_SERVICE_ACCOUNT }}
          # The known-audio gate runs only for development, where the Firebase
          # auth project (based-hardware) is not this lane's deploy project.
          # This environment secret is the production Firebase signer key; the
          # token minter refuses it if its project does not match.
          firebase_probe_signer_credentials: ${{ github.event.inputs.environment == 'development' && secrets.GCP_SERVICE_ACCOUNT || '' }}
          runtime_env: ${{ vars.ENV }}
          region: ${{ env.REGION }}
          service: ${{ env.SERVICE }}
          transcription_candidate_tag: ${{ github.event.inputs.environment == 'development' && env.TRANSCRIPTION_CANDIDATE_TAG || '' }}
          sync_backfill_budget_alerts: ${{ github.event.inputs.environment == 'prod' && 'true' || 'false' }}
          sync_backfill_alert_channels: ${{ vars.SYNC_BACKFILL_ALERT_NOTIFICATION_CHANNELS }}
          telegram_bot_token: ${{ secrets.TELEGRAM_BOT_TOKEN }}
          telegram_chat_id: ${{ secrets.TELEGRAM_CHAT_ID }}