GistPin/infrastructure/monitoring/mttr-alerts.yml at main · ghzhost/GistPin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
groups:
  - name: mttr-tracking
    rules:
      # Alert when MTTR (7-day rolling avg) exceeds SLO target of 30 minutes
      - alert: MTTRBreachingTarget
        expr: |
          avg_over_time(
            (
              avg(
                (ALERTS{alertstate="resolved"} - ALERTS{alertstate="firing"})
              ) by (alertname, job)
            )[7d:1h]
          ) > 30
        for: 0m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "MTTR exceeding 30-minute target"
          description: "7-day rolling MTTR for {{ $labels.alertname }} ({{ $labels.job }}) is {{ $value | humanizeDuration }}."

      # Alert on high-severity incident in progress
      - alert: CriticalIncidentActive
        expr: ALERTS{alertstate="firing", severity="critical"} == 1
        for: 5m
        labels:
          severity: critical
          team: platform
        annotations:
          summary: "Critical incident active: {{ $labels.alertname }}"
          description: "Service {{ $labels.job }} has had a critical incident for > 5 min — impacts MTTR."

      # Alert on replica lag (feeds into MTTR for DB incidents)
      - alert: ReplicaLagHigh
        expr: aws_rds_replica_lag_average > 30
        for: 2m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "RDS replica lag > 30s"
          description: "Replica {{ $labels.db_instance_identifier }} lag: {{ $value }}s."

      # Incident frequency spike
      - alert: IncidentFrequencySpike
        expr: |
          increase(ALERTS{alertstate="firing", severity=~"critical|warning"}[1h]) > 5
        for: 0m
        labels:
          severity: warning
          team: platform
        annotations:
          summary: "Incident frequency spike detected"
          description: "More than 5 alerts fired in the last hour — MTTR at risk."