GistPin/infrastructure/monitoring/tf-traces.yml at main · ghzhost/GistPin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
groups:
  - name: tf-tracing
    interval: 30s
    rules:
      - alert: TfTraceExporterDown
        expr: up{job="terraform-tracing"} == 0
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Terraform trace exporter is down"
          description: "The trace exporter for Terraform operations is unreachable for over 2 minutes"

      - alert: TfTraceExportHighFailureRate
        expr: rate(terraform_trace_export_failures_total[5m]) > 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High Terraform trace export failure rate"
          description: "Export failure rate is {{ $value | humanizePercentage }} over 5 minutes"

      - alert: TfTraceExportLatencySpike
        expr: histogram_quantile(0.95, rate(terraform_trace_export_duration_seconds_bucket[5m])) > 5
        for: 2m
        labels:
          severity: warning
        annotations:
          summary: "Terraform trace export latency spike"
          description: "p95 latency is {{ $value }}s, expected < 5s"

      - alert: TfTraceExportQueueBacklog
        expr: terraform_trace_queue_size > 1000
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Terraform trace queue backlog growing"
          description: "{{ $value }} traces pending export, possible bottleneck"

      - alert: TfTraceBackendUnavailable
        expr: terraform_trace_backend_health{status="unavailable"} == 1
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Tracing backend unavailable"
          description: "The OpenTelemetry collector backend is unreachable from Terraform tracing"

      - alert: TfTraceSpanDropped
        expr: rate(terraform_trace_spans_dropped_total[5m]) > 0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Terraform trace spans being dropped"
          description: "{{ $value }} spans/sec dropped due to buffer overflow"