GistPin/infrastructure/monitoring/health-score.yml at main · ghzhost/GistPin · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
groups:
  - name: health-scores
    interval: 60s
    rules:
      - record: health:availability:score
        expr: |
          (avg by (namespace) (kube_deployment_status_replicas_available / kube_deployment_status_replicas) * 100)

      - record: health:latency:score
        expr: |
          clamp_max(
            100 - (histogram_quantile(0.95, rate(http_request_duration_seconds_bucket[5m])) * 100),
            100
          )

      - record: health:error_rate:score
        expr: |
          clamp_max(
            100 - (rate(http_requests_total{status=~"5.."}[5m]) / rate(http_requests_total[5m]) * 1000),
            100
          )

      - record: health:resource_usage:score
        expr: |
          avg by (namespace) (
            100 - (
              (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes) * 100
              +
              (1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) * 100
            ) / 2
          )

      - record: health:overall:score
        expr: |
          (health:availability:score * 0.35)
          + (health:latency:score * 0.25)
          + (health:error_rate:score * 0.25)
          + (health:resource_usage:score * 0.15)

  - name: health-alerts
    rules:
      - alert: HealthScoreDropped
        expr: |
          health:overall:score < 80
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Platform health score dropped to {{ $value }}"

      - alert: HealthScoreCritical
        expr: |
          health:overall:score < 60
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Platform health score critical: {{ $value }}"

      - alert: AvailabilityScoreLow
        expr: |
          health:availability:score < 90
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Availability score low: {{ $value }}"