StellarRouter/metrics/alerts.yml at main · ghzhost/StellarRouter · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
groups:
  - name: stellar-router.rules
    rules:
      - alert: RouterMiddlewareCircuitBreakerOpen
        expr: |
          sum by (contract, route) (router_middleware_circuit_open{job!=""}) > 0
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "Circuit breaker open for route {{ $labels.route }}"
          description: "Circuit breaker is open on route {{ $labels.route }} for contract {{ $labels.contract }}. Investigate upstream failures or increase tolerance."

      - alert: RouterMiddlewareHighFailureRate
        expr: |
          increase(router_middleware_failure_count[5m]) > 10
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "High failure count on route"
          description: "Route {{ $labels.route }} on contract {{ $labels.contract }} has high consecutive failures (increase > 10 in 5m). Consider throttling traffic or investigating downstream services."

      - alert: RouterMiddlewareHighErrorRate
        expr: |
          increase(router_scrape_errors_total[5m]) > 0
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "Scrape errors detected for router exporter"
          description: "The exporter has observed scrape errors for router contracts in the last 10m. Check RPC connectivity and contract availability."

      - alert: RouterMiddlewareRateLimitViolations
        expr: |
          increase(router_middleware_total_calls[5m]) > 10000
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "High request volume detected"
          description: "Total pre_call invocations increased by >10k in 5m for contract {{ $labels.contract }}, which may indicate abuse or traffic spike."
  - name: stellar-router-alerts
    interval: 30s
    rules:
      # Circuit Breaker Alerts
      - alert: CircuitBreakerOpen
        expr: router_middleware_circuit_open == 1
        for: 1m
        labels:
          severity: warning
          component: middleware
        annotations:
          summary: "Circuit breaker open for route {{ $labels.route }}"
          description: "Circuit breaker has been open on route {{ $labels.route }} for more than 1 minute. This suggests repeated failures in the routed contract."

      - alert: CircuitBreakerStuck
        expr: router_middleware_circuit_open == 1
        for: 5m
        labels:
          severity: critical
          component: middleware
        annotations:
          summary: "Circuit breaker stuck open on route {{ $labels.route }}"
          description: "Circuit breaker has been open on route {{ $labels.route }} for more than 5 minutes. Manual intervention may be required to reset the circuit."

      # Failure Rate Alerts
      - alert: HighConsecutiveFailures
        expr: router_middleware_failure_count > 10
        for: 2m
        labels:
          severity: warning
          component: middleware
        annotations:
          summary: "High consecutive failures on route {{ $labels.route }}"
          description: "Route {{ $labels.route }} has {{ $value }} consecutive failures. Circuit breaker may trigger soon."

      - alert: CriticalFailureCount
        expr: router_middleware_failure_count > 20
        for: 1m
        labels:
          severity: critical
          component: middleware
        annotations:
          summary: "Critical failure count on route {{ $labels.route }}"
          description: "Route {{ $labels.route }} has {{ $value }} consecutive failures. Immediate investigation required."

      # Route Pause Alerts
      - alert: RoutePaused
        expr: router_core_route_paused == 1
        for: 5m
        labels:
          severity: warning
          component: core
        annotations:
          summary: "Route {{ $labels.route }} is paused"
          description: "Route {{ $labels.route }} has been paused for more than 5 minutes. Clients cannot access this route."

      - alert: GlobalRouterPaused
        expr: router_core_paused == 1
        for: 1m
        labels:
          severity: critical
          component: core
        annotations:
          summary: "Global router pause activated"
          description: "The stellar-router has been globally paused. All routing is blocked. This may indicate an incident or maintenance."

      # Rate Limit Alerts
      - alert: RateLimitApproaching
        expr: |
          (rate(router_middleware_total_calls[5m]) / 60) > 0.8 *
          scalar(count(router_middleware_total_calls) by (route))
        for: 2m
        labels:
          severity: warning
          component: middleware
        annotations:
          summary: "Rate limit approaching for route {{ $labels.route }}"
          description: "Route {{ $labels.route }} is at 80% of its configured rate limit. Requests may start being rejected soon."

      # Scraper Health Alerts
      - alert: MetricsScraperFailing
        expr: increase(router_scrape_errors_total[5m]) > 5
        for: 2m
        labels:
          severity: warning
          component: exporter
        annotations:
          summary: "Metrics scraper failing for contract {{ $labels.contract }}"
          description: "Contract {{ $labels.contract }} scrape has failed {{ $value }} times in the last 5 minutes."

      - alert: MetricsExporterDown
        expr: router_up == 0
        for: 1m
        labels:
          severity: critical
          component: exporter
        annotations:
          summary: "Metrics exporter is down"
          description: "The stellar-router metrics exporter has failed to scrape contract state. Metrics are stale."

      # Scrape Duration Alerts
      - alert: HighScrapeDuration
        expr: histogram_quantile(0.95, router_scrape_duration_seconds_bucket) > 5
        for: 2m
        labels:
          severity: warning
          component: exporter
        annotations:
          summary: "High p95 scrape latency for contract {{ $labels.contract }}"
          description: "P95 scrape latency for {{ $labels.contract }} is {{ $value }}s. RPC endpoint may be slow or overloaded."

      # No Activity Alerts
      - alert: NoRoutingActivity
        expr: increase(router_core_total_routed[10m]) == 0
        for: 10m
        labels:
          severity: info
          component: core
        annotations:
          summary: "No routing activity detected"
          description: "No routes have been resolved in the last 10 minutes. This may indicate a network issue or lack of traffic."

      # Registry Alerts
      - alert: RegistryEmpty
        expr: router_registry_total_names == 0
        for: 5m
        labels:
          severity: warning
          component: registry
        annotations:
          summary: "Registry contains no entries"
          description: "The router-registry is empty. No routes can be registered or resolved."