Atlas-Workforce-System/alert.rules.yml at main · Senthil455/Atlas-Workforce-System · GitHub

1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
94
95
96
97
98
99
100
101
102
103
104
105
106
107
108
109
110
111
112
groups:
  - name: atlas_service_health
    rules:
      - alert: ServiceDown
        expr: up == 0
        for: 30s
        labels:
          severity: critical
        annotations:
          summary: "Service {{ $labels.job }} is down"
          description: "{{ $labels.job }} has been unreachable for more than 30 seconds."

      - alert: HighErrorRate
        expr: rate(atlas_http_requests_total{status_code=~"5.."}[5m]) > 0.05
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "High error rate on {{ $labels.job }}"
          description: "Error rate on {{ $labels.job }} is {{ $value | humanizePercentage }}"

      - alert: HighLatencyP99
        expr: histogram_quantile(0.99, sum by (job, le) (rate(atlas_http_request_duration_seconds_bucket[5m]))) > 1.0
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "High p99 latency on {{ $labels.job }}"
          description: "p99 latency on {{ $labels.job }} is {{ $value | humanizeDuration }}"

      - alert: HighLatencyP99Critical
        expr: histogram_quantile(0.99, sum by (job, le) (rate(atlas_http_request_duration_seconds_bucket[5m]))) > 2.5
        for: 5m
        labels:
          severity: critical
        annotations:
          summary: "Critical p99 latency on {{ $labels.job }}"
          description: "p99 latency on {{ $labels.job }} is {{ $value | humanizeDuration }}"

      - alert: HighInProgressRequests
        expr: sum by (job) (atlas_http_requests_in_progress) > 100
        for: 1m
        labels:
          severity: warning
        annotations:
          summary: "High in-flight requests on {{ $labels.job }}"
          description: "{{ $labels.job }} has {{ $value }} requests in progress"

  - name: atlas_infrastructure
    rules:
      - alert: DiskSpaceWarning
        expr: (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) < 0.1
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Disk space low on {{ $labels.device }}"
          description: "Disk space on {{ $labels.device }} ({{ $labels.mountpoint }}) is {{ $value | humanizePercentage }} available"

      - alert: DiskSpaceCritical
        expr: (node_filesystem_avail_bytes{fstype!="tmpfs"} / node_filesystem_size_bytes{fstype!="tmpfs"}) < 0.05
        for: 1m
        labels:
          severity: critical
        annotations:
          summary: "Disk space critical on {{ $labels.device }}"
          description: "Disk space on {{ $labels.device }} ({{ $labels.mountpoint }}) is {{ $value | humanizePercentage }} available"

      - alert: NodeMemoryUsageWarning
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.9
        for: 5m
        labels:
          severity: warning
        annotations:
          summary: "Node memory usage above 90%"
          description: "Memory usage on {{ $labels.instance }} is {{ $value | humanizePercentage }}"

      - alert: NodeMemoryUsageCritical
        expr: (1 - (node_memory_MemAvailable_bytes / node_memory_MemTotal_bytes)) > 0.95
        for: 2m
        labels:
          severity: critical
        annotations:
          summary: "Node memory usage above 95%"
          description: "Memory usage on {{ $labels.instance }} is {{ $value | humanizePercentage }}"

      - alert: NodeCPUUsageHigh
        expr: (1 - avg by (instance) (rate(node_cpu_seconds_total{mode="idle"}[5m]))) > 0.8
        for: 10m
        labels:
          severity: warning
        annotations:
          summary: "CPU usage above 80% on {{ $labels.instance }}"
          description: "CPU usage on {{ $labels.instance }} is {{ $value | humanizePercentage }}"

      - alert: TLSCertExpiryWarning
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 30
        for: 1h
        labels:
          severity: warning
        annotations:
          summary: "TLS certificate expires in {{ $value | humanizeDuration }}"
          description: "TLS certificate for {{ $labels.target }} expires in {{ $value | humanizeDuration }}"

      - alert: TLSCertExpiryCritical
        expr: probe_ssl_earliest_cert_expiry - time() < 86400 * 7
        for: 1h
        labels:
          severity: critical
        annotations:
          summary: "TLS certificate expires in {{ $value | humanizeDuration }}"
          description: "TLS certificate for {{ $labels.target }} expires in less than 7 days"