Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage Alerts: Missing cluster label #885

Merged
merged 6 commits into from
Dec 11, 2023
Merged

Storage Alerts: Missing cluster label #885

Show file tree
Hide file tree
Changes from 1 commit
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
de8329f
Storage Alerts: Missing cluster label
jkroepke Nov 23, 2023
b459474
Add cluster label to all tests
jkroepke Nov 24, 2023
6855a07
Add cluster label to all all tests
jkroepke Nov 24, 2023
0667c69
fix tests
jkroepke Nov 24, 2023
4e71813
fix tests
jkroepke Nov 24, 2023
6212c1e
fix tests
jkroepke Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions alerts/storage_alerts.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@
) < 0.03
and
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is only {{ $value | humanizePercentage }} free.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -52,17 +52,17 @@
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_available_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -76,17 +76,17 @@
) < 0.03
and
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} only has {{ $value | humanizePercentage }} free inodes.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -102,17 +102,17 @@
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -126,7 +126,7 @@
severity: 'critical',
},
annotations: {
description: 'The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.',
description: 'The persistent volume {{ $labels.persistentvolume }} on Cluster {{ $labels.%(clusterLabel)s }} has status {{ $labels.phase }}.' % $._config,
summary: 'PersistentVolume is having issues with provisioning.',
},
},
Expand Down
4 changes: 2 additions & 2 deletions rules/node.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
expr: |||
count by (%(clusterLabel)s, node) (
node_cpu_seconds_total{mode="idle",%(nodeExporterSelector)s}
* on (namespace, %(podLabel)s) group_left(node)
topk by(namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
* on (%(clusterLabel)s, namespace, %(podLabel)s) group_left(node)
topk by(%(clusterLabel)s, namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
)
||| % $._config,
},
Expand Down
18 changes: 9 additions & 9 deletions tests.yaml
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -33,7 +33,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolume is filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.562% free.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster is only 1.562% free.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup

# Don't alert when PVC access_mode is ReadOnlyMany
Expand Down Expand Up @@ -119,7 +119,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolume is filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.294% free.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster is only 1.294% free.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup

- interval: 1m
Expand All @@ -143,7 +143,7 @@ tests:
severity: warning
exp_annotations:
summary: "PersistentVolume is filling up."
description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring is expected to fill up within four days. Currently 1.263% is available.'
description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring on Cluster is expected to fill up within four days. Currently 1.263% is available.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup
- exp_labels:
job: kubelet
Expand All @@ -152,7 +152,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolume is filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring is only 1.263% free.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster is only 1.263% free.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumefillingup

# Block volume mounts can report 0 for the kubelet_volume_stats_used_bytes metric but it shouldn't trigger the KubePersistentVolumeFillingUp alert.
Expand Down Expand Up @@ -228,7 +228,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolumeInodes are filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.562% free inodes.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster only has 1.562% free inodes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup

# Don't alert when PVC access_mode is ReadOnlyMany
Expand Down Expand Up @@ -314,7 +314,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolumeInodes are filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.294% free inodes.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster only has 1.294% free inodes.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup

- interval: 1m
Expand All @@ -338,7 +338,7 @@ tests:
severity: warning
exp_annotations:
summary: "PersistentVolumeInodes are filling up."
description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring is expected to run out of inodes within four days. Currently 1.263% of its inodes are free.'
description: 'Based on recent sampling, the PersistentVolume claimed by somepvc in Namespace monitoring on Cluster is expected to run out of inodes within four days. Currently 1.263% of its inodes are free.'
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup
- exp_labels:
job: kubelet
Expand All @@ -347,7 +347,7 @@ tests:
severity: critical
exp_annotations:
summary: "PersistentVolumeInodes are filling up."
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring only has 1.263% free inodes.'
description: 'The PersistentVolume claimed by somepvc in Namespace monitoring on Cluster only has 1.263% free inodes.'
jkroepke marked this conversation as resolved.
Show resolved Hide resolved
runbook_url: https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepersistentvolumeinodesfillingup

# Block volume mounts can report 0 for the kubelet_volume_stats_inodes_used metric but it shouldn't trigger the KubePersistentVolumeInodesFillingUp alert.
Expand Down Expand Up @@ -1137,7 +1137,7 @@ tests:
runbook_url: "https://github.com/kubernetes-monitoring/kubernetes-mixin/tree/master/runbook.md#alert-name-kubepodcrashlooping"
summary: "Pod is crash looping."
- eval_time: 20m
alertname: KubePodCrashLooping # alert fired for a period of 5 minutes after resolution because the alert looks back at the last 5 minutes of data and the range vector doesn't take stale samples into account
alertname: KubePodCrashLooping # alert fired for a period of 5 minutes after resolution because the alert looks back at the last 5 minutes of data and the range vector doesn't take stale samples into account
exp_alerts:
- exp_labels:
severity: "warning"
Expand Down
Loading