Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Storage Alerts: Missing cluster label #885

Merged
merged 6 commits into from
Dec 11, 2023
Merged

Storage Alerts: Missing cluster label #885

Show file tree
Hide file tree
Changes from all commits
Commits
Show all changes
6 commits
Select commit Hold shift + click to select a range
de8329f
Storage Alerts: Missing cluster label
jkroepke Nov 23, 2023
b459474
Add cluster label to all tests
jkroepke Nov 24, 2023
6855a07
Add cluster label to all all tests
jkroepke Nov 24, 2023
0667c69
fix tests
jkroepke Nov 24, 2023
4e71813
fix tests
jkroepke Nov 24, 2023
6212c1e
fix tests
jkroepke Nov 24, 2023
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
26 changes: 13 additions & 13 deletions alerts/storage_alerts.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -26,17 +26,17 @@
) < 0.03
and
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is only {{ $value | humanizePercentage }} free.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -52,17 +52,17 @@
kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_available_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.' % $._config,
summary: 'PersistentVolume is filling up.',
},
},
Expand All @@ -76,17 +76,17 @@
) < 0.03
and
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1m',
labels: {
severity: 'critical',
},
annotations: {
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.',
description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} only has {{ $value | humanizePercentage }} free inodes.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -102,17 +102,17 @@
kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0
and
predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1
unless on(namespace, persistentvolumeclaim)
unless on(%(clusterLabel)s, namespace, persistentvolumeclaim)
kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1
||| % $._config,
'for': '1h',
labels: {
severity: 'warning',
},
annotations: {
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.',
description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.' % $._config,
summary: 'PersistentVolumeInodes are filling up.',
},
},
Expand All @@ -126,7 +126,7 @@
severity: 'critical',
},
annotations: {
description: 'The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.',
description: 'The persistent volume {{ $labels.persistentvolume }} on Cluster {{ $labels.%(clusterLabel)s }} has status {{ $labels.phase }}.' % $._config,
summary: 'PersistentVolume is having issues with provisioning.',
},
},
Expand Down
4 changes: 2 additions & 2 deletions rules/node.libsonnet
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -32,8 +32,8 @@
expr: |||
count by (%(clusterLabel)s, node) (
node_cpu_seconds_total{mode="idle",%(nodeExporterSelector)s}
* on (namespace, %(podLabel)s) group_left(node)
topk by(namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
* on (%(clusterLabel)s, namespace, %(podLabel)s) group_left(node)
topk by(%(clusterLabel)s, namespace, %(podLabel)s) (1, node_namespace_pod:kube_pod_info:)
)
||| % $._config,
},
Expand Down
Loading
Loading