diff --git a/alerts/storage_alerts.libsonnet b/alerts/storage_alerts.libsonnet index ce028683f..e70c3b404 100644 --- a/alerts/storage_alerts.libsonnet +++ b/alerts/storage_alerts.libsonnet @@ -26,9 +26,9 @@ ) < 0.03 and kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1 ||| % $._config, 'for': '1m', @@ -36,7 +36,7 @@ severity: 'critical', }, annotations: { - description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is only {{ $value | humanizePercentage }} free.', + description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is only {{ $value | humanizePercentage }} free.', summary: 'PersistentVolume is filling up.', }, }, @@ -52,9 +52,9 @@ kubelet_volume_stats_used_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0 and predict_linear(kubelet_volume_stats_available_bytes{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1 ||| % $._config, 'for': '1h', @@ -62,7 +62,7 @@ severity: 'warning', }, annotations: { - description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.', + description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to fill up within four days. Currently {{ $value | humanizePercentage }} is available.', summary: 'PersistentVolume is filling up.', }, }, @@ -76,9 +76,9 @@ ) < 0.03 and kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1 ||| % $._config, 'for': '1m', @@ -86,7 +86,7 @@ severity: 'critical', }, annotations: { - description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} only has {{ $value | humanizePercentage }} free inodes.', + description: 'The PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} only has {{ $value | humanizePercentage }} free inodes.', summary: 'PersistentVolumeInodes are filling up.', }, }, @@ -102,9 +102,9 @@ kubelet_volume_stats_inodes_used{%(prefixedNamespaceSelector)s%(kubeletSelector)s} > 0 and predict_linear(kubelet_volume_stats_inodes_free{%(prefixedNamespaceSelector)s%(kubeletSelector)s}[%(volumeFullPredictionSampleTime)s], 4 * 24 * 3600) < 0 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_access_mode{%(prefixedNamespaceSelector)s access_mode="ReadOnlyMany"} == 1 - unless on(namespace, persistentvolumeclaim) + unless on(%(clusterLabel)s, namespace, persistentvolumeclaim) kube_persistentvolumeclaim_labels{%(prefixedNamespaceSelector)s%(pvExcludedSelector)s} == 1 ||| % $._config, 'for': '1h', @@ -112,7 +112,7 @@ severity: 'warning', }, annotations: { - description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.', + description: 'Based on recent sampling, the PersistentVolume claimed by {{ $labels.persistentvolumeclaim }} in Namespace {{ $labels.namespace }} on Cluster {{ $labels.%(clusterLabel)s }} is expected to run out of inodes within four days. Currently {{ $value | humanizePercentage }} of its inodes are free.', summary: 'PersistentVolumeInodes are filling up.', }, }, @@ -126,7 +126,7 @@ severity: 'critical', }, annotations: { - description: 'The persistent volume {{ $labels.persistentvolume }} has status {{ $labels.phase }}.', + description: 'The persistent volume {{ $labels.persistentvolume }} on Cluster {{ $labels.%(clusterLabel)s }} has status {{ $labels.phase }}.', summary: 'PersistentVolume is having issues with provisioning.', }, },