Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support exclude_alerts parameter in Prometheus list rules endpoint #9300

Merged
merged 8 commits into from
Sep 23, 2024
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -88,6 +88,7 @@
* [ENHANCEMENT] Querier: attach logs emitted during query consistency check to trace span for query. #9213
* [ENHANCEMENT] Query-scheduler: Experimental `-query-scheduler.prioritize-query-components` flag enables the querier-worker queue priority algorithm to take precedence over tenant rotation when dequeuing requests. #9220
* [ENHANCEMENT] Add application credential arguments for Openstack Swift storage backend. #9181
* [ENHANCEMENT] Ruler: Support `exclude_alerts` parameter in `<prometheus-http-prefix>/api/v1/rules` endpoint. #9300
* [BUGFIX] Ruler: add support for draining any outstanding alert notifications before shutting down. This can be enabled with the `-ruler.drain-notification-queue-on-shutdown=true` CLI flag. #8346
* [BUGFIX] Query-frontend: fix `-querier.max-query-lookback` enforcement when `-compactor.blocks-retention-period` is not set, and viceversa. #8388
* [BUGFIX] Ingester: fix sporadic `not found` error causing an internal server error if label names are queried with matchers during head compaction. #8391
Expand Down
4 changes: 3 additions & 1 deletion docs/sources/mimir/references/http-api/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -853,7 +853,7 @@ List all tenant rules. This endpoint is not part of ruler-API and is always avai
### List Prometheus rules

```
GET <prometheus-http-prefix>/api/v1/rules?type={alert|record}&file={}&rule_group={}&rule_name={}
GET <prometheus-http-prefix>/api/v1/rules?type={alert|record}&file={}&rule_group={}&rule_name={}&exclude_alerts={true|false}
```

Prometheus-compatible rules endpoint to list alerting and recording rules that are currently loaded.
Expand All @@ -862,6 +862,8 @@ The `type` parameter is optional. If set, only the specified type of rule is ret

The `file`, `rule_group` and `rule_name` parameters are optional, and can accept multiple values. If set, the response content is filtered accordingly.

The `exclude_alerts` parameter is optional. If set, it only returns rules and excludes active alerts.

For more information, refer to Prometheus [rules](https://prometheus.io/docs/prometheus/latest/querying/api/#rules).

Requires [authentication](#authentication).
Expand Down
39 changes: 32 additions & 7 deletions pkg/ruler/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,18 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
return
}

excludeAlerts, err := parseExcludeAlerts(req)
if err != nil {
respondInvalidRequest(logger, w, "invalid exclude_alerts parameter")
return
}

rulesReq := RulesRequest{
Filter: AnyRule,
RuleName: req.URL.Query()["rule_name"],
RuleGroup: req.URL.Query()["rule_group"],
File: req.URL.Query()["file"],
Filter: AnyRule,
RuleName: req.URL.Query()["rule_name"],
RuleGroup: req.URL.Query()["rule_group"],
File: req.URL.Query()["file"],
ExcludeAlerts: excludeAlerts,
}

ruleTypeFilter := strings.ToLower(req.URL.Query().Get("type"))
Expand Down Expand Up @@ -209,9 +216,12 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {

for i, rl := range g.ActiveRules {
if g.ActiveRules[i].Rule.Alert != "" {
alerts := make([]*Alert, 0, len(rl.Alerts))
for _, a := range rl.Alerts {
alerts = append(alerts, alertStateDescToPrometheusAlert(a))
var alerts []*Alert
if !excludeAlerts {
alerts = make([]*Alert, 0, len(rl.Alerts))
for _, a := range rl.Alerts {
alerts = append(alerts, alertStateDescToPrometheusAlert(a))
}
}
grp.Rules[i] = alertingRule{
State: rl.GetState(),
Expand Down Expand Up @@ -265,6 +275,21 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
}
}

func parseExcludeAlerts(req *http.Request) (bool, error) {
excludeAlerts := req.URL.Query().Get("exclude_alerts")
if excludeAlerts == "" {
return false, nil
}

value, err := strconv.ParseBool(excludeAlerts)
if err != nil {
return false, fmt.Errorf("unable to parse exclude_alerts value %w", err)
}

return value, nil

}

func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) {
logger, ctx := spanlogger.NewWithLogger(req.Context(), a.logger, "API.PrometheusAlerts")
defer logger.Finish()
Expand Down
71 changes: 71 additions & 0 deletions pkg/ruler/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,77 @@ func TestRuler_PrometheusRules(t *testing.T) {
},
},
},
"API request with exclude_alerts=true returns alerting rules without alerts": {
configuredRules: rulespb.RuleGroupList{
&rulespb.RuleGroupDesc{
Name: "group1",
Namespace: "namespace1",
User: userID,
Rules: []*rulespb.RuleDesc{createAlertingRule("UP_ALERT", "up < 1")},
Interval: interval,
},
},
expectedConfigured: 1,
queryParams: "?exclude_alerts=true",
limits: validation.MockDefaultOverrides(),
expectedRules: []*RuleGroup{
{
Name: "group1",
File: "namespace1",
Rules: []rule{
&alertingRule{
Name: "UP_ALERT",
Query: "up < 1",
State: "inactive",
Health: "unknown",
Type: "alerting",
Alerts: nil,
},
},
Interval: 60,
},
},
},
"API request with exclude_alerts=false returns alerting rules including alerts": {
configuredRules: rulespb.RuleGroupList{
&rulespb.RuleGroupDesc{
Name: "group1",
Namespace: "namespace1",
User: userID,
Rules: []*rulespb.RuleDesc{createAlertingRule("UP_ALERT", "up < 1")},
Interval: interval,
},
},
expectedConfigured: 1,
queryParams: "?exclude_alerts=false",
limits: validation.MockDefaultOverrides(),
expectedRules: []*RuleGroup{
{
Name: "group1",
File: "namespace1",
Rules: []rule{
&alertingRule{
Name: "UP_ALERT",
Query: "up < 1",
State: "inactive",
Health: "unknown",
Type: "alerting",
Alerts: []*Alert{},
},
},
Interval: 60,
},
},
},
"Invalid exclude_alerts param": {
configuredRules: rulespb.RuleGroupList{},
expectedConfigured: 0,
queryParams: "?exclude_alerts=foo",
limits: validation.MockDefaultOverrides(),
expectedStatusCode: http.StatusBadRequest,
expectedErrorType: v1.ErrBadData,
expectedRules: []*RuleGroup{},
},
"Invalid type param": {
configuredRules: rulespb.RuleGroupList{},
expectedConfigured: 0,
Expand Down
32 changes: 18 additions & 14 deletions pkg/ruler/ruler.go
Original file line number Diff line number Diff line change
Expand Up @@ -1109,20 +1109,24 @@ func (r *Ruler) getLocalRules(ctx context.Context, userID string, req RulesReque
continue
}

alerts := []*AlertStateDesc{}
for _, a := range rule.ActiveAlerts() {
alerts = append(alerts, &AlertStateDesc{
State: a.State.String(),
Labels: mimirpb.FromLabelsToLabelAdapters(a.Labels),
Annotations: mimirpb.FromLabelsToLabelAdapters(a.Annotations),
Value: a.Value,
ActiveAt: a.ActiveAt,
FiredAt: a.FiredAt,
ResolvedAt: a.ResolvedAt,
LastSentAt: a.LastSentAt,
ValidUntil: a.ValidUntil,
KeepFiringSince: a.KeepFiringSince,
})
var alerts []*AlertStateDesc
if !req.ExcludeAlerts {
activeAlerts := rule.ActiveAlerts()
alerts = make([]*AlertStateDesc, 0, len(activeAlerts))
for _, a := range activeAlerts {
alerts = append(alerts, &AlertStateDesc{
State: a.State.String(),
Labels: mimirpb.FromLabelsToLabelAdapters(a.Labels),
Annotations: mimirpb.FromLabelsToLabelAdapters(a.Annotations),
Value: a.Value,
ActiveAt: a.ActiveAt,
FiredAt: a.FiredAt,
ResolvedAt: a.ResolvedAt,
LastSentAt: a.LastSentAt,
ValidUntil: a.ValidUntil,
KeepFiringSince: a.KeepFiringSince,
})
}
}
ruleDesc = &RuleStateDesc{
Rule: &rulespb.RuleDesc{
Expand Down
Loading
Loading