Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Support exclude_alerts parameter in Prometheus list rules endpoint #9300

Open
wants to merge 5 commits into
base: main
Choose a base branch
from
Open
Show file tree
Hide file tree
Changes from 4 commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
1 change: 1 addition & 0 deletions CHANGELOG.md
Original file line number Diff line number Diff line change
Expand Up @@ -86,6 +86,7 @@
* [ENHANCEMENT] OTLP: If the flag `-distributor.otel-created-timestamp-zero-ingestion-enabled` is true, OTel start timestamps are converted to Prometheus zero samples to mark series start. #9131
* [ENHANCEMENT] Querier: attach logs emitted during query consistency check to trace span for query. #9213
* [ENHANCEMENT] Query-scheduler: Experimental `-query-scheduler.prioritize-query-components` flag enables the querier-worker queue priority algorithm to take precedence over tenant rotation when dequeuing requests. #9220
* [ENHANCEMENT] Support `exclude_alerts` parameter in `<prometheus-http-prefix>/api/v1/rules` endpoint. #9300
* [BUGFIX] Ruler: add support for draining any outstanding alert notifications before shutting down. This can be enabled with the `-ruler.drain-notification-queue-on-shutdown=true` CLI flag. #8346
* [BUGFIX] Query-frontend: fix `-querier.max-query-lookback` enforcement when `-compactor.blocks-retention-period` is not set, and viceversa. #8388
* [BUGFIX] Ingester: fix sporadic `not found` error causing an internal server error if label names are queried with matchers during head compaction. #8391
Expand Down
4 changes: 3 additions & 1 deletion docs/sources/mimir/references/http-api/index.md
Original file line number Diff line number Diff line change
Expand Up @@ -815,7 +815,7 @@ List all tenant rules. This endpoint is not part of ruler-API and is always avai
### List Prometheus rules

```
GET <prometheus-http-prefix>/api/v1/rules?type={alert|record}&file={}&rule_group={}&rule_name={}
GET <prometheus-http-prefix>/api/v1/rules?type={alert|record}&file={}&rule_group={}&rule_name={}&exclude_alerts={true|false}
```

Prometheus-compatible rules endpoint to list alerting and recording rules that are currently loaded.
Expand All @@ -824,6 +824,8 @@ The `type` parameter is optional. If set, only the specified type of rule is ret

The `file`, `rule_group` and `rule_name` parameters are optional, and can accept multiple values. If set, the response content is filtered accordingly.

The `exclude_alerts` parameter is optional. If set, only rules are returned, active alerts are excluded.
alexander-akhmetov marked this conversation as resolved.
Show resolved Hide resolved

For more information, refer to Prometheus [rules](https://prometheus.io/docs/prometheus/latest/querying/api/#rules).

Requires [authentication](#authentication).
Expand Down
39 changes: 32 additions & 7 deletions pkg/ruler/api.go
Original file line number Diff line number Diff line change
Expand Up @@ -166,11 +166,18 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
return
}

excludeAlerts, err := parseExcludeAlerts(req)
if err != nil {
respondInvalidRequest(logger, w, "invalid exclude_alerts parameter")
return
}

rulesReq := RulesRequest{
Filter: AnyRule,
RuleName: req.URL.Query()["rule_name"],
RuleGroup: req.URL.Query()["rule_group"],
File: req.URL.Query()["file"],
Filter: AnyRule,
RuleName: req.URL.Query()["rule_name"],
RuleGroup: req.URL.Query()["rule_group"],
File: req.URL.Query()["file"],
ExcludeAlerts: excludeAlerts,
}

ruleTypeFilter := strings.ToLower(req.URL.Query().Get("type"))
Expand Down Expand Up @@ -209,9 +216,12 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {

for i, rl := range g.ActiveRules {
if g.ActiveRules[i].Rule.Alert != "" {
alerts := make([]*Alert, 0, len(rl.Alerts))
for _, a := range rl.Alerts {
alerts = append(alerts, alertStateDescToPrometheusAlert(a))
var alerts []*Alert
if !excludeAlerts {
alerts = make([]*Alert, 0, len(rl.Alerts))
for _, a := range rl.Alerts {
alerts = append(alerts, alertStateDescToPrometheusAlert(a))
}
}
grp.Rules[i] = alertingRule{
State: rl.GetState(),
Expand Down Expand Up @@ -265,6 +275,21 @@ func (a *API) PrometheusRules(w http.ResponseWriter, req *http.Request) {
}
}

func parseExcludeAlerts(req *http.Request) (bool, error) {
excludeAlerts := req.URL.Query().Get("exclude_alerts")
if excludeAlerts == "" {
return false, nil
}

value, err := strconv.ParseBool(excludeAlerts)
if err != nil {
return false, fmt.Errorf("unable to parse exclude_alerts value %w", err)
}

return value, nil

}

func (a *API) PrometheusAlerts(w http.ResponseWriter, req *http.Request) {
logger, ctx := spanlogger.NewWithLogger(req.Context(), a.logger, "API.PrometheusAlerts")
defer logger.Finish()
Expand Down
71 changes: 71 additions & 0 deletions pkg/ruler/api_test.go
Original file line number Diff line number Diff line change
Expand Up @@ -542,6 +542,77 @@ func TestRuler_PrometheusRules(t *testing.T) {
},
},
},
"API request with exclude_alerts=true returns alerting rules without alerts": {
configuredRules: rulespb.RuleGroupList{
&rulespb.RuleGroupDesc{
Name: "group1",
Namespace: "namespace1",
User: userID,
Rules: []*rulespb.RuleDesc{createAlertingRule("UP_ALERT", "up < 1")},
Interval: interval,
},
},
expectedConfigured: 1,
queryParams: "?exclude_alerts=true",
limits: validation.MockDefaultOverrides(),
expectedRules: []*RuleGroup{
{
Name: "group1",
File: "namespace1",
Rules: []rule{
&alertingRule{
Name: "UP_ALERT",
Query: "up < 1",
State: "inactive",
Health: "unknown",
Type: "alerting",
Alerts: nil,
},
},
Interval: 60,
},
},
},
"API request with exclude_alerts=false returns alerting rules including alerts": {
configuredRules: rulespb.RuleGroupList{
&rulespb.RuleGroupDesc{
Name: "group1",
Namespace: "namespace1",
User: userID,
Rules: []*rulespb.RuleDesc{createAlertingRule("UP_ALERT", "up < 1")},
Interval: interval,
},
},
expectedConfigured: 1,
queryParams: "?exclude_alerts=false",
limits: validation.MockDefaultOverrides(),
expectedRules: []*RuleGroup{
{
Name: "group1",
File: "namespace1",
Rules: []rule{
&alertingRule{
Name: "UP_ALERT",
Query: "up < 1",
State: "inactive",
Health: "unknown",
Type: "alerting",
Alerts: []*Alert{},
},
},
Interval: 60,
},
},
},
"Invalid exclude_alerts param": {
configuredRules: rulespb.RuleGroupList{},
expectedConfigured: 0,
queryParams: "?exclude_alerts=foo",
limits: validation.MockDefaultOverrides(),
expectedStatusCode: http.StatusBadRequest,
expectedErrorType: v1.ErrBadData,
expectedRules: []*RuleGroup{},
},
"Invalid type param": {
configuredRules: rulespb.RuleGroupList{},
expectedConfigured: 0,
Expand Down
32 changes: 18 additions & 14 deletions pkg/ruler/ruler.go
Original file line number Diff line number Diff line change
Expand Up @@ -1111,20 +1111,24 @@ func (r *Ruler) getLocalRules(ctx context.Context, userID string, req RulesReque
continue
}

alerts := []*AlertStateDesc{}
for _, a := range rule.ActiveAlerts() {
alerts = append(alerts, &AlertStateDesc{
State: a.State.String(),
Labels: mimirpb.FromLabelsToLabelAdapters(a.Labels),
Annotations: mimirpb.FromLabelsToLabelAdapters(a.Annotations),
Value: a.Value,
ActiveAt: a.ActiveAt,
FiredAt: a.FiredAt,
ResolvedAt: a.ResolvedAt,
LastSentAt: a.LastSentAt,
ValidUntil: a.ValidUntil,
KeepFiringSince: a.KeepFiringSince,
})
var alerts []*AlertStateDesc
if !req.ExcludeAlerts {
activeAlerts := rule.ActiveAlerts()
alerts = make([]*AlertStateDesc, 0, len(activeAlerts))
for _, a := range activeAlerts {
alerts = append(alerts, &AlertStateDesc{
State: a.State.String(),
Labels: mimirpb.FromLabelsToLabelAdapters(a.Labels),
Annotations: mimirpb.FromLabelsToLabelAdapters(a.Annotations),
Value: a.Value,
ActiveAt: a.ActiveAt,
FiredAt: a.FiredAt,
ResolvedAt: a.ResolvedAt,
LastSentAt: a.LastSentAt,
ValidUntil: a.ValidUntil,
KeepFiringSince: a.KeepFiringSince,
})
}
}
ruleDesc = &RuleStateDesc{
Rule: &rulespb.RuleDesc{
Expand Down
Loading
Loading