Skip to content

[QoS controls] upgrade bandwidth QoS logging to Warn, but rate-limit to one message every 5 min #10566

New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

Sign up for GitHub

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

Draft
coutinhop wants to merge 1 commit into
base: master
Choose a base branch
from
Draft

[QoS controls] upgrade bandwidth QoS logging to Warn, but rate-limit to one message every 5 min #10566

Changes from all commits
Commits
Show all changes
1 commit
Select commit
File filter

Filter by extension

Filter by extension
Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
16 changes: 12 additions & 4 deletions felix/dataplane/linux/endpoint_mgr.go
View file
Open in desktop
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@ import (
"reflect"
"regexp"
"strings"
"time"

apiv3 "github.com/projectcalico/api/pkg/apis/projectcalico/v3"
log "github.com/sirupsen/logrus"
Expand All @@ -37,6 +38,7 @@ import (
"github.com/projectcalico/calico/felix/routetable"
"github.com/projectcalico/calico/felix/rules"
"github.com/projectcalico/calico/felix/types"
lclogutils "github.com/projectcalico/calico/libcalico-go/lib/logutils"
"github.com/projectcalico/calico/libcalico-go/lib/set"
)

Expand Down Expand Up @@ -218,6 +220,9 @@ type endpointManager struct {
callbacks endpointManagerCallbacks
bpfEnabled bool
bpfEndpointManager hepListener

// Rate-limited logger, currently used for QoS validation errors
rateLimitedLogger *lclogutils.RateLimitedLogger
}

type EndpointStatusUpdateCallback func(ipVersion uint8, id interface{}, status string, extraInfo interface{})
Expand Down Expand Up @@ -371,6 +376,9 @@ func newEndpointManagerWithShims(
callbacks: newEndpointManagerCallbacks(callbacks, ipVersion),

linkAddrsMgr: linkAddrsMgr,

// Rate-limit logging to one message every 5 minutes
rateLimitedLogger: lclogutils.NewRateLimitedLogger(lclogutils.OptInterval(5 * time.Minute)),
}

return epManager
Expand Down Expand Up @@ -692,10 +700,10 @@ func (m *endpointManager) resolveWorkloadEndpoints() {
if !m.bpfEnabled {
// QoS state should be removed before the workload itself is removed
if oldWorkload != nil {
logCxt.Info("Deleting QoS bandwidth state if present")
logCxt.Debug("Deleting QoS bandwidth state if present")
err := m.maybeUpdateQoSBandwidth(oldWorkload, nil)
if err != nil {
logCxt.WithError(err).WithField("workload", oldWorkload).Debug("Error deleting QoS bandwidth state, workload may have been already removed.")
m.rateLimitedLogger.WithFields(logCxt.Data).WithError(err).WithField("workload", oldWorkload).Warn("Error deleting QoS bandwidth state, workload may have been already removed.")
}
}
}
Expand Down Expand Up @@ -840,10 +848,10 @@ func (m *endpointManager) resolveWorkloadEndpoints() {
delete(m.pendingWlEpUpdates, id)

if !m.bpfEnabled {
logCxt.Info("Updating QoS bandwidth state if changed")
logCxt.Debug("Updating QoS bandwidth state if changed")
err := m.maybeUpdateQoSBandwidth(oldWorkload, workload)
if err != nil {
logCxt.WithError(err).WithFields(log.Fields{"oldWorkload": oldWorkload, "newWorkload": workload}).Debug("Error updating QoS bandwidth state")
m.rateLimitedLogger.WithFields(logCxt.Data).WithError(err).WithFields(log.Fields{"oldWorkload": oldWorkload, "newWorkload": workload}).Warn("Error updating QoS bandwidth state")
}
}

Expand Down