diff --git a/generate_mocks.sh b/generate_mocks.sh index 266519cdc..75e9eca37 100755 --- a/generate_mocks.sh +++ b/generate_mocks.sh @@ -24,6 +24,7 @@ mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter +mockgen -destination=mock/moira-alert/metrics/meters_collection.go -package=mock_moira_alert github.com/moira-alert/moira/metrics MetersCollection mockgen -destination=mock/moira-alert/prometheus_api.go -package=mock_moira_alert github.com/moira-alert/moira/metric_source/prometheus PrometheusApi mockgen -destination=mock/moira-alert/database_stats.go -package=mock_moira_alert github.com/moira-alert/moira/database/stats StatsReporter diff --git a/metrics/notifier.go b/metrics/notifier.go index 3af294d63..7cdbe43ce 100644 --- a/metrics/notifier.go +++ b/metrics/notifier.go @@ -6,37 +6,41 @@ import ( // NotifierMetrics is a collection of metrics used in notifier. type NotifierMetrics struct { - SubsMalformed Meter - EventsReceived Meter - EventsMalformed Meter - EventsProcessingFailed Meter - EventsByState MetersCollection - SendingFailed Meter - SendersOkMetrics MetersCollection - SendersFailedMetrics MetersCollection - SendersDroppedNotifications MetersCollection - PlotsBuildDurationMs Histogram - PlotsEvaluateTriggerDurationMs Histogram - fetchNotificationsDurationMs Histogram - notifierIsAlive Meter + SubsMalformed Meter + EventsReceived Meter + EventsMalformed Meter + EventsProcessingFailed Meter + EventsByState MetersCollection + SendingFailed Meter + ContactsSendingNotificationsOK MetersCollection + ContactsSendingNotificationsFailed MetersCollection + ContactsDroppedNotifications MetersCollection + ContactsDeliveryNotificationsOK MetersCollection + ContactsDeliveryNotificationsFailed MetersCollection + PlotsBuildDurationMs Histogram + PlotsEvaluateTriggerDurationMs Histogram + fetchNotificationsDurationMs Histogram + notifierIsAlive Meter } // ConfigureNotifierMetrics is notifier metrics configurator. func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics { return &NotifierMetrics{ - SubsMalformed: registry.NewMeter("subs", "malformed"), - EventsReceived: registry.NewMeter("events", "received"), - EventsMalformed: registry.NewMeter("events", "malformed"), - EventsProcessingFailed: registry.NewMeter("events", "failed"), - EventsByState: NewMetersCollection(registry), - SendingFailed: registry.NewMeter("sending", "failed"), - SendersOkMetrics: NewMetersCollection(registry), - SendersFailedMetrics: NewMetersCollection(registry), - SendersDroppedNotifications: NewMetersCollection(registry), - PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), - PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), - fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), - notifierIsAlive: registry.NewMeter("", "alive"), + SubsMalformed: registry.NewMeter("subs", "malformed"), + EventsReceived: registry.NewMeter("events", "received"), + EventsMalformed: registry.NewMeter("events", "malformed"), + EventsProcessingFailed: registry.NewMeter("events", "failed"), + EventsByState: NewMetersCollection(registry), + SendingFailed: registry.NewMeter("sending", "failed"), + ContactsSendingNotificationsOK: NewMetersCollection(registry), + ContactsSendingNotificationsFailed: NewMetersCollection(registry), + ContactsDroppedNotifications: NewMetersCollection(registry), + ContactsDeliveryNotificationsOK: NewMetersCollection(registry), + ContactsDeliveryNotificationsFailed: NewMetersCollection(registry), + PlotsBuildDurationMs: registry.NewHistogram("plots", "build", "duration", "ms"), + PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"), + fetchNotificationsDurationMs: registry.NewHistogram("fetch", "notifications", "duration", "ms"), + notifierIsAlive: registry.NewMeter("", "alive"), } } @@ -45,23 +49,23 @@ func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotifica metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds()) } -// MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications. -func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) { - if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found { +// MarkContactDroppedNotifications marks metrics as 1 by contactType for dropped notifications. +func (metrics *NotifierMetrics) MarkContactDroppedNotifications(contactType string) { + if metric, found := metrics.ContactsDroppedNotifications.GetRegisteredMeter(contactType); found { metric.Mark(1) } } -// MarkSendersOkMetrics marks metrics as 1 by contactType when notifications were successfully sent. -func (metrics *NotifierMetrics) MarkSendersOkMetrics(contactType string) { - if metric, found := metrics.SendersOkMetrics.GetRegisteredMeter(contactType); found { +// MarkContactSendingNotificationOK marks metrics as 1 by contactType when notifications were successfully sent. +func (metrics *NotifierMetrics) MarkContactSendingNotificationOK(contactType string) { + if metric, found := metrics.ContactsSendingNotificationsOK.GetRegisteredMeter(contactType); found { metric.Mark(1) } } -// MarkSendersFailedMetrics marks metrics as 1 by contactType when notifications were unsuccessfully sent. -func (metrics *NotifierMetrics) MarkSendersFailedMetrics(contactType string) { - if metric, found := metrics.SendersFailedMetrics.GetRegisteredMeter(contactType); found { +// MarkContactSendingNotificationFailed marks metrics as 1 by contactType when notifications were unsuccessfully sent. +func (metrics *NotifierMetrics) MarkContactSendingNotificationFailed(contactType string) { + if metric, found := metrics.ContactsSendingNotificationsFailed.GetRegisteredMeter(contactType); found { metric.Mark(1) } } diff --git a/metrics/sender.go b/metrics/sender.go new file mode 100644 index 000000000..e1414c207 --- /dev/null +++ b/metrics/sender.go @@ -0,0 +1,17 @@ +package metrics + +// SenderMetrics should be used for sender which can understand if the notification was delivered or not. +type SenderMetrics struct { + ContactDeliveryNotificationOK Meter + ContactDeliveryNotificationFailed Meter +} + +// ConfigureSenderMetrics configures SenderMetrics using NotifierMetrics with given graphiteIdent for senderContactType. +func ConfigureSenderMetrics(notifierMetrics *NotifierMetrics, graphiteIdent string, senderContactType string) *SenderMetrics { + return &SenderMetrics{ + ContactDeliveryNotificationOK: notifierMetrics.ContactsDeliveryNotificationsOK. + RegisterMeter(senderContactType, graphiteIdent, "delivery_ok"), + ContactDeliveryNotificationFailed: notifierMetrics.ContactsDeliveryNotificationsFailed. + RegisterMeter(senderContactType, graphiteIdent, "delivery_failed"), + } +} diff --git a/mock/moira-alert/metrics/meters_collection.go b/mock/moira-alert/metrics/meters_collection.go new file mode 100644 index 000000000..a3dc1bc97 --- /dev/null +++ b/mock/moira-alert/metrics/meters_collection.go @@ -0,0 +1,74 @@ +// Code generated by MockGen. DO NOT EDIT. +// Source: github.com/moira-alert/moira/metrics (interfaces: MetersCollection) +// +// Generated by this command: +// +// mockgen -destination=mock/moira-alert/metrics/meters_collection.go -package=mock_moira_alert github.com/moira-alert/moira/metrics MetersCollection +// + +// Package mock_moira_alert is a generated GoMock package. +package mock_moira_alert + +import ( + reflect "reflect" + + metrics "github.com/moira-alert/moira/metrics" + gomock "go.uber.org/mock/gomock" +) + +// MockMetersCollection is a mock of MetersCollection interface. +type MockMetersCollection struct { + ctrl *gomock.Controller + recorder *MockMetersCollectionMockRecorder +} + +// MockMetersCollectionMockRecorder is the mock recorder for MockMetersCollection. +type MockMetersCollectionMockRecorder struct { + mock *MockMetersCollection +} + +// NewMockMetersCollection creates a new mock instance. +func NewMockMetersCollection(ctrl *gomock.Controller) *MockMetersCollection { + mock := &MockMetersCollection{ctrl: ctrl} + mock.recorder = &MockMetersCollectionMockRecorder{mock} + return mock +} + +// EXPECT returns an object that allows the caller to indicate expected use. +func (m *MockMetersCollection) EXPECT() *MockMetersCollectionMockRecorder { + return m.recorder +} + +// GetRegisteredMeter mocks base method. +func (m *MockMetersCollection) GetRegisteredMeter(arg0 string) (metrics.Meter, bool) { + m.ctrl.T.Helper() + ret := m.ctrl.Call(m, "GetRegisteredMeter", arg0) + ret0, _ := ret[0].(metrics.Meter) + ret1, _ := ret[1].(bool) + return ret0, ret1 +} + +// GetRegisteredMeter indicates an expected call of GetRegisteredMeter. +func (mr *MockMetersCollectionMockRecorder) GetRegisteredMeter(arg0 any) *gomock.Call { + mr.mock.ctrl.T.Helper() + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "GetRegisteredMeter", reflect.TypeOf((*MockMetersCollection)(nil).GetRegisteredMeter), arg0) +} + +// RegisterMeter mocks base method. +func (m *MockMetersCollection) RegisterMeter(arg0 string, arg1 ...string) metrics.Meter { + m.ctrl.T.Helper() + varargs := []any{arg0} + for _, a := range arg1 { + varargs = append(varargs, a) + } + ret := m.ctrl.Call(m, "RegisterMeter", varargs...) + ret0, _ := ret[0].(metrics.Meter) + return ret0 +} + +// RegisterMeter indicates an expected call of RegisterMeter. +func (mr *MockMetersCollectionMockRecorder) RegisterMeter(arg0 any, arg1 ...any) *gomock.Call { + mr.mock.ctrl.T.Helper() + varargs := append([]any{arg0}, arg1...) + return mr.mock.ctrl.RecordCallWithMethodType(mr.mock, "RegisterMeter", reflect.TypeOf((*MockMetersCollection)(nil).RegisterMeter), varargs...) +} diff --git a/notifier/notifier.go b/notifier/notifier.go index 6dd11c0bc..7d2ecb702 100644 --- a/notifier/notifier.go +++ b/notifier/notifier.go @@ -142,17 +142,17 @@ func (notifier *StandardNotifier) GetReadBatchSize() int64 { func (notifier *StandardNotifier) reschedule(pkg *NotificationPackage, reason string) { if pkg.DontResend { - notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type) + notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type) return } notifier.metrics.MarkSendingFailed() - notifier.metrics.MarkSendersFailedMetrics(pkg.Contact.Type) + notifier.metrics.MarkContactSendingNotificationFailed(pkg.Contact.Type) logger := getLogWithPackageContext(¬ifier.logger, pkg, ¬ifier.config) if notifier.needToStop(pkg.FailCount) { - notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type) + notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type) logger.Error(). Msg("Stop resending. Notification interval is timed out") return @@ -227,7 +227,7 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific err = sender.SendEvents(pkg.Events, pkg.Contact, pkg.Trigger, plots, pkg.Throttled) if err == nil { - notifier.metrics.MarkSendersOkMetrics(pkg.Contact.Type) + notifier.metrics.MarkContactSendingNotificationOK(pkg.Contact.Type) continue } switch e := err.(type) { // nolint:errorlint @@ -235,7 +235,7 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific log.Warning(). Error(e). Msg("Cannot send to broken contact") - notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type) + notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type) default: if pkg.FailCount > notifier.config.MaxFailAttemptToSendAvailable { log.Error(). diff --git a/notifier/plotting_test.go b/notifier/plotting_test.go index b91f341b0..106e3a38f 100644 --- a/notifier/plotting_test.go +++ b/notifier/plotting_test.go @@ -197,10 +197,10 @@ func TestFetchAvailableSeries(t *testing.T) { to = 67 ) Convey("Run fetchAvailableSeries", t, func() { - mockCtrl := gomock.NewController(t) - defer mockCtrl.Finish() - source := mockMetricSource.NewMockMetricSource(mockCtrl) - result := mockMetricSource.NewMockFetchResult(mockCtrl) + mockController := gomock.NewController(t) + defer mockController.Finish() + source := mockMetricSource.NewMockMetricSource(mockController) + result := mockMetricSource.NewMockFetchResult(mockController) Convey("without errors", func() { gomock.InOrder( diff --git a/notifier/registrator.go b/notifier/registrator.go index d1c4bfebb..104635dbc 100644 --- a/notifier/registrator.go +++ b/notifier/registrator.go @@ -6,6 +6,7 @@ import ( "strings" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" "github.com/moira-alert/moira/senders/discord" "github.com/moira-alert/moira/senders/mail" "github.com/moira-alert/moira/senders/mattermost" @@ -107,11 +108,16 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro } func (notifier *StandardNotifier) registerMetrics(senderContactType string) { - notifier.metrics.SendersOkMetrics.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok") - notifier.metrics.SendersFailedMetrics.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed") - notifier.metrics.SendersDroppedNotifications.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped") + notifier.metrics.ContactsSendingNotificationsOK.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok") + notifier.metrics.ContactsSendingNotificationsFailed.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed") + notifier.metrics.ContactsDroppedNotifications.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped") } +const ( + senderMetricsEnabledKey = "enable_metrics" + senderMetricsKey = "sender_metrics" +) + // RegisterSender adds sender for notification type and registers metrics. func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]interface{}, sender moira.Sender) error { senderType, ok := senderSettings["sender_type"].(string) @@ -128,6 +134,17 @@ func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]inter return fmt.Errorf("failed to initialize sender [%s], err [%w]", senderContactType, ErrSenderRegistered) } + if senderMetricsEnabled, ok := senderSettings[senderMetricsEnabledKey].(bool); ok && senderMetricsEnabled { + senderSettings[senderMetricsKey] = metrics.ConfigureSenderMetrics( + notifier.metrics, + getGraphiteSenderIdent(senderContactType), + senderContactType) + notifier.logger.Info(). + String("sender_contact_type", senderContactType). + String("sender_type", senderType). + Msg("Enable sender metrics") + } + err := sender.Init(senderSettings, notifier.logger, notifier.config.Location, notifier.config.DateTimeFormat) if err != nil { return fmt.Errorf("failed to initialize sender [%s], err [%w]", senderContactType, err) diff --git a/notifier/registrator_test.go b/notifier/registrator_test.go index 47a65c2c9..ce020bdc2 100644 --- a/notifier/registrator_test.go +++ b/notifier/registrator_test.go @@ -5,6 +5,8 @@ import ( "testing" "time" + "github.com/moira-alert/moira/metrics" + mock_metrics "github.com/moira-alert/moira/mock/moira-alert/metrics" . "github.com/smartystreets/goconvey/convey" ) @@ -60,6 +62,68 @@ func TestRegisterSender(t *testing.T) { err := standardNotifier.RegisterSender(senderSettings, sender) So(err, ShouldBeNil) }) + + Convey("Successfully register sender with damaged enable_metrics", func() { + sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl) + + notifierMetrics := &metrics.NotifierMetrics{ + ContactsSendingNotificationsOK: sendersOkMetrics, + ContactsSendingNotificationsFailed: sendersFailedMetrics, + ContactsDroppedNotifications: sendersDroppedNotifications, + } + standardNotifier.metrics = notifierMetrics + + senderContactType := "test_contact_new_1" + senderSettings := map[string]interface{}{ + "sender_type": "test_type", + "contact_type": "test_contact_new_1", + "enable_metrics": "abracdabra", + } + + sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1) + sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1) + sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1) + sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat) + + err := standardNotifier.RegisterSender(senderSettings, sender) + So(err, ShouldBeNil) + }) + + Convey("Register sender with additional metrics", func() { + sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersDeliveryOK := mock_metrics.NewMockMetersCollection(mockCtrl) + sendersDeliveryFailed := mock_metrics.NewMockMetersCollection(mockCtrl) + + notifierMetrics := &metrics.NotifierMetrics{ + ContactsSendingNotificationsOK: sendersOkMetrics, + ContactsSendingNotificationsFailed: sendersFailedMetrics, + ContactsDroppedNotifications: sendersDroppedNotifications, + ContactsDeliveryNotificationsOK: sendersDeliveryOK, + ContactsDeliveryNotificationsFailed: sendersDeliveryFailed, + } + standardNotifier.metrics = notifierMetrics + + senderContactType := "test_contact_new_2" + senderSettings := map[string]interface{}{ + "sender_type": "test_type", + "contact_type": senderContactType, + "enable_metrics": true, + } + + sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1) + sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1) + sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1) + sendersDeliveryOK.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_ok").Times(1) + sendersDeliveryFailed.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_failed").Times(1) + sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat) + + err := standardNotifier.RegisterSender(senderSettings, sender) + So(err, ShouldBeNil) + }) }) } diff --git a/senders/webhook/webhook.go b/senders/webhook/webhook.go index 036c877cb..6fe62758c 100644 --- a/senders/webhook/webhook.go +++ b/senders/webhook/webhook.go @@ -8,6 +8,7 @@ import ( "github.com/mitchellh/mapstructure" "github.com/moira-alert/moira" + "github.com/moira-alert/moira/metrics" ) // Structure that represents the Webhook configuration in the YAML file. @@ -29,8 +30,11 @@ type Sender struct { headers map[string]string client *http.Client log moira.Logger + metrics *metrics.SenderMetrics } +const senderMetricsKey = "sender_metrics" + // Init read yaml config. func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, location *time.Location, dateTimeFormat string) error { var cfg config @@ -69,6 +73,11 @@ func (sender *Sender) Init(senderSettings interface{}, logger moira.Logger, loca Transport: &http.Transport{DisableKeepAlives: true}, } + senderSettingsMap := senderSettings.(map[string]interface{}) + if val, ok := senderSettingsMap[senderMetricsKey]; ok { + sender.metrics = val.(*metrics.SenderMetrics) + } + return nil } diff --git a/senders/webhook/webhook_test.go b/senders/webhook/webhook_test.go index 845054881..ad998d706 100644 --- a/senders/webhook/webhook_test.go +++ b/senders/webhook/webhook_test.go @@ -6,6 +6,7 @@ import ( "encoding/base64" "errors" "fmt" + "maps" "net/http" "net/http/httptest" "net/url" @@ -15,9 +16,10 @@ import ( "github.com/go-playground/validator/v10" "github.com/moira-alert/moira" - logging "github.com/moira-alert/moira/logging/zerolog_adapter" + "github.com/moira-alert/moira/metrics" . "github.com/smartystreets/goconvey/convey" + "go.uber.org/mock/gomock" ) const ( @@ -40,6 +42,9 @@ var ( func TestSender_Init(t *testing.T) { Convey("Test Init function", t, func() { + mockCtrl := gomock.NewController(t) + defer mockCtrl.Finish() + validatorErr := validator.ValidationErrors{} Convey("With empty url", func() { @@ -81,7 +86,7 @@ func TestSender_Init(t *testing.T) { "timeout": 120, } sender := Sender{} - expectedHeaders := defaultHeaders + expectedHeaders := maps.Clone(defaultHeaders) expectedHeaders["testHeader"] = "test" err := sender.Init(settings, logger, location, dateTimeFormat) @@ -99,6 +104,29 @@ func TestSender_Init(t *testing.T) { log: logger, }) }) + + Convey("With url and metricsMarker", func() { + senderMetrics := &metrics.SenderMetrics{} + + settings := map[string]interface{}{ + "url": testURL, + senderMetricsKey: senderMetrics, + } + + sender := Sender{} + err := sender.Init(settings, logger, location, dateTimeFormat) + So(err, ShouldBeNil) + So(sender, ShouldResemble, Sender{ + url: testURL, + headers: defaultHeaders, + client: &http.Client{ + Timeout: 30 * time.Second, + Transport: &http.Transport{DisableKeepAlives: true}, + }, + log: logger, + metrics: senderMetrics, + }) + }) }) }