feat: senders delivery metrics (#1143)

generate_mocks.sh

@@ -24,6 +24,7 @@ mockgen -destination=mock/notifier/mattermost/client.go -package=mock_mattermost
 
 mockgen -destination=mock/moira-alert/metrics/registry.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Registry
 mockgen -destination=mock/moira-alert/metrics/meter.go -package=mock_moira_alert github.com/moira-alert/moira/metrics Meter
+mockgen -destination=mock/moira-alert/metrics/meters_collection.go -package=mock_moira_alert github.com/moira-alert/moira/metrics MetersCollection
 mockgen -destination=mock/moira-alert/prometheus_api.go -package=mock_moira_alert github.com/moira-alert/moira/metric_source/prometheus PrometheusApi
 
 mockgen -destination=mock/moira-alert/database_stats.go -package=mock_moira_alert github.com/moira-alert/moira/database/stats StatsReporter

metrics/notifier.go

@@ -6,37 +6,41 @@ import (
 
 // NotifierMetrics is a collection of metrics used in notifier.
 type NotifierMetrics struct {
-	SubsMalformed                  Meter
-	EventsReceived                 Meter
-	EventsMalformed                Meter
-	EventsProcessingFailed         Meter
-	EventsByState                  MetersCollection
-	SendingFailed                  Meter
-	SendersOkMetrics               MetersCollection
-	SendersFailedMetrics           MetersCollection
-	SendersDroppedNotifications    MetersCollection
-	PlotsBuildDurationMs           Histogram
-	PlotsEvaluateTriggerDurationMs Histogram
-	fetchNotificationsDurationMs   Histogram
-	notifierIsAlive                Meter
+	SubsMalformed                       Meter
+	EventsReceived                      Meter
+	EventsMalformed                     Meter
+	EventsProcessingFailed              Meter
+	EventsByState                       MetersCollection
+	SendingFailed                       Meter
+	ContactsSendingNotificationsOK      MetersCollection
+	ContactsSendingNotificationsFailed  MetersCollection
+	ContactsDroppedNotifications        MetersCollection
+	ContactsDeliveryNotificationsOK     MetersCollection
+	ContactsDeliveryNotificationsFailed MetersCollection
+	PlotsBuildDurationMs                Histogram
+	PlotsEvaluateTriggerDurationMs      Histogram
+	fetchNotificationsDurationMs        Histogram
+	notifierIsAlive                     Meter
 }
 
 // ConfigureNotifierMetrics is notifier metrics configurator.
 func ConfigureNotifierMetrics(registry Registry, prefix string) *NotifierMetrics {
 	return &NotifierMetrics{
-		SubsMalformed:                  registry.NewMeter("subs", "malformed"),
-		EventsReceived:                 registry.NewMeter("events", "received"),
-		EventsMalformed:                registry.NewMeter("events", "malformed"),
-		EventsProcessingFailed:         registry.NewMeter("events", "failed"),
-		EventsByState:                  NewMetersCollection(registry),
-		SendingFailed:                  registry.NewMeter("sending", "failed"),
-		SendersOkMetrics:               NewMetersCollection(registry),
-		SendersFailedMetrics:           NewMetersCollection(registry),
-		SendersDroppedNotifications:    NewMetersCollection(registry),
-		PlotsBuildDurationMs:           registry.NewHistogram("plots", "build", "duration", "ms"),
-		PlotsEvaluateTriggerDurationMs: registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"),
-		fetchNotificationsDurationMs:   registry.NewHistogram("fetch", "notifications", "duration", "ms"),
-		notifierIsAlive:                registry.NewMeter("", "alive"),
+		SubsMalformed:                       registry.NewMeter("subs", "malformed"),
+		EventsReceived:                      registry.NewMeter("events", "received"),
+		EventsMalformed:                     registry.NewMeter("events", "malformed"),
+		EventsProcessingFailed:              registry.NewMeter("events", "failed"),
+		EventsByState:                       NewMetersCollection(registry),
+		SendingFailed:                       registry.NewMeter("sending", "failed"),
+		ContactsSendingNotificationsOK:      NewMetersCollection(registry),
+		ContactsSendingNotificationsFailed:  NewMetersCollection(registry),
+		ContactsDroppedNotifications:        NewMetersCollection(registry),
+		ContactsDeliveryNotificationsOK:     NewMetersCollection(registry),
+		ContactsDeliveryNotificationsFailed: NewMetersCollection(registry),
+		PlotsBuildDurationMs:                registry.NewHistogram("plots", "build", "duration", "ms"),
+		PlotsEvaluateTriggerDurationMs:      registry.NewHistogram("plots", "evaluate", "trigger", "duration", "ms"),
+		fetchNotificationsDurationMs:        registry.NewHistogram("fetch", "notifications", "duration", "ms"),
+		notifierIsAlive:                     registry.NewMeter("", "alive"),
 	}
 }
 
@@ -45,23 +49,23 @@ func (metrics *NotifierMetrics) UpdateFetchNotificationsDurationMs(fetchNotifica
 	metrics.fetchNotificationsDurationMs.Update(time.Since(fetchNotificationsStartTime).Milliseconds())
 }
 
-// MarkSendersDroppedNotifications marks metrics as 1 by contactType for dropped notifications.
-func (metrics *NotifierMetrics) MarkSendersDroppedNotifications(contactType string) {
-	if metric, found := metrics.SendersDroppedNotifications.GetRegisteredMeter(contactType); found {
+// MarkContactDroppedNotifications marks metrics as 1 by contactType for dropped notifications.
+func (metrics *NotifierMetrics) MarkContactDroppedNotifications(contactType string) {
+	if metric, found := metrics.ContactsDroppedNotifications.GetRegisteredMeter(contactType); found {
 		metric.Mark(1)
 	}
 }
 
-// MarkSendersOkMetrics marks metrics as 1 by contactType when notifications were successfully sent.
-func (metrics *NotifierMetrics) MarkSendersOkMetrics(contactType string) {
-	if metric, found := metrics.SendersOkMetrics.GetRegisteredMeter(contactType); found {
+// MarkContactSendingNotificationOK marks metrics as 1 by contactType when notifications were successfully sent.
+func (metrics *NotifierMetrics) MarkContactSendingNotificationOK(contactType string) {
+	if metric, found := metrics.ContactsSendingNotificationsOK.GetRegisteredMeter(contactType); found {
 		metric.Mark(1)
 	}
 }
 
-// MarkSendersFailedMetrics marks metrics as 1 by contactType when notifications were unsuccessfully sent.
-func (metrics *NotifierMetrics) MarkSendersFailedMetrics(contactType string) {
-	if metric, found := metrics.SendersFailedMetrics.GetRegisteredMeter(contactType); found {
+// MarkContactSendingNotificationFailed marks metrics as 1 by contactType when notifications were unsuccessfully sent.
+func (metrics *NotifierMetrics) MarkContactSendingNotificationFailed(contactType string) {
+	if metric, found := metrics.ContactsSendingNotificationsFailed.GetRegisteredMeter(contactType); found {
 		metric.Mark(1)
 	}
 }

metrics/sender.go

@@ -0,0 +1,17 @@
+package metrics
+
+// SenderMetrics should be used for sender which can understand if the notification was delivered or not.
+type SenderMetrics struct {
+	ContactDeliveryNotificationOK     Meter
+	ContactDeliveryNotificationFailed Meter
+}
+
+// ConfigureSenderMetrics configures SenderMetrics using NotifierMetrics with given graphiteIdent for senderContactType.
+func ConfigureSenderMetrics(notifierMetrics *NotifierMetrics, graphiteIdent string, senderContactType string) *SenderMetrics {
+	return &SenderMetrics{
+		ContactDeliveryNotificationOK: notifierMetrics.ContactsDeliveryNotificationsOK.
+			RegisterMeter(senderContactType, graphiteIdent, "delivery_ok"),
+		ContactDeliveryNotificationFailed: notifierMetrics.ContactsDeliveryNotificationsFailed.
+			RegisterMeter(senderContactType, graphiteIdent, "delivery_failed"),
+	}
+}

notifier/notifier.go

@@ -142,17 +142,17 @@ func (notifier *StandardNotifier) GetReadBatchSize() int64 {
 
 func (notifier *StandardNotifier) reschedule(pkg *NotificationPackage, reason string) {
 	if pkg.DontResend {
-		notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type)
+		notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type)
 		return
 	}
 
 	notifier.metrics.MarkSendingFailed()
-	notifier.metrics.MarkSendersFailedMetrics(pkg.Contact.Type)
+	notifier.metrics.MarkContactSendingNotificationFailed(pkg.Contact.Type)
 
 	logger := getLogWithPackageContext(&notifier.logger, pkg, &notifier.config)
 
 	if notifier.needToStop(pkg.FailCount) {
-		notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type)
+		notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type)
 		logger.Error().
 			Msg("Stop resending. Notification interval is timed out")
 		return
@@ -227,15 +227,15 @@ func (notifier *StandardNotifier) runSender(sender moira.Sender, ch chan Notific
 
 		err = sender.SendEvents(pkg.Events, pkg.Contact, pkg.Trigger, plots, pkg.Throttled)
 		if err == nil {
-			notifier.metrics.MarkSendersOkMetrics(pkg.Contact.Type)
+			notifier.metrics.MarkContactSendingNotificationOK(pkg.Contact.Type)
 			continue
 		}
 		switch e := err.(type) { // nolint:errorlint
 		case moira.SenderBrokenContactError:
 			log.Warning().
 				Error(e).
 				Msg("Cannot send to broken contact")
-			notifier.metrics.MarkSendersDroppedNotifications(pkg.Contact.Type)
+			notifier.metrics.MarkContactDroppedNotifications(pkg.Contact.Type)
 		default:
 			if pkg.FailCount > notifier.config.MaxFailAttemptToSendAvailable {
 				log.Error().

notifier/plotting_test.go

@@ -197,10 +197,10 @@ func TestFetchAvailableSeries(t *testing.T) {
 		to     = 67
 	)
 	Convey("Run fetchAvailableSeries", t, func() {
-		mockCtrl := gomock.NewController(t)
-		defer mockCtrl.Finish()
-		source := mockMetricSource.NewMockMetricSource(mockCtrl)
-		result := mockMetricSource.NewMockFetchResult(mockCtrl)
+		mockController := gomock.NewController(t)
+		defer mockController.Finish()
+		source := mockMetricSource.NewMockMetricSource(mockController)
+		result := mockMetricSource.NewMockFetchResult(mockController)
 
 		Convey("without errors", func() {
 			gomock.InOrder(

notifier/registrator.go

@@ -6,6 +6,7 @@ import (
 	"strings"
 
 	"github.com/moira-alert/moira"
+	"github.com/moira-alert/moira/metrics"
 	"github.com/moira-alert/moira/senders/discord"
 	"github.com/moira-alert/moira/senders/mail"
 	"github.com/moira-alert/moira/senders/mattermost"
@@ -107,11 +108,16 @@ func (notifier *StandardNotifier) RegisterSenders(connector moira.Database) erro
 }
 
 func (notifier *StandardNotifier) registerMetrics(senderContactType string) {
-	notifier.metrics.SendersOkMetrics.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok")
-	notifier.metrics.SendersFailedMetrics.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed")
-	notifier.metrics.SendersDroppedNotifications.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped")
+	notifier.metrics.ContactsSendingNotificationsOK.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok")
+	notifier.metrics.ContactsSendingNotificationsFailed.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed")
+	notifier.metrics.ContactsDroppedNotifications.RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped")
 }
 
+const (
+	senderMetricsEnabledKey = "enable_metrics"
+	senderMetricsKey        = "sender_metrics"
+)
+
 // RegisterSender adds sender for notification type and registers metrics.
 func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]interface{}, sender moira.Sender) error {
 	senderType, ok := senderSettings["sender_type"].(string)
@@ -128,6 +134,17 @@ func (notifier *StandardNotifier) RegisterSender(senderSettings map[string]inter
 		return fmt.Errorf("failed to initialize sender [%s], err [%w]", senderContactType, ErrSenderRegistered)
 	}
 
+	if senderMetricsEnabled, ok := senderSettings[senderMetricsEnabledKey].(bool); ok && senderMetricsEnabled {
+		senderSettings[senderMetricsKey] = metrics.ConfigureSenderMetrics(
+			notifier.metrics,
+			getGraphiteSenderIdent(senderContactType),
+			senderContactType)
+		notifier.logger.Info().
+			String("sender_contact_type", senderContactType).
+			String("sender_type", senderType).
+			Msg("Enable sender metrics")
+	}
+
 	err := sender.Init(senderSettings, notifier.logger, notifier.config.Location, notifier.config.DateTimeFormat)
 	if err != nil {
 		return fmt.Errorf("failed to initialize sender [%s], err [%w]", senderContactType, err)

notifier/registrator_test.go

@@ -5,6 +5,8 @@ import (
 	"testing"
 	"time"
 
+	"github.com/moira-alert/moira/metrics"
+	mock_metrics "github.com/moira-alert/moira/mock/moira-alert/metrics"
 	. "github.com/smartystreets/goconvey/convey"
 )
 
@@ -60,6 +62,68 @@ func TestRegisterSender(t *testing.T) {
 			err := standardNotifier.RegisterSender(senderSettings, sender)
 			So(err, ShouldBeNil)
 		})
+
+		Convey("Successfully register sender with damaged enable_metrics", func() {
+			sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
+
+			notifierMetrics := &metrics.NotifierMetrics{
+				ContactsSendingNotificationsOK:     sendersOkMetrics,
+				ContactsSendingNotificationsFailed: sendersFailedMetrics,
+				ContactsDroppedNotifications:       sendersDroppedNotifications,
+			}
+			standardNotifier.metrics = notifierMetrics
+
+			senderContactType := "test_contact_new_1"
+			senderSettings := map[string]interface{}{
+				"sender_type":    "test_type",
+				"contact_type":   "test_contact_new_1",
+				"enable_metrics": "abracdabra",
+			}
+
+			sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
+			sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
+			sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
+			sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat)
+
+			err := standardNotifier.RegisterSender(senderSettings, sender)
+			So(err, ShouldBeNil)
+		})
+
+		Convey("Register sender with additional metrics", func() {
+			sendersOkMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersFailedMetrics := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersDroppedNotifications := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersDeliveryOK := mock_metrics.NewMockMetersCollection(mockCtrl)
+			sendersDeliveryFailed := mock_metrics.NewMockMetersCollection(mockCtrl)
+
+			notifierMetrics := &metrics.NotifierMetrics{
+				ContactsSendingNotificationsOK:      sendersOkMetrics,
+				ContactsSendingNotificationsFailed:  sendersFailedMetrics,
+				ContactsDroppedNotifications:        sendersDroppedNotifications,
+				ContactsDeliveryNotificationsOK:     sendersDeliveryOK,
+				ContactsDeliveryNotificationsFailed: sendersDeliveryFailed,
+			}
+			standardNotifier.metrics = notifierMetrics
+
+			senderContactType := "test_contact_new_2"
+			senderSettings := map[string]interface{}{
+				"sender_type":    "test_type",
+				"contact_type":   senderContactType,
+				"enable_metrics": true,
+			}
+
+			sendersOkMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_ok").Times(1)
+			sendersFailedMetrics.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "sends_failed").Times(1)
+			sendersDroppedNotifications.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "notifications_dropped").Times(1)
+			sendersDeliveryOK.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_ok").Times(1)
+			sendersDeliveryFailed.EXPECT().RegisterMeter(senderContactType, getGraphiteSenderIdent(senderContactType), "delivery_failed").Times(1)
+			sender.EXPECT().Init(senderSettings, standardNotifier.logger, standardNotifier.config.Location, standardNotifier.config.DateTimeFormat)
+
+			err := standardNotifier.RegisterSender(senderSettings, sender)
+			So(err, ShouldBeNil)
+		})
 	})
 }