Slack Notifier

config.go

@@ -4,28 +4,33 @@ import (
 	"os"
 	"time"
 
-	"gopkg.in/yaml.v2"
 	"github.com/prometheus/alertmanager/template"
+	"gopkg.in/yaml.v2"
 )
 
 type Config struct {
-	Interval time.Duration
-	Notify   *Notify
-	Evaluate *Evaluate
+	Interval time.Duration `yaml:"interval"`
+	Notify   *Notify       `yaml:"notify"`
+	Evaluate *Evaluate     `yaml:"evaluate"`
 }
 
 type Notify struct {
-	Pagerduty *Pagerduty
+	Pagerduty *PagerdutyConfig `yaml:"pagerduty"`
+	Slack     *SlackConfig     `yaml:"slack"`
+}
+
+type PagerdutyConfig struct {
+	Key string `yaml:"key"`
 }
 
-type Pagerduty struct {
-	Key string
+type SlackConfig struct {
+	WebhookURL string `yaml:"webhookurl"`
 }
 
 type EvaluateType string
 
-const(
-	EvaluateEqual EvaluateType = "equal"
+const (
+	EvaluateEqual   EvaluateType = "equal"
 	EvaluateInclude EvaluateType = "include"
 )
 
@@ -35,12 +40,14 @@ type Evaluate struct {
 }
 
 func ParseConfig(path string) (*Config, error) {
-	file, err := os.Open(path)
+	content, err := os.ReadFile(path)
 	if err != nil {
 		return nil, err
 	}
+	// Expand environment variables if possible
+	content = []byte(os.ExpandEnv(string(content)))
 	config := &Config{}
-	err = yaml.NewDecoder(file).Decode(config)
+	err = yaml.Unmarshal(content, config)
 	if err != nil {
 		return nil, err
 	}

dead_mans_switch.go

@@ -41,19 +41,19 @@ func init() {
 }
 
 type DeadmansSwitch struct {
-	message  <-chan string
-	interval time.Duration
-	ticker   *time.Ticker
-	closer   chan struct{}
-	notifier func(summary, detail string) error
+	message   <-chan string
+	interval  time.Duration
+	ticker    *time.Ticker
+	closer    chan struct{}
+	notifiers []NotifyInterface
 }
 
-func NewDeadMansSwitch(message <-chan string, interval time.Duration, notifier func(summary, detail string) error) *DeadmansSwitch {
+func NewDeadMansSwitch(message <-chan string, interval time.Duration, notifiers []NotifyInterface) *DeadmansSwitch {
 	return &DeadmansSwitch{
-		message:  message,
-		interval: interval,
-		notifier: notifier,
-		closer:   make(chan struct{}),
+		message:   message,
+		interval:  interval,
+		notifiers: notifiers,
+		closer:    make(chan struct{}),
 	}
 }
 
@@ -62,7 +62,7 @@ func (d *DeadmansSwitch) Run() error {
 	d.ticker = time.NewTicker(d.interval)
 
 	skip := false
-
+outer:
 	for {
 		select {
 		case <-d.ticker.C:
@@ -85,16 +85,19 @@ func (d *DeadmansSwitch) Run() error {
 			}
 
 		case <-d.closer:
-			break
+			break outer
 		}
 	}
+	return nil
 }
 
 // Notify send special message to notifier
 func (d *DeadmansSwitch) Notify(summary, detail string) {
-	if err := d.notifier(summary, detail); err != nil {
-		failedNotifications.Inc()
-		fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+	for _, notifier := range d.notifiers {
+		if err := notifier.Notify(summary, detail); err != nil {
+			failedNotifications.Inc()
+			fmt.Fprintf(os.Stderr, "%s\n", err.Error())
+		}
 	}
 }
 

dead_mans_switch_test.go

@@ -7,23 +7,33 @@ import (
 	"github.com/prometheus/client_golang/prometheus/testutil"
 )
 
+type testNotifier struct {
+	Summary string
+	Detail  string
+}
+
+func (tn *testNotifier) Notify(summary, detail string) error {
+	tn.Summary = summary
+	tn.Detail = detail
+	return nil
+}
+
 func TestDeadMansSwitchDoesntTrigger(t *testing.T) {
 	evaluateMessage := make(chan string)
 	defer close(evaluateMessage)
 
-	notify := ""
-	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, func(summary, detail string) error {
-		notify = summary
-		return nil
-	})
+	tn := testNotifier{}
+	notifiers := []NotifyInterface{&tn}
+
+	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, notifiers)
 
 	go d.Run()
 	defer d.Stop()
 
 	evaluateMessage <- ""
 
 	time.Sleep(150 * time.Millisecond)
-	if notify != "" {
+	if tn.Summary != "" {
 		t.Fatal("deadman should not trigger!")
 	}
 }
@@ -32,18 +42,16 @@ func TestDeadMansSwitchTrigger(t *testing.T) {
 	evaluateMessage := make(chan string)
 	defer close(evaluateMessage)
 
-	notify := ""
-	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, func(summary, detail string) error {
-		notify = summary
+	tn := testNotifier{}
+	notifiers := []NotifyInterface{&tn}
 
-		return nil
-	})
+	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, notifiers)
 
 	go d.Run()
 	defer d.Stop()
 
 	time.Sleep(150 * time.Millisecond)
-	if notify != "WatchdogDown" {
+	if tn.Summary != "WatchdogDown" {
 		t.Fatal("deadman should trigger!")
 	}
 }
@@ -52,9 +60,10 @@ func TestEvaluateMessageNotNull(t *testing.T) {
 	evaluateMessage := make(chan string)
 	defer close(evaluateMessage)
 
-	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, func(summary, detail string) error {
-		return nil
-	})
+	tn := testNotifier{}
+	notifiers := []NotifyInterface{&tn}
+
+	d := NewDeadMansSwitch(evaluateMessage, 100*time.Millisecond, notifiers)
 
 	go d.Run()
 	defer d.Stop()

main.go

@@ -37,8 +37,16 @@ func main() {
 		log.Fatal(s.ListenAndServe())
 	}()
 
-	pagerDuty := NewPagerDutyNotify(config.Notify.Pagerduty.Key)
-	dms := NewDeadMansSwitch(evaluateMessage, config.Interval, pagerDuty.Notify)
+	var notifiers []NotifyInterface
+	if config.Notify.Pagerduty != nil {
+		pagerDuty := NewPagerDutyNotify(config.Notify.Pagerduty.Key)
+		notifiers = append(notifiers, pagerDuty)
+	}
+	if config.Notify.Slack != nil {
+		slack := NewSlackNotify(config.Notify.Slack.WebhookURL)
+		notifiers = append(notifiers, slack)
+	}
+	dms := NewDeadMansSwitch(evaluateMessage, config.Interval, notifiers)
 	go dms.Run()
 
 	stop := make(chan os.Signal, 1)

manifest/deploy/configmap-dead-mans-switch-config.yaml

@@ -1,12 +1,14 @@
 apiVersion: v1
+kind: ConfigMap
+metadata:
+  name: dead-mans-switch-config-t13uziow
+  namespace: monitoring
 data:
   config.yaml: |2-
 
     interval: 5m
     notify:
       pagerduty:
         key: <YOUR_PAGERDUTY_KEY>
-kind: ConfigMap
-metadata:
-  name: dead-mans-switch-config-t13uziow
-  namespace: monitoring
+      slack:
+        webhookurl: <YOUR_SLACK_WEBHOOK_URL>

notify_slack.go

@@ -0,0 +1,74 @@
+package main
+
+import (
+	"bytes"
+	"fmt"
+	"log"
+	"net/http"
+	"time"
+)
+
+var _ NotifyInterface = new(Slack)
+
+type Slack struct {
+	WebhookURL string
+}
+
+func NewSlackNotify(webhookURL string) *Slack {
+	return &Slack{WebhookURL: webhookURL}
+}
+
+// Notify send notify message to Slack
+func (s *Slack) Notify(summary, detail string) error {
+	log.Printf("sending notify: %s to slack\n", summary)
+
+	rendered := fmt.Sprintf(slackTmpl, summary, detail, time.Now().Format(time.RFC3339))
+
+	req, err := http.NewRequest(http.MethodPost, s.WebhookURL, bytes.NewBuffer([]byte(rendered)))
+	if err != nil {
+		return err
+	}
+
+	req.Header.Add("Content-Type", "application/json")
+
+	client := &http.Client{Timeout: 10 * time.Second}
+	resp, err := client.Do(req)
+	if err != nil {
+		return err
+	}
+
+	buf := new(bytes.Buffer)
+	buf.ReadFrom(resp.Body)
+
+	if resp.StatusCode >= 400 || buf.String() != "ok" {
+		return fmt.Errorf("failed to send message to slack. status: %d - %s, body: %s", resp.StatusCode, resp.Status, buf.String())
+	}
+
+	return nil
+}
+
+const slackTmpl = `
+{
+	"blocks": [
+		{
+			"type": "section",
+			"text": {
+				"type": "mrkdwn",
+				"text": "DeadMansSwitch - %s"
+			}
+		},
+		{
+			"type": "section",
+			"fields": [
+				{
+					"type": "mrkdwn",
+					"text": "*Details:*\n%s"
+				},
+				{
+					"type": "mrkdwn",
+					"text": "*When:*\n%s"
+				}
+			]
+		}
+	]
+}`