From 3a0f2dc160208fb544cc886b20c17c193105ff0b Mon Sep 17 00:00:00 2001 From: Victor Coutellier Date: Wed, 20 May 2020 08:49:53 +0200 Subject: [PATCH] Alerting: Adds support for multiple URLs in Alertmanager notifier (#24196) * Alerting: Adds support for multiple URLs in Alertmanager notifier Adds support for multiple URLs in Alertmanager notifier following alertmanager documentation for high availability setup. Update the documentation for alertmanager notifier. Closes #24195 Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com> Co-authored-by: bergquist --- docs/sources/alerting/notifications.md | 6 +++ .../alerting/notifiers/alertmanager.go | 54 ++++++++++++------- .../alerting/notifiers/alertmanager_test.go | 23 +++++++- 3 files changed, 61 insertions(+), 22 deletions(-) diff --git a/docs/sources/alerting/notifications.md b/docs/sources/alerting/notifications.md index 359992aae4f..ae78167fb45 100644 --- a/docs/sources/alerting/notifications.md +++ b/docs/sources/alerting/notifications.md @@ -196,6 +196,12 @@ Notifications can be sent by setting up an incoming webhook in Google Hangouts c Squadcast helps you get alerted via Phone call, SMS, Email and Push notifications and lets you take actions on those alerts. Grafana notifications can be sent to Squadcast via a simple incoming webhook. Refer the official [Squadcast support documentation](https://support.squadcast.com/docs/grafana) for configuring these webhooks. +### Prometheus Alertmanager + +Alertmanager handles alerts sent by client applications such as Prometheus server or Grafana. It takes care of deduplicating, grouping, and routing them to the correct receiver. Grafana notifications can be sent to Alertmanager via a simple incoming webhook. Refer to the official [Prometheus Alertmanager documentation](https://prometheus.io/docs/alerting/alertmanager) for configuration information. + +> **Caution:** In case of a high-availability setup, do not load balance traffic between Grafana and Alertmanagers to keep coherence between all your Alertmanager instances. Instead, point Grafana to a list of all Alertmanagers, by listing their URLs comma-separated in the notification channel configuration. + ## Enable images in notifications {#external-image-store} Grafana can render the panel associated with the alert rule as a PNG image and include that in the notification. Read more about the requirements and how to configure diff --git a/pkg/services/alerting/notifiers/alertmanager.go b/pkg/services/alerting/notifiers/alertmanager.go index e11065495b0..0b7ea2efca1 100644 --- a/pkg/services/alerting/notifiers/alertmanager.go +++ b/pkg/services/alerting/notifiers/alertmanager.go @@ -3,6 +3,7 @@ package notifiers import ( "context" "regexp" + "strings" "time" "github.com/grafana/grafana/pkg/bus" @@ -20,17 +21,20 @@ func init() { Factory: NewAlertmanagerNotifier, OptionsTemplate: `

Alertmanager settings

-
- Url - +
+ Url(s) + + + As specified in Alertmanager documentation, do not specify a load balancer here. Enter all your Alertmanager URLs comma-separated. +
-
+
Basic Auth User - +
-
+
Basic Auth Password - +
`, @@ -39,10 +43,18 @@ func init() { // NewAlertmanagerNotifier returns a new Alertmanager notifier func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier, error) { - url := model.Settings.Get("url").MustString() - if url == "" { + urlString := model.Settings.Get("url").MustString() + if urlString == "" { return nil, alerting.ValidationError{Reason: "Could not find url property in settings"} } + + var url []string + for _, u := range strings.Split(urlString, ",") { + u = strings.TrimSpace(u) + if u != "" { + url = append(url, u) + } + } basicAuthUser := model.Settings.Get("basicAuthUser").MustString() basicAuthPassword := model.Settings.Get("basicAuthPassword").MustString() @@ -58,7 +70,7 @@ func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier // AlertmanagerNotifier sends alert notifications to the alert manager type AlertmanagerNotifier struct { NotifierBase - URL string + URL []string BasicAuthUser string BasicAuthPassword string log log.Logger @@ -153,17 +165,19 @@ func (am *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error bodyJSON := simplejson.NewFromAny(alerts) body, _ := bodyJSON.MarshalJSON() - cmd := &models.SendWebhookSync{ - Url: am.URL + "/api/v1/alerts", - User: am.BasicAuthUser, - Password: am.BasicAuthPassword, - HttpMethod: "POST", - Body: string(body), - } + for _, url := range am.URL { + cmd := &models.SendWebhookSync{ + Url: strings.TrimSuffix(url, "/") + "/api/v1/alerts", + User: am.BasicAuthUser, + Password: am.BasicAuthPassword, + HttpMethod: "POST", + Body: string(body), + } - if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { - am.log.Error("Failed to send alertmanager", "error", err, "alertmanager", am.Name) - return err + if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { + am.log.Error("Failed to send alertmanager", "error", err, "alertmanager", am.Name, "url", url) + return err + } } return nil diff --git a/pkg/services/alerting/notifiers/alertmanager_test.go b/pkg/services/alerting/notifiers/alertmanager_test.go index 1ee12e76cb9..fe9649443c1 100644 --- a/pkg/services/alerting/notifiers/alertmanager_test.go +++ b/pkg/services/alerting/notifiers/alertmanager_test.go @@ -97,7 +97,7 @@ func TestAlertmanagerNotifier(t *testing.T) { }) Convey("from settings", func() { - json := `{ "url": "http://127.0.0.1:9093/" }` + json := `{ "url": "http://127.0.0.1:9093/", "basicAuthUser": "user", "basicAuthPassword": "password" }` settingsJSON, _ := simplejson.NewJson([]byte(json)) model := &models.AlertNotification{ @@ -110,7 +110,26 @@ func TestAlertmanagerNotifier(t *testing.T) { alertmanagerNotifier := not.(*AlertmanagerNotifier) So(err, ShouldBeNil) - So(alertmanagerNotifier.URL, ShouldEqual, "http://127.0.0.1:9093/") + So(alertmanagerNotifier.BasicAuthUser, ShouldEqual, "user") + So(alertmanagerNotifier.BasicAuthPassword, ShouldEqual, "password") + So(alertmanagerNotifier.URL, ShouldResemble, []string{"http://127.0.0.1:9093/"}) + }) + + Convey("from settings with multiple alertmanager", func() { + json := `{ "url": "http://alertmanager1:9093,http://alertmanager2:9093" }` + + settingsJSON, _ := simplejson.NewJson([]byte(json)) + model := &models.AlertNotification{ + Name: "alertmanager", + Type: "alertmanager", + Settings: settingsJSON, + } + + not, err := NewAlertmanagerNotifier(model) + alertmanagerNotifier := not.(*AlertmanagerNotifier) + + So(err, ShouldBeNil) + So(alertmanagerNotifier.URL, ShouldResemble, []string{"http://alertmanager1:9093", "http://alertmanager2:9093"}) }) }) })