Alerting: Adds support for multiple URLs in Alertmanager notifier (#24196)

* Alerting: Adds support for multiple URLs in Alertmanager notifier

Adds support for multiple URLs in Alertmanager notifier following
alertmanager documentation for high availability setup.
Update the documentation for alertmanager notifier.

Closes #24195

Co-authored-by: Diana Payton <52059945+oddlittlebird@users.noreply.github.com>
Co-authored-by: bergquist <carl.bergquist@gmail.com>

docs/sources/alerting/notifications.md

@@ -196,6 +196,12 @@ Notifications can be sent by setting up an incoming webhook in Google Hangouts c
 
 Squadcast helps you get alerted via Phone call, SMS, Email and Push notifications and lets you take actions on those alerts. Grafana notifications can be sent to Squadcast via a simple incoming webhook. Refer the official [Squadcast support documentation](https://support.squadcast.com/docs/grafana) for configuring these webhooks.
 
+### Prometheus Alertmanager
+
+Alertmanager handles alerts sent by client applications such as Prometheus server or Grafana. It takes care of deduplicating, grouping, and routing them to the correct receiver. Grafana notifications can be sent to Alertmanager via a simple incoming webhook. Refer to the official [Prometheus Alertmanager documentation](https://prometheus.io/docs/alerting/alertmanager) for configuration information.
+
+> **Caution:** In case of a high-availability setup, do not load balance traffic between Grafana and Alertmanagers to keep coherence between all your Alertmanager instances. Instead, point Grafana to a list of all Alertmanagers, by listing their URLs comma-separated in the notification channel configuration.
+
 ## Enable images in notifications {#external-image-store}
 
 Grafana can render the panel associated with the alert rule as a PNG image and include that in the notification. Read more about the requirements and how to configure

pkg/services/alerting/notifiers/alertmanager.go

@@ -3,6 +3,7 @@ package notifiers
 import (
 	"context"
 	"regexp"
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/bus"
@@ -20,17 +21,20 @@ func init() {
 		Factory:     NewAlertmanagerNotifier,
 		OptionsTemplate: `
       <h3 class="page-heading">Alertmanager settings</h3>
-		<div class="gf-form">
-        	<span class="gf-form-label width-10">Url</span>
-        	<input type="text" required class="gf-form-input max-width-26" ng-model="ctrl.model.settings.url" placeholder="http://localhost:9093"></input>
+		<div class="gf-form max-width-30">
+            <span class="gf-form-label width-10">Url(s)</span>
+            <input type="text" required class="gf-form-input max-width-30" ng-model="ctrl.model.settings.url" placeholder="http://localhost:9093"></input>
+            <info-popover mode="right-absolute">
+              As specified in Alertmanager documentation, do not specify a load balancer here. Enter all your Alertmanager URLs comma-separated.
+            </info-popover>
 		</div>
-		<div class="gf-form">
+		<div class="gf-form max-width-30">
         	<span class="gf-form-label width-10">Basic Auth User</span>
-        	<input type="text" class="gf-form-input max-width-26" ng-model="ctrl.model.settings.basicAuthUser" placeholder=""></input>
+            <input type="text" class="gf-form-input max-width-30" ng-model="ctrl.model.settings.basicAuthUser" placeholder=""></input>
 		</div>
-		<div class="gf-form">
+		<div class="gf-form max-width-30">
         	<span class="gf-form-label width-10">Basic Auth Password</span>
-        	<input type="text" class="gf-form-input max-width-26" ng-model="ctrl.model.settings.basicAuthPassword" placeholder=""></input>
+            <input type="text" class="gf-form-input max-width-30" ng-model="ctrl.model.settings.basicAuthPassword" placeholder=""></input>
 		</div>
       </div>
     `,
@@ -39,10 +43,18 @@ func init() {
 
 // NewAlertmanagerNotifier returns a new Alertmanager notifier
 func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier, error) {
-	url := model.Settings.Get("url").MustString()
-	if url == "" {
+	urlString := model.Settings.Get("url").MustString()
+	if urlString == "" {
 		return nil, alerting.ValidationError{Reason: "Could not find url property in settings"}
 	}
+
+	var url []string
+	for _, u := range strings.Split(urlString, ",") {
+		u = strings.TrimSpace(u)
+		if u != "" {
+			url = append(url, u)
+		}
+	}
 	basicAuthUser := model.Settings.Get("basicAuthUser").MustString()
 	basicAuthPassword := model.Settings.Get("basicAuthPassword").MustString()
 
@@ -58,7 +70,7 @@ func NewAlertmanagerNotifier(model *models.AlertNotification) (alerting.Notifier
 // AlertmanagerNotifier sends alert notifications to the alert manager
 type AlertmanagerNotifier struct {
 	NotifierBase
-	URL               string
+	URL               []string
 	BasicAuthUser     string
 	BasicAuthPassword string
 	log               log.Logger
@@ -153,17 +165,19 @@ func (am *AlertmanagerNotifier) Notify(evalContext *alerting.EvalContext) error
 	bodyJSON := simplejson.NewFromAny(alerts)
 	body, _ := bodyJSON.MarshalJSON()
 
-	cmd := &models.SendWebhookSync{
-		Url:        am.URL + "/api/v1/alerts",
-		User:       am.BasicAuthUser,
-		Password:   am.BasicAuthPassword,
-		HttpMethod: "POST",
-		Body:       string(body),
-	}
+	for _, url := range am.URL {
+		cmd := &models.SendWebhookSync{
+			Url:        strings.TrimSuffix(url, "/") + "/api/v1/alerts",
+			User:       am.BasicAuthUser,
+			Password:   am.BasicAuthPassword,
+			HttpMethod: "POST",
+			Body:       string(body),
+		}
 
-	if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
-		am.log.Error("Failed to send alertmanager", "error", err, "alertmanager", am.Name)
-		return err
+		if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil {
+			am.log.Error("Failed to send alertmanager", "error", err, "alertmanager", am.Name, "url", url)
+			return err
+		}
 	}
 
 	return nil

pkg/services/alerting/notifiers/alertmanager_test.go

@@ -97,7 +97,7 @@ func TestAlertmanagerNotifier(t *testing.T) {
 			})
 
 			Convey("from settings", func() {
-				json := `{ "url": "http://127.0.0.1:9093/" }`
+				json := `{ "url": "http://127.0.0.1:9093/", "basicAuthUser": "user", "basicAuthPassword": "password" }`
 
 				settingsJSON, _ := simplejson.NewJson([]byte(json))
 				model := &models.AlertNotification{
@@ -110,7 +110,26 @@ func TestAlertmanagerNotifier(t *testing.T) {
 				alertmanagerNotifier := not.(*AlertmanagerNotifier)
 
 				So(err, ShouldBeNil)
-				So(alertmanagerNotifier.URL, ShouldEqual, "http://127.0.0.1:9093/")
+				So(alertmanagerNotifier.BasicAuthUser, ShouldEqual, "user")
+				So(alertmanagerNotifier.BasicAuthPassword, ShouldEqual, "password")
+				So(alertmanagerNotifier.URL, ShouldResemble, []string{"http://127.0.0.1:9093/"})
+			})
+
+			Convey("from settings with multiple alertmanager", func() {
+				json := `{ "url": "http://alertmanager1:9093,http://alertmanager2:9093" }`
+
+				settingsJSON, _ := simplejson.NewJson([]byte(json))
+				model := &models.AlertNotification{
+					Name:     "alertmanager",
+					Type:     "alertmanager",
+					Settings: settingsJSON,
+				}
+
+				not, err := NewAlertmanagerNotifier(model)
+				alertmanagerNotifier := not.(*AlertmanagerNotifier)
+
+				So(err, ShouldBeNil)
+				So(alertmanagerNotifier.URL, ShouldResemble, []string{"http://alertmanager1:9093", "http://alertmanager2:9093"})
 			})
 		})
 	})