Alerting: Do not hard fail on templating errors in channels (#35165)

* Alerting: Do not hard fail on templating errors in channels

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

* Fix review

Signed-off-by: Ganesh Vernekar <ganeshvern@gmail.com>

pkg/services/ngalert/notifier/channels/dingding.go

@@ -61,10 +61,7 @@ type DingDingNotifier struct {
 func (dd *DingDingNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 	dd.log.Info("Sending dingding")
 
-	ruleURL, err := joinUrlPath(dd.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(dd.tmpl.ExternalURL.String(), "/alerting/list", dd.log)
 
 	q := url.Values{
 		"pc_slide": {"false"},
@@ -76,10 +73,7 @@ func (dd *DingDingNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
 	messageURL := "dingtalk://dingtalkclient/page/link?" + q.Encode()
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, dd.tmpl, as, dd.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, dd.tmpl, as, dd.log, &tmplErr)
 
 	message := tmpl(dd.Message)
 	title := tmpl(`{{ template "default.title" . }}`)
@@ -109,7 +103,7 @@ func (dd *DingDingNotifier) Notify(ctx context.Context, as ...*types.Alert) (boo
 	}
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template DingDing message: %w", tmplErr)
+		dd.log.Debug("failed to template DingDing message", "err", tmplErr.Error())
 	}
 
 	body, err := json.Marshal(bodyMsg)

pkg/services/ngalert/notifier/channels/dingding_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"net/url"
 	"testing"
 
@@ -89,13 +88,6 @@ func TestDingdingNotifier(t *testing.T) {
 			name:         "Error in initing",
 			settings:     `{}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find url property in settings"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"url": "http://localhost",
-				"message": "{{ .Status }"
-			}`,
-			expMsgError: errors.New("failed to template DingDing message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/discord.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 	"strconv"
 	"strings"
 
@@ -66,10 +65,8 @@ func (d DiscordNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	bodyJSON.Set("username", "Grafana")
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, d.tmpl, as, d.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, d.tmpl, as, d.log, &tmplErr)
+
 	if d.Content != "" {
 		bodyJSON.Set("content", tmpl(d.Content))
 	}
@@ -91,16 +88,13 @@ func (d DiscordNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	color, _ := strconv.ParseInt(strings.TrimLeft(getAlertStatusColor(alerts.Status()), "#"), 16, 0)
 	embed.Set("color", color)
 
-	ruleURL, err := joinUrlPath(d.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(d.tmpl.ExternalURL.String(), "/alerting/list", d.log)
 	embed.Set("url", ruleURL)
 
 	bodyJSON.Set("embeds", []interface{}{embed})
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template discord message: %w", tmplErr)
+		d.log.Debug("failed to template Discord message", "err", tmplErr.Error())
 	}
 
 	body, err := json.Marshal(bodyJSON)

pkg/services/ngalert/notifier/channels/discord_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"net/url"
 	"testing"
 
@@ -104,14 +103,6 @@ func TestDiscordNotifier(t *testing.T) {
 			settings:     `{}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find webhook url property in settings"},
 		},
-		{
-			name: "Error in building messsage",
-			settings: `{
-				"url": "http://localhost",
-				"message": "{{ .Status }"
-			}`,
-			expMsgError: errors.New("failed to template discord message: template: :1: unexpected \"}\" in operand"),
-		},
 	}
 
 	for _, c := range cases {

pkg/services/ngalert/notifier/channels/email.go

@@ -2,7 +2,6 @@ package channels
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"path"
 
@@ -64,22 +63,22 @@ func NewEmailNotifier(model *NotificationChannelConfig, t *template.Template) (*
 // Notify sends the alert notification.
 func (en *EmailNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 	var tmplErr error
-	tmpl, data, err := TmplText(ctx, en.tmpl, as, en.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, data := TmplText(ctx, en.tmpl, as, en.log, &tmplErr)
 
 	title := tmpl(`{{ template "default.title" . }}`)
 
+	alertPageURL := en.tmpl.ExternalURL.String()
+	ruleURL := en.tmpl.ExternalURL.String()
 	u, err := url.Parse(en.tmpl.ExternalURL.String())
-	if err != nil {
-		return false, fmt.Errorf("failed to parse external URL: %w", err)
+	if err == nil {
+		basePath := u.Path
+		u.Path = path.Join(basePath, "/alerting/list")
+		ruleURL = u.String()
+		u.RawQuery = "alertState=firing&view=state"
+		alertPageURL = u.String()
+	} else {
+		en.log.Debug("failed to parse external URL", "url", en.tmpl.ExternalURL.String(), "err", err.Error())
 	}
-	basePath := u.Path
-	u.Path = path.Join(basePath, "/alerting/list")
-	ruleURL := u.String()
-	u.RawQuery = "alertState=firing&view=state"
-	alertPageURL := u.String()
 
 	cmd := &models.SendEmailCommandSync{
 		SendEmailCommand: models.SendEmailCommand{
@@ -103,7 +102,7 @@ func (en *EmailNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	}
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template email message: %w", tmplErr)
+		en.log.Debug("failed to template email message", "err", tmplErr.Error())
 	}
 
 	if err := bus.DispatchCtx(ctx, cmd); err != nil {

pkg/services/ngalert/notifier/channels/googlechat.go

@@ -51,10 +51,7 @@ func (gcn *GoogleChatNotifier) Notify(ctx context.Context, as ...*types.Alert) (
 	gcn.log.Debug("Executing Google Chat notification")
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, gcn.tmpl, as, gcn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, gcn.tmpl, as, gcn.log, &tmplErr)
 
 	widgets := []widget{}
 
@@ -68,10 +65,7 @@ func (gcn *GoogleChatNotifier) Notify(ctx context.Context, as ...*types.Alert) (
 		})
 	}
 
-	ruleURL, err := joinUrlPath(gcn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(gcn.tmpl.ExternalURL.String(), "/alerting/list", gcn.log)
 	// Add a button widget (link to Grafana).
 	widgets = append(widgets, buttonWidget{
 		Buttons: []button{
@@ -114,7 +108,7 @@ func (gcn *GoogleChatNotifier) Notify(ctx context.Context, as ...*types.Alert) (
 	}
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template GoogleChat message: %w", tmplErr)
+		gcn.log.Debug("failed to template GoogleChat message", "err", tmplErr.Error())
 	}
 
 	body, err := json.Marshal(res)

pkg/services/ngalert/notifier/channels/kafka.go

@@ -2,7 +2,6 @@ package channels
 
 import (
 	"context"
-	"fmt"
 	"strings"
 
 	"github.com/prometheus/alertmanager/notify"
@@ -67,10 +66,7 @@ func (kn *KafkaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	kn.log.Debug("Notifying Kafka", "alert_state", state)
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, kn.tmpl, as, kn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, kn.tmpl, as, kn.log, &tmplErr)
 
 	bodyJSON := simplejson.New()
 	bodyJSON.Set("alert_state", state)
@@ -78,10 +74,7 @@ func (kn *KafkaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	bodyJSON.Set("client", "Grafana")
 	bodyJSON.Set("details", tmpl(`{{ template "default.message" . }}`))
 
-	ruleURL, err := joinUrlPath(kn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(kn.tmpl.ExternalURL.String(), "/alerting/list", kn.log)
 	bodyJSON.Set("client_url", ruleURL)
 
 	groupKey, err := notify.ExtractGroupKey(ctx)
@@ -97,7 +90,7 @@ func (kn *KafkaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	recordJSON.Set("records", []interface{}{valueJSON})
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template Kafka message: %w", tmplErr)
+		kn.log.Debug("failed to template Kafka message", "err", tmplErr.Error())
 	}
 
 	body, err := recordJSON.MarshalJSON()

pkg/services/ngalert/notifier/channels/line.go

@@ -57,10 +57,7 @@ func (ln *LineNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, e
 	ruleURL := path.Join(ln.tmpl.ExternalURL.String(), "/alerting/list")
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, ln.tmpl, as, ln.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, ln.tmpl, as, ln.log, &tmplErr)
 
 	body := fmt.Sprintf(
 		"%s\n%s\n\n%s",
@@ -69,7 +66,7 @@ func (ln *LineNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, e
 		tmpl(`{{ template "default.message" . }}`),
 	)
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template Line message: %w", tmplErr)
+		ln.log.Debug("failed to template Line message", "err", tmplErr.Error())
 	}
 
 	form := url.Values{}

pkg/services/ngalert/notifier/channels/opsgenie.go

@@ -148,16 +148,10 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
 		return nil, "", nil
 	}
 
-	ruleURL, err := joinUrlPath(on.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return nil, "", err
-	}
+	ruleURL := joinUrlPath(on.tmpl.ExternalURL.String(), "/alerting/list", on.log)
 
 	var tmplErr error
-	tmpl, data, err := TmplText(ctx, on.tmpl, as, on.log, &tmplErr)
-	if err != nil {
-		return nil, "", err
-	}
+	tmpl, data := TmplText(ctx, on.tmpl, as, on.log, &tmplErr)
 
 	title := tmpl(`{{ template "default.title" . }}`)
 	description := fmt.Sprintf(
@@ -210,10 +204,10 @@ func (on *OpsgenieNotifier) buildOpsgenieMessage(ctx context.Context, alerts mod
 	apiURL = on.APIUrl
 
 	if tmplErr != nil {
-		return nil, "", fmt.Errorf("failed to template Opsgenie message: %w", tmplErr)
+		on.log.Debug("failed to template Opsgenie message", "err", tmplErr.Error())
 	}
 
-	return bodyJSON, apiURL, err
+	return bodyJSON, apiURL, nil
 }
 
 func (on *OpsgenieNotifier) SendResolved() bool {

pkg/services/ngalert/notifier/channels/pagerduty.go

@@ -124,10 +124,7 @@ func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts m
 	}
 
 	var tmplErr error
-	tmpl, data, err := TmplText(ctx, pn.tmpl, as, pn.log, &tmplErr)
-	if err != nil {
-		return nil, "", err
-	}
+	tmpl, data := TmplText(ctx, pn.tmpl, as, pn.log, &tmplErr)
 
 	details := make(map[string]string, len(pn.CustomDetails))
 	for k, v := range pn.CustomDetails {
@@ -170,7 +167,7 @@ func (pn *PagerdutyNotifier) buildPagerdutyMessage(ctx context.Context, alerts m
 	}
 
 	if tmplErr != nil {
-		return nil, "", fmt.Errorf("failed to template PagerDuty message: %w", tmplErr)
+		pn.log.Debug("failed to template PagerDuty message", "err", tmplErr.Error())
 	}
 
 	return msg, eventType, nil

pkg/services/ngalert/notifier/channels/pagerduty_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"net/url"
 	"os"
 	"testing"
@@ -124,13 +123,6 @@ func TestPagerdutyNotifier(t *testing.T) {
 			name:         "Error in initing",
 			settings:     `{}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find integration key property in settings"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"integrationKey": "abcdefgh0123456789",
-				"class": "{{ .Status }"
-			}`,
-			expMsgError: errors.New("build pagerduty message: failed to template PagerDuty message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/pushover.go

@@ -12,7 +12,6 @@ import (
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/alerting"
 	old_notifiers "github.com/grafana/grafana/pkg/services/alerting/notifiers"
-	"github.com/pkg/errors"
 	"github.com/prometheus/alertmanager/template"
 	"github.com/prometheus/alertmanager/types"
 	"github.com/prometheus/common/model"
@@ -124,30 +123,24 @@ func (pn *PushoverNotifier) SendResolved() bool {
 func (pn *PushoverNotifier) genPushoverBody(ctx context.Context, as ...*types.Alert) (map[string]string, bytes.Buffer, error) {
 	var b bytes.Buffer
 
-	ruleURL, err := joinUrlPath(pn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return nil, b, err
-	}
+	ruleURL := joinUrlPath(pn.tmpl.ExternalURL.String(), "/alerting/list", pn.log)
 
 	alerts := types.Alerts(as...)
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, pn.tmpl, as, pn.log, &tmplErr)
-	if err != nil {
-		return nil, b, err
-	}
+	tmpl, _ := TmplText(ctx, pn.tmpl, as, pn.log, &tmplErr)
 
 	w := multipart.NewWriter(&b)
 	boundary := GetBoundary()
 	if boundary != "" {
-		err = w.SetBoundary(boundary)
+		err := w.SetBoundary(boundary)
 		if err != nil {
 			return nil, b, err
 		}
 	}
 
 	// Add the user token
-	err = w.WriteField("user", pn.UserKey)
+	err := w.WriteField("user", pn.UserKey)
 	if err != nil {
 		return nil, b, err
 	}
@@ -224,7 +217,7 @@ func (pn *PushoverNotifier) genPushoverBody(ctx context.Context, as ...*types.Al
 	}
 
 	if tmplErr != nil {
-		return nil, b, errors.Wrap(tmplErr, "failed to template pushover message")
+		pn.log.Debug("failed to template pushover message", "err", tmplErr.Error())
 	}
 
 	// Mark as html message

pkg/services/ngalert/notifier/channels/pushover_test.go

@@ -121,14 +121,6 @@ func TestPushoverNotifier(t *testing.T) {
 				"userKey": "<userKey>"
 			}`,
 			expInitError: alerting.ValidationError{Reason: "API token not found"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"apiToken": "<apiToken>",
-				"userKey": "<userKey>",
-				"message": "{{ .BrokenTemplate }"
-			}`,
-			expMsgError: errors.New("failed to template pushover message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/sensugo.go

@@ -73,10 +73,7 @@ func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 	sn.log.Debug("Sending Sensu Go result")
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, sn.tmpl, as, sn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, sn.tmpl, as, sn.log, &tmplErr)
 
 	// Sensu Go alerts require an entity and a check. We set it to the user-specified
 	// value (optional), else we fallback and use the grafana rule anme  and ruleID.
@@ -107,10 +104,7 @@ func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 		handlers = []string{sn.Handler}
 	}
 
-	ruleURL, err := joinUrlPath(sn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(sn.tmpl.ExternalURL.String(), "/alerting/list", sn.log)
 	bodyMsgType := map[string]interface{}{
 		"entity": map[string]interface{}{
 			"metadata": map[string]interface{}{
@@ -135,7 +129,7 @@ func (sn *SensuGoNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 	}
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template sensugo message: %w", tmplErr)
+		sn.log.Debug("failed to template sensugo message", "err", tmplErr.Error())
 	}
 
 	body, err := json.Marshal(bodyMsgType)

pkg/services/ngalert/notifier/channels/sensugo_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"net/url"
 	"testing"
 	"time"
@@ -129,14 +128,6 @@ func TestSensuGoNotifier(t *testing.T) {
 				"url": "http://sensu-api.local:8080"
 			}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find the API key property in settings"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"url": "http://sensu-api.local:8080",
-				"apikey": "<apikey>",
-				"message": "{{ .Status }"
-			}`,
-			expMsgError: errors.New("failed to template sensugo message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/slack.go

@@ -244,15 +244,9 @@ var sendSlackRequest = func(request *http.Request, logger log.Logger) error {
 func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, as []*types.Alert) (*slackMessage, error) {
 	alerts := types.Alerts(as...)
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, sn.tmpl, as, sn.log, &tmplErr)
-	if err != nil {
-		return nil, err
-	}
+	tmpl, _ := TmplText(ctx, sn.tmpl, as, sn.log, &tmplErr)
 
-	ruleURL, err := joinUrlPath(sn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return nil, err
-	}
+	ruleURL := joinUrlPath(sn.tmpl.ExternalURL.String(), "/alerting/list", sn.log)
 
 	req := &slackMessage{
 		Channel:   tmpl(sn.Recipient),
@@ -274,7 +268,7 @@ func (sn *SlackNotifier) buildSlackMessage(ctx context.Context, as []*types.Aler
 		},
 	}
 	if tmplErr != nil {
-		return nil, fmt.Errorf("failed to template Slack message: %w", tmplErr)
+		sn.log.Debug("failed to template Slack message", "err", tmplErr.Error())
 	}
 
 	mentionsBuilder := strings.Builder{}

pkg/services/ngalert/notifier/channels/slack_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"io"
 	"net/http"
 	"net/url"
@@ -160,13 +159,6 @@ func TestSlackNotifier(t *testing.T) {
 				"token": "1234"
 			}`,
 			expInitError: alerting.ValidationError{Reason: "recipient must be specified when using the Slack chat API"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"url": "https://test.slack.com",
-				"title": "{{ .BrokenTemplate }"
-			}`,
-			expMsgError: errors.New("build slack message: failed to template Slack message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/teams.go

@@ -54,15 +54,9 @@ func NewTeamsNotifier(model *NotificationChannelConfig, t *template.Template) (*
 // Notify send an alert notification to Microsoft teams.
 func (tn *TeamsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool, error) {
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
 
-	ruleURL, err := joinUrlPath(tn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(tn.tmpl.ExternalURL.String(), "/alerting/list", tn.log)
 
 	title := tmpl(`{{ template "default.title" . }}`)
 	body := map[string]interface{}{
@@ -95,7 +89,7 @@ func (tn *TeamsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool,
 	}
 
 	if tmplErr != nil {
-		return false, errors.Wrap(tmplErr, "failed to template Teams message")
+		tn.log.Debug("failed to template Teams message", "err", tmplErr.Error())
 	}
 
 	b, err := json.Marshal(&body)

pkg/services/ngalert/notifier/channels/teams_test.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"errors"
 	"net/url"
 	"testing"
 
@@ -113,13 +112,6 @@ func TestTeamsNotifier(t *testing.T) {
 			name:         "Error in initing",
 			settings:     `{}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find url property in settings"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"url": "http://localhost",
-				"message": "{{ .Status }"
-			}`,
-			expMsgError: errors.New("failed to template Teams message: template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/telegram.go

@@ -123,14 +123,11 @@ func (tn *TelegramNotifier) buildTelegramMessage(ctx context.Context, as []*type
 	msg["parse_mode"] = "html"
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
-	if err != nil {
-		return nil, err
-	}
+	tmpl, _ := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
 
 	message := tmpl(tn.Message)
 	if tmplErr != nil {
-		return nil, tmplErr
+		tn.log.Debug("failed to template Telegram message", "err", tmplErr.Error())
 	}
 
 	msg["text"] = message

pkg/services/ngalert/notifier/channels/telegram_test.go

@@ -2,7 +2,6 @@ package channels
 
 import (
 	"context"
-	"errors"
 	"net/url"
 	"testing"
 
@@ -84,14 +83,6 @@ func TestTelegramNotifier(t *testing.T) {
 			name:         "Error in initing",
 			settings:     `{}`,
 			expInitError: alerting.ValidationError{Reason: "Could not find Bot Token in settings"},
-		}, {
-			name: "Error in building message",
-			settings: `{
-				"bottoken": "abcdefgh0123456789",
-				"chatid": "someid",
-				"message": "{{ .BrokenTemplate }"
-			}`,
-			expMsgError: errors.New("template: :1: unexpected \"}\" in operand"),
 		},
 	}
 

pkg/services/ngalert/notifier/channels/template_data.go

@@ -2,7 +2,6 @@ package channels
 
 import (
 	"context"
-	"fmt"
 	"net/url"
 	"path"
 	"sort"
@@ -55,11 +54,12 @@ func removePrivateItems(kv template.KV) template.KV {
 	return kv
 }
 
-func extendAlert(alert template.Alert, externalURL string) (*ExtendedAlert, error) {
-	extended := ExtendedAlert{
+func extendAlert(alert template.Alert, externalURL string, logger log.Logger) *ExtendedAlert {
+	// remove "private" annotations & labels so they don't show up in the template
+	extended := &ExtendedAlert{
 		Status:       alert.Status,
-		Labels:       alert.Labels,
-		Annotations:  alert.Annotations,
+		Labels:       removePrivateItems(alert.Labels),
+		Annotations:  removePrivateItems(alert.Annotations),
 		StartsAt:     alert.StartsAt,
 		EndsAt:       alert.EndsAt,
 		GeneratorURL: alert.GeneratorURL,
@@ -67,50 +67,45 @@ func extendAlert(alert template.Alert, externalURL string) (*ExtendedAlert, erro
 	}
 
 	// fill in some grafana-specific urls
-	if len(externalURL) > 0 {
-		u, err := url.Parse(externalURL)
-		if err != nil {
-			return nil, fmt.Errorf("failed to parse external URL: %w", err)
-		}
-		externalPath := u.Path
-		dashboardUid := alert.Annotations["__dashboardUid__"]
-		if len(dashboardUid) > 0 {
-			u.Path = path.Join(externalPath, "/d/", dashboardUid)
-			extended.DashboardURL = u.String()
-			panelId := alert.Annotations["__panelId__"]
-			if len(panelId) > 0 {
-				u.RawQuery = "viewPanel=" + panelId
-				extended.PanelURL = u.String()
-			}
+	if len(externalURL) == 0 {
+		return extended
+	}
+	u, err := url.Parse(externalURL)
+	if err != nil {
+		logger.Debug("failed to parse external URL while extending template data", "url", externalURL, "err", err.Error())
+		return extended
+	}
+	externalPath := u.Path
+	dashboardUid := alert.Annotations["__dashboardUid__"]
+	if len(dashboardUid) > 0 {
+		u.Path = path.Join(externalPath, "/d/", dashboardUid)
+		extended.DashboardURL = u.String()
+		panelId := alert.Annotations["__panelId__"]
+		if len(panelId) > 0 {
+			u.RawQuery = "viewPanel=" + panelId
+			extended.PanelURL = u.String()
 		}
+	}
 
-		matchers := make([]string, 0)
-		for key, value := range alert.Labels {
-			if !(strings.HasPrefix(key, "__") && strings.HasSuffix(key, "__")) {
-				matchers = append(matchers, key+"="+value)
-			}
+	matchers := make([]string, 0)
+	for key, value := range alert.Labels {
+		if !(strings.HasPrefix(key, "__") && strings.HasSuffix(key, "__")) {
+			matchers = append(matchers, key+"="+value)
 		}
-		sort.Strings(matchers)
-		u.Path = path.Join(externalPath, "/alerting/silence/new")
-		u.RawQuery = "alertmanager=grafana&matchers=" + url.QueryEscape(strings.Join(matchers, ","))
-		extended.SilenceURL = u.String()
 	}
+	sort.Strings(matchers)
+	u.Path = path.Join(externalPath, "/alerting/silence/new")
+	u.RawQuery = "alertmanager=grafana&matchers=" + url.QueryEscape(strings.Join(matchers, ","))
+	extended.SilenceURL = u.String()
 
-	// remove "private" annotations & labels so they don't show up in the template
-	extended.Annotations = removePrivateItems(extended.Annotations)
-	extended.Labels = removePrivateItems(extended.Labels)
-
-	return &extended, nil
+	return extended
 }
 
-func ExtendData(data *template.Data) (*ExtendedData, error) {
+func ExtendData(data *template.Data, logger log.Logger) *ExtendedData {
 	alerts := []ExtendedAlert{}
 
 	for _, alert := range data.Alerts {
-		extendedAlert, err := extendAlert(alert, data.ExternalURL)
-		if err != nil {
-			return nil, err
-		}
+		extendedAlert := extendAlert(alert, data.ExternalURL, logger)
 		alerts = append(alerts, *extendedAlert)
 	}
 
@@ -124,15 +119,12 @@ func ExtendData(data *template.Data) (*ExtendedData, error) {
 
 		ExternalURL: data.ExternalURL,
 	}
-	return extended, nil
+	return extended
 }
 
-func TmplText(ctx context.Context, tmpl *template.Template, alerts []*types.Alert, l log.Logger, tmplErr *error) (func(string) string, *ExtendedData, error) {
+func TmplText(ctx context.Context, tmpl *template.Template, alerts []*types.Alert, l log.Logger, tmplErr *error) (func(string) string, *ExtendedData) {
 	promTmplData := notify.GetTemplateData(ctx, tmpl, alerts, gokit_log.NewLogfmtLogger(logging.NewWrapper(l)))
-	data, err := ExtendData(promTmplData)
-	if err != nil {
-		return nil, nil, err
-	}
+	data := ExtendData(promTmplData, l)
 
 	return func(name string) (s string) {
 		if *tmplErr != nil {
@@ -140,7 +132,7 @@ func TmplText(ctx context.Context, tmpl *template.Template, alerts []*types.Aler
 		}
 		s, *tmplErr = tmpl.ExecuteTextString(name, data)
 		return s
-	}, data, nil
+	}, data
 }
 
 // Firing returns the subset of alerts that are firing.

pkg/services/ngalert/notifier/channels/threema.go

@@ -84,10 +84,7 @@ func (tn *ThreemaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 	tn.log.Debug("Sending threema alert notification", "from", tn.GatewayID, "to", tn.RecipientID)
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, tn.tmpl, as, tn.log, &tmplErr)
 
 	// Set up basic API request data
 	data := url.Values{}
@@ -112,7 +109,7 @@ func (tn *ThreemaNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 	data.Set("text", message)
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template Theema message: %w", tmplErr)
+		tn.log.Debug("failed to template Threema message", "err", tmplErr.Error())
 	}
 
 	cmd := &models.SendWebhookSync{

pkg/services/ngalert/notifier/channels/utils.go

@@ -112,15 +112,16 @@ var sendHTTPRequest = func(ctx context.Context, url *url.URL, cfg httpCfg, logge
 	return respBody, nil
 }
 
-func joinUrlPath(base, additionalPath string) (string, error) {
+func joinUrlPath(base, additionalPath string, logger log.Logger) string {
 	u, err := url.Parse(base)
 	if err != nil {
-		return "", fmt.Errorf("failed to parse URL: %w", err)
+		logger.Debug("failed to parse URL while joining URL", "url", base, "err", err.Error())
+		return base
 	}
 
 	u.Path = path.Join(u.Path, additionalPath)
 
-	return u.String(), nil
+	return u.String()
 }
 
 // GetBoundary is used for overriding the behaviour for tests

pkg/services/ngalert/notifier/channels/victorops.go

@@ -75,10 +75,7 @@ func (vn *VictoropsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo
 	}
 
 	var tmplErr error
-	tmpl, _, err := TmplText(ctx, vn.tmpl, as, vn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, _ := TmplText(ctx, vn.tmpl, as, vn.log, &tmplErr)
 
 	groupKey, err := notify.ExtractGroupKey(ctx)
 	if err != nil {
@@ -93,12 +90,13 @@ func (vn *VictoropsNotifier) Notify(ctx context.Context, as ...*types.Alert) (bo
 	bodyJSON.Set("state_message", tmpl(`{{ template "default.message" . }}`))
 	bodyJSON.Set("monitoring_tool", "Grafana v"+setting.BuildVersion)
 
-	ruleURL, err := joinUrlPath(vn.tmpl.ExternalURL.String(), "/alerting/list")
-	if err != nil {
-		return false, err
-	}
+	ruleURL := joinUrlPath(vn.tmpl.ExternalURL.String(), "/alerting/list", vn.log)
 	bodyJSON.Set("alert_url", ruleURL)
 
+	if tmplErr != nil {
+		vn.log.Debug("failed to template VictorOps message", "err", tmplErr.Error())
+	}
+
 	b, err := bodyJSON.MarshalJSON()
 	if err != nil {
 		return false, err

pkg/services/ngalert/notifier/channels/webhook.go

@@ -3,7 +3,6 @@ package channels
 import (
 	"context"
 	"encoding/json"
-	"fmt"
 
 	"github.com/prometheus/alertmanager/notify"
 	"github.com/prometheus/alertmanager/template"
@@ -80,10 +79,7 @@ func (wn *WebhookNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 
 	as, numTruncated := truncateAlerts(wn.MaxAlerts, as)
 	var tmplErr error
-	tmpl, data, err := TmplText(ctx, wn.tmpl, as, wn.log, &tmplErr)
-	if err != nil {
-		return false, err
-	}
+	tmpl, data := TmplText(ctx, wn.tmpl, as, wn.log, &tmplErr)
 	msg := &webhookMessage{
 		Version:         "1",
 		ExtendedData:    data,
@@ -100,7 +96,7 @@ func (wn *WebhookNotifier) Notify(ctx context.Context, as ...*types.Alert) (bool
 	}
 
 	if tmplErr != nil {
-		return false, fmt.Errorf("failed to template webhook message: %w", tmplErr)
+		wn.log.Debug("failed to template webhook message", "err", tmplErr.Error())
 	}
 
 	body, err := json.Marshal(msg)