Alerting: Send templates from extra configuration to remote Alertmanager (#107981)

* extract logging of MergedResult into method

* convert GetMergedTemplateDefinitions to return PostableApiTemplate

* update mergeExtracConfigs to return GrafanaAlertmanagerConfig

* pass by value, not pointer

* add template definition to payload

* update tests

* rename to Templates

* log merge results

* fix reference in workspace

go.work

@@ -24,6 +24,6 @@ use (
 	./pkg/semconv
 )
 
-replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36
+replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6
 
 replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56

pkg/services/ngalert/api/tooling/definitions/alertmanager.go

@@ -5,10 +5,10 @@ import (
 	"encoding/json"
 	"errors"
 	"fmt"
+	"strings"
 	"time"
 
 	"github.com/go-openapi/strfmt"
-	alertingTemplates "github.com/grafana/alerting/templates"
 	amv2 "github.com/prometheus/alertmanager/api/v2/models"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/pkg/labels"
@@ -17,6 +17,7 @@ import (
 
 	"github.com/grafana/alerting/definition"
 	alertingmodels "github.com/grafana/alerting/models"
+
 	"github.com/grafana/grafana/pkg/apimachinery/errutil"
 )
 
@@ -267,9 +268,32 @@ type (
 	PostableApiReceiver       = definition.PostableApiReceiver
 	PostableGrafanaReceivers  = definition.PostableGrafanaReceivers
 	ReceiverType              = definition.ReceiverType
-	MergeResult               = definition.MergeResult
 )
 
+type MergeResult definition.MergeResult
+
+func (m MergeResult) LogContext() []any {
+	if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 {
+		return nil
+	}
+	logCtx := make([]any, 0, 4)
+	if len(m.RenamedTimeIntervals) > 0 {
+		rcvBuilder := strings.Builder{}
+		for from, to := range m.RenamedReceivers {
+			rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
+		}
+		logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
+	}
+	if len(m.RenamedTimeIntervals) > 0 {
+		rcvBuilder := strings.Builder{}
+		for from, to := range m.RenamedTimeIntervals {
+			rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
+		}
+		logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
+	}
+	return logCtx
+}
+
 const (
 	GrafanaReceiverType      = definition.GrafanaReceiverType
 	AlertmanagerReceiverType = definition.AlertmanagerReceiverType
@@ -779,31 +803,20 @@ func (c *PostableUserConfig) GetMergedAlertmanagerConfig() (MergeResult, error)
 		return MergeResult{}, fmt.Errorf("failed to get mimir alertmanager config: %w", err)
 	}
 
-	return definition.Merge(c.AlertmanagerConfig, mcfg, opts)
+	m, err := definition.Merge(c.AlertmanagerConfig, mcfg, opts)
+	if err != nil {
+		return MergeResult{}, fmt.Errorf("failed to merge alertmanager config: %w", err)
+	}
+	return MergeResult(m), nil
 }
 
-// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of TemplateDefinitions.
-func (c *PostableUserConfig) GetMergedTemplateDefinitions() []alertingTemplates.TemplateDefinition {
-	out := make([]alertingTemplates.TemplateDefinition, 0, len(c.TemplateFiles))
-	for name, tmpl := range c.TemplateFiles {
-		out = append(out, alertingTemplates.TemplateDefinition{
-			Name:     name,
-			Template: tmpl,
-			Kind:     alertingTemplates.GrafanaKind,
-		})
-	}
-	if len(c.ExtraConfigs) == 0 {
+// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of Templates.
+func (c *PostableUserConfig) GetMergedTemplateDefinitions() []definition.PostableApiTemplate {
+	out := definition.TemplatesMapToPostableAPITemplates(c.TemplateFiles, definition.GrafanaTemplateKind)
+	if len(c.ExtraConfigs) == 0 || len(c.ExtraConfigs[0].TemplateFiles) == 0 {
 		return out
 	}
-	// support only one config for now
-	for name, tmpl := range c.ExtraConfigs[0].TemplateFiles {
-		out = append(out, alertingTemplates.TemplateDefinition{
-			Name:     name,
-			Template: tmpl,
-			Kind:     alertingTemplates.MimirKind,
-		})
-	}
-	return out
+	return append(out, definition.TemplatesMapToPostableAPITemplates(c.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)...)
 }
 
 func (c *PostableUserConfig) UnmarshalJSON(b []byte) error {

pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go

@@ -6,7 +6,7 @@ import (
 	"strings"
 	"testing"
 
-	alertingTemplates "github.com/grafana/alerting/templates"
+	"github.com/grafana/alerting/definition"
 	"github.com/prometheus/alertmanager/config"
 	"github.com/prometheus/alertmanager/pkg/labels"
 	"github.com/prometheus/common/model"
@@ -383,21 +383,21 @@ func TestPostableUserConfig_GetMergedTemplateDefinitions(t *testing.T) {
 			require.Len(t, result, tc.expectedTemplates)
 
 			templateMap := make(map[string]string)
-			kindMap := make(map[string]alertingTemplates.Kind)
+			kindMap := make(map[string]definition.TemplateKind)
 			for _, tmpl := range result {
-				templateMap[tmpl.Name] = tmpl.Template
+				templateMap[tmpl.Name] = tmpl.Content
 				kindMap[tmpl.Name] = tmpl.Kind
 			}
 
 			for name, content := range tc.config.TemplateFiles {
 				require.Equal(t, content, templateMap[name])
-				require.Equal(t, alertingTemplates.GrafanaKind, kindMap[name])
+				require.Equal(t, definition.GrafanaTemplateKind, kindMap[name])
 			}
 
 			if len(tc.config.ExtraConfigs) > 0 {
 				for name, content := range tc.config.ExtraConfigs[0].TemplateFiles {
 					require.Equal(t, content, templateMap[name])
-					require.Equal(t, alertingTemplates.MimirKind, kindMap[name])
+					require.Equal(t, definition.MimirTemplateKind, kindMap[name])
 				}
 			}
 		})

pkg/services/ngalert/notifier/alertmanager.go

@@ -7,7 +7,6 @@ import (
 	"encoding/json"
 	"fmt"
 	"strconv"
-	"strings"
 	"time"
 
 	alertingNotify "github.com/grafana/alerting/notify"
@@ -325,29 +324,6 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu
 	}
 }
 
-func logMergeResult(l log.Logger, m apimodels.MergeResult) {
-	if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 {
-		return
-	}
-
-	logCtx := make([]any, 0, 4)
-	if len(m.RenamedTimeIntervals) > 0 {
-		rcvBuilder := strings.Builder{}
-		for from, to := range m.RenamedReceivers {
-			rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
-		}
-		logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
-	}
-	if len(m.RenamedTimeIntervals) > 0 {
-		rcvBuilder := strings.Builder{}
-		for from, to := range m.RenamedTimeIntervals {
-			rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to))
-		}
-		logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1]))
-	}
-	l.Info("Configurations merged successfully but some resources were renamed", logCtx...)
-}
-
 // applyConfig applies a new configuration by re-initializing all components using the configuration provided.
 // It returns a boolean indicating whether the user config was changed and an error.
 // It is not safe to call concurrently.
@@ -361,9 +337,11 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable
 	if err != nil {
 		return false, fmt.Errorf("failed to get full alertmanager configuration: %w", err)
 	}
-	logMergeResult(am.logger, mergeResult)
+	if logInfo := mergeResult.LogContext(); len(logInfo) > 0 {
+		am.logger.Info("Configurations merged successfully but some resources were renamed", logInfo...)
+	}
 	amConfig := mergeResult.Config
-	templates := cfg.GetMergedTemplateDefinitions()
+	templates := alertingNotify.PostableAPITemplatesToTemplateDefinitions(cfg.GetMergedTemplateDefinitions())
 
 	// Now add autogenerated config to the route.
 	err = AddAutogenConfig(ctx, am.logger, am.Store, am.Base.TenantID(), &amConfig, skipInvalid)

pkg/services/ngalert/remote/alertmanager.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"github.com/go-openapi/strfmt"
+	"github.com/grafana/alerting/definition"
 	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
 	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
 	amgeneral "github.com/prometheus/alertmanager/api/v2/client/general"
@@ -290,10 +291,10 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config
 	}
 
 	// Decrypt and merge extra configs
-	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+	payload, err := am.mergeExtraConfigs(ctx, decryptedCfg)
+	if err != nil {
 		return fmt.Errorf("unable to merge extra configurations: %w", err)
 	}
-	payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
 	rawPayload, err := json.Marshal(payload)
 	if err != nil {
 		return fmt.Errorf("unable to marshal decrypted configuration: %w", err)
@@ -352,27 +353,36 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn {
 }
 
 // mergeExtraConfigs decrypts and applies merged configuration if extra configs exist.
-func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error {
+func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) (remoteClient.GrafanaAlertmanagerConfig, error) {
 	if len(config.ExtraConfigs) == 0 {
-		return nil
+		return remoteClient.GrafanaAlertmanagerConfig{
+			TemplateFiles:      config.TemplateFiles,
+			AlertmanagerConfig: config.AlertmanagerConfig,
+			Templates:          nil,
+		}, nil
 	}
 
 	if err := am.crypto.DecryptExtraConfigs(ctx, config); err != nil {
-		return fmt.Errorf("unable to decrypt extra configs: %w", err)
+		return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to decrypt extra configs: %w", err)
 	}
 
 	mergeResult, err := config.GetMergedAlertmanagerConfig()
 	if err != nil {
-		return fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
+		return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to get merged Alertmanager configuration: %w", err)
 	}
-	config.AlertmanagerConfig = mergeResult.Config
-	// Clear ExtraConfigs to avoid re-processing them later
-	config.ExtraConfigs = nil
-
-	return nil
+	if logctx := mergeResult.LogContext(); len(logctx) > 0 {
+		am.log.Debug("Configurations merged successfully but some resources were renamed", logctx...)
+	}
+	templates := definition.TemplatesMapToPostableAPITemplates(config.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)
+	return remoteClient.GrafanaAlertmanagerConfig{
+		// TODO keep sending Grafana templates as a map to not break old Mimir
+		TemplateFiles:      config.TemplateFiles,
+		AlertmanagerConfig: mergeResult.Config,
+		Templates:          templates,
+	}, nil
 }
 
-func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg *remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
+func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
 	am.metrics.ConfigSyncsTotal.Inc()
 	if err := am.mimirClient.CreateGrafanaAlertmanagerConfig(
 		ctx,
@@ -455,10 +465,10 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P
 		return err
 	}
 
-	if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil {
+	payload, err := am.mergeExtraConfigs(ctx, decryptedCfg)
+	if err != nil {
 		return fmt.Errorf("unable to merge extra configurations: %w", err)
 	}
-	payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
 	rawCfg, err := json.Marshal(payload)
 	if err != nil {
 		return err
@@ -484,7 +494,10 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error {
 		return err
 	}
 
-	payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg)
+	payload := remoteClient.GrafanaAlertmanagerConfig{
+		TemplateFiles:      c.TemplateFiles,
+		AlertmanagerConfig: decryptedCfg.AlertmanagerConfig,
+	}
 	rawCfg, err := json.Marshal(payload)
 	if err != nil {
 		return err

pkg/services/ngalert/remote/alertmanager_test.go

@@ -484,12 +484,18 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 
 	test, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
 	require.NoError(t, err)
-	cfgWithDecryptedSecret := PostableUserConfigToGrafanaAlertmanagerConfig(test)
+	cfgWithDecryptedSecret := client.GrafanaAlertmanagerConfig{
+		TemplateFiles:      test.TemplateFiles,
+		AlertmanagerConfig: test.AlertmanagerConfig,
+	}
 
 	testAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret))
 	require.NoError(t, err)
 	require.NoError(t, testAutogenFn(nil, nil, 0, &testAutogenRoutes.AlertmanagerConfig, false))
-	cfgWithAutogenRoutes := PostableUserConfigToGrafanaAlertmanagerConfig(testAutogenRoutes)
+	cfgWithAutogenRoutes := client.GrafanaAlertmanagerConfig{
+		TemplateFiles:      testAutogenRoutes.TemplateFiles,
+		AlertmanagerConfig: testAutogenRoutes.AlertmanagerConfig,
+	}
 
 	// Calculate hashes for expected configurations
 	cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret)
@@ -506,9 +512,10 @@ func TestCompareAndSendConfiguration(t *testing.T) {
 	require.NoError(t, err)
 	r, err := cfgWithExtraUnmerged.GetMergedAlertmanagerConfig()
 	require.NoError(t, err)
-	cfgWithExtraMerged := &client.GrafanaAlertmanagerConfig{
+	cfgWithExtraMerged := client.GrafanaAlertmanagerConfig{
 		TemplateFiles:      cfgWithExtraUnmerged.TemplateFiles,
 		AlertmanagerConfig: r.Config,
+		Templates:          definition.TemplatesMapToPostableAPITemplates(cfgWithExtraUnmerged.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind),
 	}
 	cfgWithExtraMergedBytes, err := json.Marshal(cfgWithExtraMerged)
 	require.NoError(t, err)
@@ -827,7 +834,7 @@ func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) {
 			// Return an empty config to ensure it gets replaced
 			w.Header().Add("content-type", "application/json")
 			require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{
-				GrafanaAlertmanagerConfig: &client.GrafanaAlertmanagerConfig{},
+				GrafanaAlertmanagerConfig: client.GrafanaAlertmanagerConfig{},
 			}))
 			return
 		}

pkg/services/ngalert/remote/client/alertmanager_configuration.go

@@ -17,24 +17,26 @@ const (
 )
 
 type GrafanaAlertmanagerConfig struct {
+	// TODO this needs to be deleted once Mimir is updated
 	TemplateFiles      map[string]string                    `yaml:"template_files" json:"template_files"`
 	AlertmanagerConfig definition.PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"`
+	Templates          []definition.PostableApiTemplate     `yaml:"templates,omitempty" json:"templates,omitempty"`
 }
 
 func (u *GrafanaAlertmanagerConfig) MarshalJSON() ([]byte, error) {
-	// this is special marshaling that makes sure that secrets are not masked
+	// This is special marshaling that makes sure that secrets are not masked.
 	type cfg GrafanaAlertmanagerConfig
 	return definition.MarshalJSONWithSecrets((*cfg)(u))
 }
 
 type UserGrafanaConfig struct {
-	GrafanaAlertmanagerConfig *GrafanaAlertmanagerConfig `json:"configuration"`
-	Hash                      string                     `json:"configuration_hash"`
-	CreatedAt                 int64                      `json:"created"`
-	Default                   bool                       `json:"default"`
-	Promoted                  bool                       `json:"promoted"`
-	ExternalURL               string                     `json:"external_url"`
-	SmtpConfig                SmtpConfig                 `json:"smtp_config"`
+	GrafanaAlertmanagerConfig GrafanaAlertmanagerConfig `json:"configuration"`
+	Hash                      string                    `json:"configuration_hash"`
+	CreatedAt                 int64                     `json:"created"`
+	Default                   bool                      `json:"default"`
+	Promoted                  bool                      `json:"promoted"`
+	ExternalURL               string                    `json:"external_url"`
+	SmtpConfig                SmtpConfig                `json:"smtp_config"`
 
 	// TODO: Remove once everything can be sent in the 'SmtpConfig' field.
 	SmtpFrom      string            `json:"smtp_from"`
@@ -64,7 +66,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana
 	return gc, nil
 }
 
-func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
+func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error {
 	payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{
 		GrafanaAlertmanagerConfig: cfg,
 		Hash:                      hash,

pkg/services/ngalert/remote/client/mimir.go

@@ -30,7 +30,7 @@ type MimirClient interface {
 	DeleteGrafanaAlertmanagerState(ctx context.Context) error
 
 	GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error)
-	CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error
+	CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error
 	DeleteGrafanaAlertmanagerConfig(ctx context.Context) error
 
 	TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error)

pkg/services/ngalert/remote/compat.go

@@ -1,13 +0,0 @@
-package remote
-
-import (
-	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
-	"github.com/grafana/grafana/pkg/services/ngalert/remote/client"
-)
-
-func PostableUserConfigToGrafanaAlertmanagerConfig(config *definitions.PostableUserConfig) *client.GrafanaAlertmanagerConfig {
-	return &client.GrafanaAlertmanagerConfig{
-		TemplateFiles:      config.TemplateFiles,
-		AlertmanagerConfig: config.AlertmanagerConfig,
-	}
-}