From 5097dd5c7d0048085798ae74daba26c2a7bc068c Mon Sep 17 00:00:00 2001 From: Yuri Tseretyan Date: Fri, 18 Jul 2025 15:16:17 -0400 Subject: [PATCH] Alerting: Send templates from extra configuration to remote Alertmanager (#107981) * extract logging of MergedResult into method * convert GetMergedTemplateDefinitions to return PostableApiTemplate * update mergeExtracConfigs to return GrafanaAlertmanagerConfig * pass by value, not pointer * add template definition to payload * update tests * rename to Templates * log merge results * fix reference in workspace --- go.work | 2 +- .../api/tooling/definitions/alertmanager.go | 59 +++++++++++-------- .../tooling/definitions/alertmanager_test.go | 10 ++-- pkg/services/ngalert/notifier/alertmanager.go | 30 ++-------- pkg/services/ngalert/remote/alertmanager.go | 43 +++++++++----- .../ngalert/remote/alertmanager_test.go | 15 +++-- .../client/alertmanager_configuration.go | 20 ++++--- pkg/services/ngalert/remote/client/mimir.go | 2 +- pkg/services/ngalert/remote/compat.go | 13 ---- 9 files changed, 97 insertions(+), 97 deletions(-) delete mode 100644 pkg/services/ngalert/remote/compat.go diff --git a/go.work b/go.work index 8d1fff1ed6a..98e83d1c070 100644 --- a/go.work +++ b/go.work @@ -24,6 +24,6 @@ use ( ./pkg/semconv ) -replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250604130045-92c8f6389b36 +replace github.com/prometheus/alertmanager => github.com/grafana/prometheus-alertmanager v0.25.1-0.20250620093340-be61a673dee6 replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56 diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go index dfd39e73f56..c73486d277a 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go @@ -5,10 +5,10 @@ import ( "encoding/json" "errors" "fmt" + "strings" "time" "github.com/go-openapi/strfmt" - alertingTemplates "github.com/grafana/alerting/templates" amv2 "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/pkg/labels" @@ -17,6 +17,7 @@ import ( "github.com/grafana/alerting/definition" alertingmodels "github.com/grafana/alerting/models" + "github.com/grafana/grafana/pkg/apimachinery/errutil" ) @@ -267,9 +268,32 @@ type ( PostableApiReceiver = definition.PostableApiReceiver PostableGrafanaReceivers = definition.PostableGrafanaReceivers ReceiverType = definition.ReceiverType - MergeResult = definition.MergeResult ) +type MergeResult definition.MergeResult + +func (m MergeResult) LogContext() []any { + if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 { + return nil + } + logCtx := make([]any, 0, 4) + if len(m.RenamedTimeIntervals) > 0 { + rcvBuilder := strings.Builder{} + for from, to := range m.RenamedReceivers { + rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to)) + } + logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1])) + } + if len(m.RenamedTimeIntervals) > 0 { + rcvBuilder := strings.Builder{} + for from, to := range m.RenamedTimeIntervals { + rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to)) + } + logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1])) + } + return logCtx +} + const ( GrafanaReceiverType = definition.GrafanaReceiverType AlertmanagerReceiverType = definition.AlertmanagerReceiverType @@ -779,31 +803,20 @@ func (c *PostableUserConfig) GetMergedAlertmanagerConfig() (MergeResult, error) return MergeResult{}, fmt.Errorf("failed to get mimir alertmanager config: %w", err) } - return definition.Merge(c.AlertmanagerConfig, mcfg, opts) + m, err := definition.Merge(c.AlertmanagerConfig, mcfg, opts) + if err != nil { + return MergeResult{}, fmt.Errorf("failed to merge alertmanager config: %w", err) + } + return MergeResult(m), nil } -// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of TemplateDefinitions. -func (c *PostableUserConfig) GetMergedTemplateDefinitions() []alertingTemplates.TemplateDefinition { - out := make([]alertingTemplates.TemplateDefinition, 0, len(c.TemplateFiles)) - for name, tmpl := range c.TemplateFiles { - out = append(out, alertingTemplates.TemplateDefinition{ - Name: name, - Template: tmpl, - Kind: alertingTemplates.GrafanaKind, - }) - } - if len(c.ExtraConfigs) == 0 { +// GetMergedTemplateDefinitions converts the given PostableUserConfig's TemplateFiles to a slice of Templates. +func (c *PostableUserConfig) GetMergedTemplateDefinitions() []definition.PostableApiTemplate { + out := definition.TemplatesMapToPostableAPITemplates(c.TemplateFiles, definition.GrafanaTemplateKind) + if len(c.ExtraConfigs) == 0 || len(c.ExtraConfigs[0].TemplateFiles) == 0 { return out } - // support only one config for now - for name, tmpl := range c.ExtraConfigs[0].TemplateFiles { - out = append(out, alertingTemplates.TemplateDefinition{ - Name: name, - Template: tmpl, - Kind: alertingTemplates.MimirKind, - }) - } - return out + return append(out, definition.TemplatesMapToPostableAPITemplates(c.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind)...) } func (c *PostableUserConfig) UnmarshalJSON(b []byte) error { diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go index 5f6fb5c2390..7c18af905e4 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test.go @@ -6,7 +6,7 @@ import ( "strings" "testing" - alertingTemplates "github.com/grafana/alerting/templates" + "github.com/grafana/alerting/definition" "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/pkg/labels" "github.com/prometheus/common/model" @@ -383,21 +383,21 @@ func TestPostableUserConfig_GetMergedTemplateDefinitions(t *testing.T) { require.Len(t, result, tc.expectedTemplates) templateMap := make(map[string]string) - kindMap := make(map[string]alertingTemplates.Kind) + kindMap := make(map[string]definition.TemplateKind) for _, tmpl := range result { - templateMap[tmpl.Name] = tmpl.Template + templateMap[tmpl.Name] = tmpl.Content kindMap[tmpl.Name] = tmpl.Kind } for name, content := range tc.config.TemplateFiles { require.Equal(t, content, templateMap[name]) - require.Equal(t, alertingTemplates.GrafanaKind, kindMap[name]) + require.Equal(t, definition.GrafanaTemplateKind, kindMap[name]) } if len(tc.config.ExtraConfigs) > 0 { for name, content := range tc.config.ExtraConfigs[0].TemplateFiles { require.Equal(t, content, templateMap[name]) - require.Equal(t, alertingTemplates.MimirKind, kindMap[name]) + require.Equal(t, definition.MimirTemplateKind, kindMap[name]) } } }) diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index faf85e3f07a..8c833c14a2b 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -7,7 +7,6 @@ import ( "encoding/json" "fmt" "strconv" - "strings" "time" alertingNotify "github.com/grafana/alerting/notify" @@ -325,29 +324,6 @@ func (am *alertmanager) aggregateInhibitMatchers(rules []config.InhibitRule, amu } } -func logMergeResult(l log.Logger, m apimodels.MergeResult) { - if len(m.RenamedReceivers) == 0 && len(m.RenamedTimeIntervals) == 0 { - return - } - - logCtx := make([]any, 0, 4) - if len(m.RenamedTimeIntervals) > 0 { - rcvBuilder := strings.Builder{} - for from, to := range m.RenamedReceivers { - rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to)) - } - logCtx = append(logCtx, "renamedReceivers", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1])) - } - if len(m.RenamedTimeIntervals) > 0 { - rcvBuilder := strings.Builder{} - for from, to := range m.RenamedTimeIntervals { - rcvBuilder.WriteString(fmt.Sprintf("'%s'->'%s',", from, to)) - } - logCtx = append(logCtx, "renamedTimeIntervals", fmt.Sprintf("[%s]", rcvBuilder.String()[0:rcvBuilder.Len()-1])) - } - l.Info("Configurations merged successfully but some resources were renamed", logCtx...) -} - // applyConfig applies a new configuration by re-initializing all components using the configuration provided. // It returns a boolean indicating whether the user config was changed and an error. // It is not safe to call concurrently. @@ -361,9 +337,11 @@ func (am *alertmanager) applyConfig(ctx context.Context, cfg *apimodels.Postable if err != nil { return false, fmt.Errorf("failed to get full alertmanager configuration: %w", err) } - logMergeResult(am.logger, mergeResult) + if logInfo := mergeResult.LogContext(); len(logInfo) > 0 { + am.logger.Info("Configurations merged successfully but some resources were renamed", logInfo...) + } amConfig := mergeResult.Config - templates := cfg.GetMergedTemplateDefinitions() + templates := alertingNotify.PostableAPITemplatesToTemplateDefinitions(cfg.GetMergedTemplateDefinitions()) // Now add autogenerated config to the route. err = AddAutogenConfig(ctx, am.logger, am.Store, am.Base.TenantID(), &amConfig, skipInvalid) diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index d80f6967692..83ffef2251f 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -12,6 +12,7 @@ import ( "time" "github.com/go-openapi/strfmt" + "github.com/grafana/alerting/definition" amalert "github.com/prometheus/alertmanager/api/v2/client/alert" amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup" amgeneral "github.com/prometheus/alertmanager/api/v2/client/general" @@ -290,10 +291,10 @@ func (am *Alertmanager) CompareAndSendConfiguration(ctx context.Context, config } // Decrypt and merge extra configs - if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil { + payload, err := am.mergeExtraConfigs(ctx, decryptedCfg) + if err != nil { return fmt.Errorf("unable to merge extra configurations: %w", err) } - payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) rawPayload, err := json.Marshal(payload) if err != nil { return fmt.Errorf("unable to marshal decrypted configuration: %w", err) @@ -352,27 +353,36 @@ func decrypter(ctx context.Context, crypto Crypto) models.DecryptFn { } // mergeExtraConfigs decrypts and applies merged configuration if extra configs exist. -func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) error { +func (am *Alertmanager) mergeExtraConfigs(ctx context.Context, config *apimodels.PostableUserConfig) (remoteClient.GrafanaAlertmanagerConfig, error) { if len(config.ExtraConfigs) == 0 { - return nil + return remoteClient.GrafanaAlertmanagerConfig{ + TemplateFiles: config.TemplateFiles, + AlertmanagerConfig: config.AlertmanagerConfig, + Templates: nil, + }, nil } if err := am.crypto.DecryptExtraConfigs(ctx, config); err != nil { - return fmt.Errorf("unable to decrypt extra configs: %w", err) + return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to decrypt extra configs: %w", err) } mergeResult, err := config.GetMergedAlertmanagerConfig() if err != nil { - return fmt.Errorf("unable to get merged Alertmanager configuration: %w", err) + return remoteClient.GrafanaAlertmanagerConfig{}, fmt.Errorf("unable to get merged Alertmanager configuration: %w", err) } - config.AlertmanagerConfig = mergeResult.Config - // Clear ExtraConfigs to avoid re-processing them later - config.ExtraConfigs = nil - - return nil + if logctx := mergeResult.LogContext(); len(logctx) > 0 { + am.log.Debug("Configurations merged successfully but some resources were renamed", logctx...) + } + templates := definition.TemplatesMapToPostableAPITemplates(config.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind) + return remoteClient.GrafanaAlertmanagerConfig{ + // TODO keep sending Grafana templates as a map to not break old Mimir + TemplateFiles: config.TemplateFiles, + AlertmanagerConfig: mergeResult.Config, + Templates: templates, + }, nil } -func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg *remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { +func (am *Alertmanager) sendConfiguration(ctx context.Context, cfg remoteClient.GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { am.metrics.ConfigSyncsTotal.Inc() if err := am.mimirClient.CreateGrafanaAlertmanagerConfig( ctx, @@ -455,10 +465,10 @@ func (am *Alertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.P return err } - if err := am.mergeExtraConfigs(ctx, decryptedCfg); err != nil { + payload, err := am.mergeExtraConfigs(ctx, decryptedCfg) + if err != nil { return fmt.Errorf("unable to merge extra configurations: %w", err) } - payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) rawCfg, err := json.Marshal(payload) if err != nil { return err @@ -484,7 +494,10 @@ func (am *Alertmanager) SaveAndApplyDefaultConfig(ctx context.Context) error { return err } - payload := PostableUserConfigToGrafanaAlertmanagerConfig(decryptedCfg) + payload := remoteClient.GrafanaAlertmanagerConfig{ + TemplateFiles: c.TemplateFiles, + AlertmanagerConfig: decryptedCfg.AlertmanagerConfig, + } rawCfg, err := json.Marshal(payload) if err != nil { return err diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index 5aad1a47a1a..8e0b639734c 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -484,12 +484,18 @@ func TestCompareAndSendConfiguration(t *testing.T) { test, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) require.NoError(t, err) - cfgWithDecryptedSecret := PostableUserConfigToGrafanaAlertmanagerConfig(test) + cfgWithDecryptedSecret := client.GrafanaAlertmanagerConfig{ + TemplateFiles: test.TemplateFiles, + AlertmanagerConfig: test.AlertmanagerConfig, + } testAutogenRoutes, err := notifier.Load([]byte(testGrafanaConfigWithSecret)) require.NoError(t, err) require.NoError(t, testAutogenFn(nil, nil, 0, &testAutogenRoutes.AlertmanagerConfig, false)) - cfgWithAutogenRoutes := PostableUserConfigToGrafanaAlertmanagerConfig(testAutogenRoutes) + cfgWithAutogenRoutes := client.GrafanaAlertmanagerConfig{ + TemplateFiles: testAutogenRoutes.TemplateFiles, + AlertmanagerConfig: testAutogenRoutes.AlertmanagerConfig, + } // Calculate hashes for expected configurations cfgWithDecryptedSecretBytes, err := json.Marshal(cfgWithDecryptedSecret) @@ -506,9 +512,10 @@ func TestCompareAndSendConfiguration(t *testing.T) { require.NoError(t, err) r, err := cfgWithExtraUnmerged.GetMergedAlertmanagerConfig() require.NoError(t, err) - cfgWithExtraMerged := &client.GrafanaAlertmanagerConfig{ + cfgWithExtraMerged := client.GrafanaAlertmanagerConfig{ TemplateFiles: cfgWithExtraUnmerged.TemplateFiles, AlertmanagerConfig: r.Config, + Templates: definition.TemplatesMapToPostableAPITemplates(cfgWithExtraUnmerged.ExtraConfigs[0].TemplateFiles, definition.MimirTemplateKind), } cfgWithExtraMergedBytes, err := json.Marshal(cfgWithExtraMerged) require.NoError(t, err) @@ -827,7 +834,7 @@ func TestCompareAndSendConfigurationWithExtraConfigs(t *testing.T) { // Return an empty config to ensure it gets replaced w.Header().Add("content-type", "application/json") require.NoError(t, json.NewEncoder(w).Encode(client.UserGrafanaConfig{ - GrafanaAlertmanagerConfig: &client.GrafanaAlertmanagerConfig{}, + GrafanaAlertmanagerConfig: client.GrafanaAlertmanagerConfig{}, })) return } diff --git a/pkg/services/ngalert/remote/client/alertmanager_configuration.go b/pkg/services/ngalert/remote/client/alertmanager_configuration.go index 35239700ea7..998f785ffd6 100644 --- a/pkg/services/ngalert/remote/client/alertmanager_configuration.go +++ b/pkg/services/ngalert/remote/client/alertmanager_configuration.go @@ -17,24 +17,26 @@ const ( ) type GrafanaAlertmanagerConfig struct { + // TODO this needs to be deleted once Mimir is updated TemplateFiles map[string]string `yaml:"template_files" json:"template_files"` AlertmanagerConfig definition.PostableApiAlertingConfig `yaml:"alertmanager_config" json:"alertmanager_config"` + Templates []definition.PostableApiTemplate `yaml:"templates,omitempty" json:"templates,omitempty"` } func (u *GrafanaAlertmanagerConfig) MarshalJSON() ([]byte, error) { - // this is special marshaling that makes sure that secrets are not masked + // This is special marshaling that makes sure that secrets are not masked. type cfg GrafanaAlertmanagerConfig return definition.MarshalJSONWithSecrets((*cfg)(u)) } type UserGrafanaConfig struct { - GrafanaAlertmanagerConfig *GrafanaAlertmanagerConfig `json:"configuration"` - Hash string `json:"configuration_hash"` - CreatedAt int64 `json:"created"` - Default bool `json:"default"` - Promoted bool `json:"promoted"` - ExternalURL string `json:"external_url"` - SmtpConfig SmtpConfig `json:"smtp_config"` + GrafanaAlertmanagerConfig GrafanaAlertmanagerConfig `json:"configuration"` + Hash string `json:"configuration_hash"` + CreatedAt int64 `json:"created"` + Default bool `json:"default"` + Promoted bool `json:"promoted"` + ExternalURL string `json:"external_url"` + SmtpConfig SmtpConfig `json:"smtp_config"` // TODO: Remove once everything can be sent in the 'SmtpConfig' field. SmtpFrom string `json:"smtp_from"` @@ -64,7 +66,7 @@ func (mc *Mimir) GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafana return gc, nil } -func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { +func (mc *Mimir) CreateGrafanaAlertmanagerConfig(ctx context.Context, cfg GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error { payload, err := definition.MarshalJSONWithSecrets(&UserGrafanaConfig{ GrafanaAlertmanagerConfig: cfg, Hash: hash, diff --git a/pkg/services/ngalert/remote/client/mimir.go b/pkg/services/ngalert/remote/client/mimir.go index 3bf10465076..59c3a6ad1ee 100644 --- a/pkg/services/ngalert/remote/client/mimir.go +++ b/pkg/services/ngalert/remote/client/mimir.go @@ -30,7 +30,7 @@ type MimirClient interface { DeleteGrafanaAlertmanagerState(ctx context.Context) error GetGrafanaAlertmanagerConfig(ctx context.Context) (*UserGrafanaConfig, error) - CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration *GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error + CreateGrafanaAlertmanagerConfig(ctx context.Context, configuration GrafanaAlertmanagerConfig, hash string, createdAt int64, isDefault bool) error DeleteGrafanaAlertmanagerConfig(ctx context.Context) error TestTemplate(ctx context.Context, c alertingNotify.TestTemplatesConfigBodyParams) (*alertingNotify.TestTemplatesResults, error) diff --git a/pkg/services/ngalert/remote/compat.go b/pkg/services/ngalert/remote/compat.go deleted file mode 100644 index 8914bc96728..00000000000 --- a/pkg/services/ngalert/remote/compat.go +++ /dev/null @@ -1,13 +0,0 @@ -package remote - -import ( - "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" - "github.com/grafana/grafana/pkg/services/ngalert/remote/client" -) - -func PostableUserConfigToGrafanaAlertmanagerConfig(config *definitions.PostableUserConfig) *client.GrafanaAlertmanagerConfig { - return &client.GrafanaAlertmanagerConfig{ - TemplateFiles: config.TemplateFiles, - AlertmanagerConfig: config.AlertmanagerConfig, - } -}