From 9e408f842cf4d55f2db77a97dc287e628ae0fea1 Mon Sep 17 00:00:00 2001 From: Santiago Date: Wed, 13 Nov 2024 11:21:44 -0300 Subject: [PATCH] Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager (#96251) * Alerting: Skip sanitizing labels when sending alerts to the remote Alertmanager * fix drone image name --- .drone.yml | 21 +++++++++------- .../blocks/mimir_backend/docker-compose.yaml | 3 ++- pkg/services/ngalert/remote/alertmanager.go | 7 +++++- .../ngalert/remote/alertmanager_test.go | 24 ++++++++++--------- pkg/services/ngalert/sender/sender.go | 24 +++++++++++++++---- scripts/drone/services/services.star | 2 +- scripts/drone/utils/images.star | 2 +- 7 files changed, 56 insertions(+), 27 deletions(-) diff --git a/.drone.yml b/.drone.yml index d3fb536d1e0..d010b7598b2 100644 --- a/.drone.yml +++ b/.drone.yml @@ -942,8 +942,9 @@ services: path: /var/lib/mysql - commands: - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled + -alertmanager.utf8-strict-mode-enabled environment: {} - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 name: mimir_backend - environment: {} image: redis:6.2.11-alpine @@ -1383,8 +1384,9 @@ services: path: /var/lib/mysql - commands: - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled + -alertmanager.utf8-strict-mode-enabled environment: {} - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 name: mimir_backend - environment: {} image: redis:6.2.11-alpine @@ -2455,8 +2457,9 @@ services: path: /var/lib/mysql - commands: - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled + -alertmanager.utf8-strict-mode-enabled environment: {} - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 name: mimir_backend - environment: {} image: redis:6.2.11-alpine @@ -3039,8 +3042,9 @@ services: path: /var/lib/mysql - commands: - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled + -alertmanager.utf8-strict-mode-enabled environment: {} - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 name: mimir_backend - environment: {} image: redis:6.2.11-alpine @@ -4978,8 +4982,9 @@ services: path: /var/lib/mysql - commands: - /bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled + -alertmanager.utf8-strict-mode-enabled environment: {} - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 name: mimir_backend - environment: {} image: redis:6.2.11-alpine @@ -5451,7 +5456,7 @@ steps: - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM plugins/slack - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM python:3.8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM postgres:12.3-alpine - - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir-alpine:r304-3872ccb + - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM grafana/mimir-alpine:r316-55f47f8 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:5.7.39 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM mysql:8.0.32 - trivy --exit-code 0 --severity UNKNOWN,LOW,MEDIUM redis:6.2.11-alpine @@ -5489,7 +5494,7 @@ steps: - trivy --exit-code 1 --severity HIGH,CRITICAL plugins/slack - trivy --exit-code 1 --severity HIGH,CRITICAL python:3.8 - trivy --exit-code 1 --severity HIGH,CRITICAL postgres:12.3-alpine - - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir-alpine:r304-3872ccb + - trivy --exit-code 1 --severity HIGH,CRITICAL grafana/mimir-alpine:r316-55f47f8 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:5.7.39 - trivy --exit-code 1 --severity HIGH,CRITICAL mysql:8.0.32 - trivy --exit-code 1 --severity HIGH,CRITICAL redis:6.2.11-alpine @@ -5735,6 +5740,6 @@ kind: secret name: gcr_credentials --- kind: signature -hmac: 250d3c2ae9ddf7cbfd2a9fed01fe81312a882ba09b8f8725f4704629da00a774 +hmac: bb28bcd274c9d2ce724db12659ed6e3dcc461f8a07ae0eb9385a64ca5daad4de ... diff --git a/devenv/docker/blocks/mimir_backend/docker-compose.yaml b/devenv/docker/blocks/mimir_backend/docker-compose.yaml index a09635e80e3..544b836e1dd 100644 --- a/devenv/docker/blocks/mimir_backend/docker-compose.yaml +++ b/devenv/docker/blocks/mimir_backend/docker-compose.yaml @@ -1,9 +1,10 @@ mimir_backend: - image: grafana/mimir-alpine:r304-3872ccb + image: grafana/mimir-alpine:r316-55f47f8 container_name: mimir_backend command: - -target=backend - -alertmanager.grafana-alertmanager-compatibility-enabled + - -alertmanager.utf8-strict-mode-enabled nginx: environment: - NGINX_ENVSUBST_OUTPUT_DIR=/etc/nginx diff --git a/pkg/services/ngalert/remote/alertmanager.go b/pkg/services/ngalert/remote/alertmanager.go index 36c6e631b67..3d209aa85b9 100644 --- a/pkg/services/ngalert/remote/alertmanager.go +++ b/pkg/services/ngalert/remote/alertmanager.go @@ -150,7 +150,12 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt return c.Do(req.WithContext(ctx)) } senderLogger := log.New("ngalert.sender.external-alertmanager") - s, err := sender.NewExternalAlertmanagerSender(senderLogger, prometheus.NewRegistry(), sender.WithDoFunc(doFunc)) + s, err := sender.NewExternalAlertmanagerSender( + senderLogger, + prometheus.NewRegistry(), + sender.WithDoFunc(doFunc), + sender.WithUTF8Labels(), + ) if err != nil { return nil, err } diff --git a/pkg/services/ngalert/remote/alertmanager_test.go b/pkg/services/ngalert/remote/alertmanager_test.go index b8cb77ef422..7ad086dc7f2 100644 --- a/pkg/services/ngalert/remote/alertmanager_test.go +++ b/pkg/services/ngalert/remote/alertmanager_test.go @@ -785,16 +785,18 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) { require.Equal(t, 0, len(alertGroups)) // Let's create two active alerts and one expired one. - alert1 := genAlert(true, map[string]string{"test_1": "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"}) - alert2 := genAlert(true, map[string]string{"test_2": "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"}) - alert3 := genAlert(false, map[string]string{"test_3": "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"}) + // UTF-8 label names should be preserved. + utf8LabelName := "test utf-8 label 😳" + alert1 := genAlert(true, map[string]string{utf8LabelName: "test_1", "empty": "", alertingModels.NamespaceUIDLabel: "test_1"}) + alert2 := genAlert(true, map[string]string{utf8LabelName: "test_2", "empty": "", alertingModels.NamespaceUIDLabel: "test_2"}) + alert3 := genAlert(false, map[string]string{utf8LabelName: "test_3", "empty": "", alertingModels.NamespaceUIDLabel: "test_3"}) postableAlerts := apimodels.PostableAlerts{ PostableAlerts: []amv2.PostableAlert{alert1, alert2, alert3}, } err = am.PutAlerts(context.Background(), postableAlerts) require.NoError(t, err) - // We should have two alerts and one group now. + // We should eventually have two active alerts. require.Eventually(t, func() bool { alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{}, "") require.NoError(t, err) @@ -806,15 +808,15 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) { require.Equal(t, 1, len(alertGroups)) // Labels with empty values and the namespace UID label should be removed. + // UTF-8 label names should remain unchanged. for _, a := range alertGroups { - require.NotContains(t, a.Labels, "empty") - require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel) + require.Len(t, a.Alerts, 2) + for _, a := range a.Alerts { + require.NotContains(t, a.Labels, "empty") + require.NotContains(t, a.Labels, alertingModels.NamespaceUIDLabel) + require.Contains(t, a.Labels, utf8LabelName) + } } - - // Filtering by `test_1=test_1` should return one alert. - alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{"test_1=test_1"}, "") - require.NoError(t, err) - require.Equal(t, 1, len(alerts)) } func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) { diff --git a/pkg/services/ngalert/sender/sender.go b/pkg/services/ngalert/sender/sender.go index 2553e124780..8f4cd77df47 100644 --- a/pkg/services/ngalert/sender/sender.go +++ b/pkg/services/ngalert/sender/sender.go @@ -37,8 +37,9 @@ type ExternalAlertmanager struct { manager *Manager - sdCancel context.CancelFunc - sdManager *discovery.Manager + sanitizeLabelSetFn func(lbls models.LabelSet) labels.Labels + sdCancel context.CancelFunc + sdManager *discovery.Manager } type ExternalAMcfg struct { @@ -57,6 +58,20 @@ func WithDoFunc(doFunc doFunc) Option { } } +// WithUTF8Labels skips sanitizing labels and annotations before sending alerts to the external Alertmanager(s). +// It assumes UTF-8 label names are supported by the Alertmanager(s). +func WithUTF8Labels() Option { + return func(s *ExternalAlertmanager) { + s.sanitizeLabelSetFn = func(lbls models.LabelSet) labels.Labels { + ls := make(labels.Labels, 0, len(lbls)) + for k, v := range lbls { + ls = append(ls, labels.Label{Name: k, Value: v}) + } + return ls + } + } +} + func (cfg *ExternalAMcfg) SHA256() string { return asSHA256([]string{cfg.headerString(), cfg.URL}) } @@ -87,6 +102,7 @@ func NewExternalAlertmanagerSender(l log.Logger, reg prometheus.Registerer, opts sdCancel: sdCancel, } + s.sanitizeLabelSetFn = s.sanitizeLabelSet s.manager = NewManager( // Injecting a new registry here means these metrics are not exported. // Once we fix the individual Alertmanager metrics we should fix this scenario too. @@ -240,8 +256,8 @@ func buildNotifierConfig(alertmanagers []ExternalAMcfg) (*config.Config, map[str func (s *ExternalAlertmanager) alertToNotifierAlert(alert models.PostableAlert) *Alert { // Prometheus alertmanager has stricter rules for annotations/labels than grafana's internal alertmanager, so we sanitize invalid keys. return &Alert{ - Labels: s.sanitizeLabelSet(alert.Alert.Labels), - Annotations: s.sanitizeLabelSet(alert.Annotations), + Labels: s.sanitizeLabelSetFn(alert.Alert.Labels), + Annotations: s.sanitizeLabelSetFn(alert.Annotations), StartsAt: time.Time(alert.StartsAt), EndsAt: time.Time(alert.EndsAt), GeneratorURL: alert.Alert.GeneratorURL.String(), diff --git a/scripts/drone/services/services.star b/scripts/drone/services/services.star index 6d3044e932a..629f6016157 100644 --- a/scripts/drone/services/services.star +++ b/scripts/drone/services/services.star @@ -57,7 +57,7 @@ def integration_test_services(): "name": "mimir_backend", "image": images["mimir"], "environment": {}, - "commands": ["/bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled"], + "commands": ["/bin/mimir -target=backend -alertmanager.grafana-alertmanager-compatibility-enabled -alertmanager.utf8-strict-mode-enabled"], }, { "name": "redis", diff --git a/scripts/drone/utils/images.star b/scripts/drone/utils/images.star index 74d4a51ec94..1c87a607abb 100644 --- a/scripts/drone/utils/images.star +++ b/scripts/drone/utils/images.star @@ -22,7 +22,7 @@ images = { "plugins_slack": "plugins/slack", "python": "python:3.8", "postgres_alpine": "postgres:12.3-alpine", - "mimir": "grafana/mimir-alpine:r304-3872ccb", + "mimir": "grafana/mimir-alpine:r316-55f47f8", "mysql5": "mysql:5.7.39", "mysql8": "mysql:8.0.32", "redis_alpine": "redis:6.2.11-alpine",