From b9cdad38144b45c714590871a26414f7f7bcfe88 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Jean-Philippe=20Qu=C3=A9m=C3=A9ner?= Date: Fri, 19 Nov 2021 16:50:55 +0100 Subject: [PATCH] Alerting: support mute timings configuration through the api for the embedded alertmanager (#41533) * Alerting: accept mute_timing_intervals through the api for the embedded alertmanager * add workaround for mutetimeinterval * add mute timings to routes * revert changes * Update pkg/services/ngalert/api/api_alertmanager.go * Update pkg/services/ngalert/api/api_alertmanager.go * Update pkg/services/ngalert/api/api_alertmanager.go * update prometheus/alertmanager dependency * add some var docs --- go.mod | 4 ++-- go.sum | 4 ++++ .../api/tooling/definitions/alertmanager.go | 9 +++++---- .../alertmanager_test_artifact.json | 13 +++++++++++++ .../alertmanager_test_artifact.yaml | 6 ++++++ pkg/services/ngalert/notifier/alertmanager.go | 19 ++++++++++++++++++- .../ngalert/notifier/multiorg_alertmanager.go | 1 + 7 files changed, 49 insertions(+), 7 deletions(-) diff --git a/go.mod b/go.mod index 3794969f1bd..af831b4e1c7 100644 --- a/go.mod +++ b/go.mod @@ -81,7 +81,7 @@ require ( github.com/patrickmn/go-cache v2.1.0+incompatible github.com/pkg/browser v0.0.0-20210904010418-6d279e18f982 // indirect github.com/pkg/errors v0.9.1 - github.com/prometheus/alertmanager v0.23.1-0.20211021072955-1b8afe7cb5aa + github.com/prometheus/alertmanager v0.23.1-0.20211116083607-e2a10119aaf7 github.com/prometheus/client_golang v1.11.0 github.com/prometheus/client_model v0.2.0 github.com/prometheus/common v0.32.1 @@ -225,7 +225,7 @@ require ( github.com/opentracing-contrib/go-stdlib v1.0.0 // indirect github.com/pmezard/go-difflib v1.0.0 // indirect github.com/prometheus/common/sigv4 v0.1.0 // indirect - github.com/prometheus/exporter-toolkit v0.6.1 // indirect + github.com/prometheus/exporter-toolkit v0.7.0 // indirect github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289 // indirect github.com/prometheus/procfs v0.6.0 // indirect github.com/protocolbuffers/txtpbfmt v0.0.0-20201118171849-f6a6b3f636fc // indirect diff --git a/go.sum b/go.sum index 8e9f44be236..c905edbf15c 100644 --- a/go.sum +++ b/go.sum @@ -1969,6 +1969,8 @@ github.com/prometheus/alertmanager v0.23.0/go.mod h1:0MLTrjQI8EuVmvykEhcfr/7X0xm github.com/prometheus/alertmanager v0.23.1-0.20210914172521-e35efbddb66a/go.mod h1:U7pGu+z7A9ZKhK8lq1MvIOp5GdVlZjwOYk+S0h3LSbA= github.com/prometheus/alertmanager v0.23.1-0.20211021072955-1b8afe7cb5aa h1:KKgVswVOfDYOn9GNtO7bR3r4vyM77WXuJsyGxMl1Zgs= github.com/prometheus/alertmanager v0.23.1-0.20211021072955-1b8afe7cb5aa/go.mod h1:pRqzxS2B4tciJfx2JUvR67udJrQeSUZ603OQQFFUrIQ= +github.com/prometheus/alertmanager v0.23.1-0.20211116083607-e2a10119aaf7 h1:OMwDo53awRp+UzaBrwmVC7HJiAMYP/niBJfKcGpPiac= +github.com/prometheus/alertmanager v0.23.1-0.20211116083607-e2a10119aaf7/go.mod h1:1UH4XA4DAXzsvofKVzcXmC0mqt6Y8BZP9JcQWKDmbFc= github.com/prometheus/client_golang v0.0.0-20180209125602-c332b6f63c06/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.8.0/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= github.com/prometheus/client_golang v0.9.1/go.mod h1:7SWBe2y4D6OKWSNQJUaRYU/AaXPKyh/dDVn+NZz0KFw= @@ -2032,6 +2034,8 @@ github.com/prometheus/exporter-toolkit v0.5.0/go.mod h1:OCkM4805mmisBhLmVFw858QY github.com/prometheus/exporter-toolkit v0.5.1/go.mod h1:OCkM4805mmisBhLmVFw858QYi3v0wKdY6/UxrT0pZVg= github.com/prometheus/exporter-toolkit v0.6.1 h1:Aqk75wQD92N9CqmTlZwjKwq6272nOGrWIbc8Z7+xQO0= github.com/prometheus/exporter-toolkit v0.6.1/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= +github.com/prometheus/exporter-toolkit v0.7.0 h1:XtYeVeeC5daG4txbc9+mieKq+/AK4gtIBLl9Mulrjnk= +github.com/prometheus/exporter-toolkit v0.7.0/go.mod h1:ZUBIj498ePooX9t/2xtDjeQYwvRpiPP2lh5u4iblj2g= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289 h1:dTUS1vaLWq+Y6XKOTnrFpoVsQKLCbCp1OLj24TDi7oM= github.com/prometheus/node_exporter v1.0.0-rc.0.0.20200428091818-01054558c289/go.mod h1:FGbBv5OPKjch+jNUJmEQpMZytIdyW0NdBtWFcfSKusc= github.com/prometheus/procfs v0.0.0-20180125133057-cb4147076ac7/go.mod h1:c3At6R/oaqEKCNdg8wHV1ftS6bRYblBhIjjI8uT2IGk= diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go index af6dc8d00e6..25dce197f52 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager.go +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager.go @@ -587,10 +587,11 @@ func (c *GettableApiAlertingConfig) validate() error { // Config is the top-level configuration for Alertmanager's config files. type Config struct { - Global *config.GlobalConfig `yaml:"global,omitempty" json:"global,omitempty"` - Route *Route `yaml:"route,omitempty" json:"route,omitempty"` - InhibitRules []*config.InhibitRule `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"` - Templates []string `yaml:"templates" json:"templates"` + Global *config.GlobalConfig `yaml:"global,omitempty" json:"global,omitempty"` + Route *Route `yaml:"route,omitempty" json:"route,omitempty"` + InhibitRules []*config.InhibitRule `yaml:"inhibit_rules,omitempty" json:"inhibit_rules,omitempty"` + MuteTimeIntervals []config.MuteTimeInterval `yaml:"mute_time_intervals,omitempty" json:"mute_time_intervals,omitempty"` + Templates []string `yaml:"templates" json:"templates"` } // A Route is a node that contains definitions of how to handle alerts. This is modified diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.json b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.json index cc68eb49359..9413f6b6524 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.json +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.json @@ -3,6 +3,19 @@ "foo": "bar" }, "alertmanager_config": { + "mute_time_intervals": [ + { + "name": "foo", + "time_intervals": [ + { + "years": [ + "2020:2022", + "2030" + ] + } + ] + } + ], "receivers": [ { "email_configs": [ diff --git a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.yaml b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.yaml index 0ecbfd8a7e4..b618b74cba0 100644 --- a/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.yaml +++ b/pkg/services/ngalert/api/tooling/definitions/alertmanager_test_artifact.yaml @@ -1,6 +1,12 @@ template_files: foo: bar alertmanager_config: | + mute_time_intervals: + - name: foo + time_intervals: + - years: + - 2020:2022 + - "2030" receivers: - email_configs: - auth_password: shh diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 0eee6ef65e8..027f976f9b7 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -18,6 +18,7 @@ import ( gokit_log "github.com/go-kit/kit/log" amv2 "github.com/prometheus/alertmanager/api/v2/models" "github.com/prometheus/alertmanager/cluster" + "github.com/prometheus/alertmanager/config" "github.com/prometheus/alertmanager/dispatch" "github.com/prometheus/alertmanager/inhibit" "github.com/prometheus/alertmanager/nflog" @@ -26,6 +27,7 @@ import ( "github.com/prometheus/alertmanager/provider/mem" "github.com/prometheus/alertmanager/silence" "github.com/prometheus/alertmanager/template" + "github.com/prometheus/alertmanager/timeinterval" "github.com/prometheus/alertmanager/types" "github.com/prometheus/client_golang/prometheus" "github.com/prometheus/common/model" @@ -110,6 +112,10 @@ type Alertmanager struct { silencer *silence.Silencer silences *silence.Silences + // muteTimes is a map where the key is the name of the mute_time_interval + // and the value represents all configured time_interval(s) + muteTimes map[string][]timeinterval.TimeInterval + stageMetrics *notify.Metrics dispatcherMetrics *dispatch.DispatcherMetrics @@ -329,6 +335,14 @@ func (am *Alertmanager) templateFromPaths(paths ...string) (*template.Template, return tmpl, nil } +func (am *Alertmanager) buildMuteTimesMap(muteTimeIntervals []config.MuteTimeInterval) map[string][]timeinterval.TimeInterval { + muteTimes := make(map[string][]timeinterval.TimeInterval, len(muteTimeIntervals)) + for _, ti := range muteTimeIntervals { + muteTimes[ti.Name] = ti.TimeIntervals + } + return muteTimes +} + // applyConfig applies a new configuration by re-initializing all components using the configuration provided. // It is not safe to call concurrently. func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig []byte) (err error) { @@ -375,6 +389,7 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig if err != nil { return fmt.Errorf("failed to build integration map: %w", err) } + // Now, let's put together our notification pipeline routingStage := make(notify.RoutingStage, len(integrationsMap)) @@ -386,14 +401,16 @@ func (am *Alertmanager) applyConfig(cfg *apimodels.PostableUserConfig, rawConfig } am.inhibitor = inhibit.NewInhibitor(am.alerts, cfg.AlertmanagerConfig.InhibitRules, am.marker, am.gokitLogger) + am.muteTimes = am.buildMuteTimesMap(cfg.AlertmanagerConfig.MuteTimeIntervals) am.silencer = silence.NewSilencer(am.silences, am.marker, am.gokitLogger) meshStage := notify.NewGossipSettleStage(am.peer) inhibitionStage := notify.NewMuteStage(am.inhibitor) + timeMuteStage := notify.NewTimeMuteStage(am.muteTimes) silencingStage := notify.NewMuteStage(am.silencer) for name := range integrationsMap { stage := am.createReceiverStage(name, integrationsMap[name], am.waitFunc, am.notificationLog) - routingStage[name] = notify.MultiStage{meshStage, silencingStage, inhibitionStage, stage} + routingStage[name] = notify.MultiStage{meshStage, silencingStage, timeMuteStage, inhibitionStage, stage} } am.route = dispatch.NewRoute(cfg.AlertmanagerConfig.Route.AsAMRoute(), nil) diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 64f408271d4..32917af897e 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -78,6 +78,7 @@ func NewMultiOrgAlertmanager(cfg *setting.Cfg, configStore store.AlertingStore, cluster.DefaultProbeTimeout, cluster.DefaultProbeInterval, nil, + true, ) if err != nil {