diff --git a/conf/defaults.ini b/conf/defaults.ini index 2a092f3517d..005f7ec510c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1063,6 +1063,9 @@ ha_redis_prefix = # provided, a random one will be generated. ha_redis_peer_name = +# The maximum number of simultaneous redis connections. +ha_redis_max_conns = 5 + # Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. ha_listen_address = "0.0.0.0:9094" diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 170aea7f4b2..f366b23df54 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -93,6 +93,7 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error { password: cfg.UnifiedAlerting.HARedisPassword, username: cfg.UnifiedAlerting.HARedisUsername, db: cfg.UnifiedAlerting.HARedisDB, + maxConns: cfg.UnifiedAlerting.HARedisMaxConns, }, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval) if err != nil { return fmt.Errorf("unable to initialize redis: %w", err) diff --git a/pkg/services/ngalert/notifier/redis_peer.go b/pkg/services/ngalert/notifier/redis_peer.go index a75a2e8ce5a..1abf79887d3 100644 --- a/pkg/services/ngalert/notifier/redis_peer.go +++ b/pkg/services/ngalert/notifier/redis_peer.go @@ -27,6 +27,7 @@ type redisConfig struct { db int name string prefix string + maxConns int } const ( @@ -44,6 +45,7 @@ const ( reasonRedisIssue = "redis_issue" heartbeatInterval = time.Second * 5 heartbeatTimeout = time.Minute + defaultPoolSize = 5 // The duration we want to return the members if the network is down. membersValidFor = time.Minute ) @@ -84,11 +86,17 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer, if cfg.name != "" { name = cfg.name } + // Allow zero through, since it'll fall back to go-redis's default. + poolSize := defaultPoolSize + if cfg.maxConns >= 0 { + poolSize = cfg.maxConns + } rdb := redis.NewClient(&redis.Options{ Addr: cfg.addr, Username: cfg.username, Password: cfg.password, DB: cfg.db, + PoolSize: poolSize, }) cmd := rdb.Ping(context.Background()) if cmd.Err() != nil { diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index 65573f44856..e6194f5d9d4 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -20,6 +20,7 @@ const ( alertmanagerDefaultGossipInterval = cluster.DefaultGossipInterval alertmanagerDefaultPushPullInterval = cluster.DefaultPushPullInterval alertmanagerDefaultConfigPollInterval = time.Minute + alertmanagerRedisDefaultMaxConns = 5 // To start, the alertmanager needs at least one route defined. // TODO: we should move this to Grafana settings and define this as the default. alertmanagerDefaultConfiguration = `{ @@ -78,6 +79,7 @@ type UnifiedAlertingSettings struct { HARedisUsername string HARedisPassword string HARedisDB int + HARedisMaxConns int MaxAttempts int64 MinInterval time.Duration EvaluationTimeout time.Duration @@ -240,6 +242,7 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error { uaCfg.HARedisUsername = ua.Key("ha_redis_username").MustString("") uaCfg.HARedisPassword = ua.Key("ha_redis_password").MustString("") uaCfg.HARedisDB = ua.Key("ha_redis_db").MustInt(0) + uaCfg.HARedisMaxConns = ua.Key("ha_redis_max_conns").MustInt(alertmanagerRedisDefaultMaxConns) peers := ua.Key("ha_peers").MustString("") uaCfg.HAPeers = make([]string, 0) if peers != "" {