From dfba94e052eb678fdf73201662a90b39de76d811 Mon Sep 17 00:00:00 2001
From: Alexander Weaver <weaver.alex.d@gmail.com>
Date: Tue, 29 Aug 2023 14:59:12 -0500
Subject: [PATCH] Alerting: Limit redis pool size to 5 and make configurable
 (#74057)

* Limit redis pool size to 5 and expose it in config ini

* Coerce negative pool sizes to the default
---
 conf/defaults.ini                                      | 3 +++
 pkg/services/ngalert/notifier/multiorg_alertmanager.go | 1 +
 pkg/services/ngalert/notifier/redis_peer.go            | 8 ++++++++
 pkg/setting/setting_unified_alerting.go                | 3 +++
 4 files changed, 15 insertions(+)

diff --git a/conf/defaults.ini b/conf/defaults.ini
index 2a092f3517d..005f7ec510c 100644
--- a/conf/defaults.ini
+++ b/conf/defaults.ini
@@ -1063,6 +1063,9 @@ ha_redis_prefix =
 # provided, a random one will be generated.
 ha_redis_peer_name =
 
+# The maximum number of simultaneous redis connections.
+ha_redis_max_conns = 5
+
 # Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port.
 ha_listen_address = "0.0.0.0:9094"
 
diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go
index 170aea7f4b2..f366b23df54 100644
--- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go
+++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go
@@ -93,6 +93,7 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error {
 			password: cfg.UnifiedAlerting.HARedisPassword,
 			username: cfg.UnifiedAlerting.HARedisUsername,
 			db:       cfg.UnifiedAlerting.HARedisDB,
+			maxConns: cfg.UnifiedAlerting.HARedisMaxConns,
 		}, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval)
 		if err != nil {
 			return fmt.Errorf("unable to initialize redis: %w", err)
diff --git a/pkg/services/ngalert/notifier/redis_peer.go b/pkg/services/ngalert/notifier/redis_peer.go
index a75a2e8ce5a..1abf79887d3 100644
--- a/pkg/services/ngalert/notifier/redis_peer.go
+++ b/pkg/services/ngalert/notifier/redis_peer.go
@@ -27,6 +27,7 @@ type redisConfig struct {
 	db       int
 	name     string
 	prefix   string
+	maxConns int
 }
 
 const (
@@ -44,6 +45,7 @@ const (
 	reasonRedisIssue        = "redis_issue"
 	heartbeatInterval       = time.Second * 5
 	heartbeatTimeout        = time.Minute
+	defaultPoolSize         = 5
 	// The duration we want to return the members if the network is down.
 	membersValidFor = time.Minute
 )
@@ -84,11 +86,17 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer,
 	if cfg.name != "" {
 		name = cfg.name
 	}
+	// Allow zero through, since it'll fall back to go-redis's default.
+	poolSize := defaultPoolSize
+	if cfg.maxConns >= 0 {
+		poolSize = cfg.maxConns
+	}
 	rdb := redis.NewClient(&redis.Options{
 		Addr:     cfg.addr,
 		Username: cfg.username,
 		Password: cfg.password,
 		DB:       cfg.db,
+		PoolSize: poolSize,
 	})
 	cmd := rdb.Ping(context.Background())
 	if cmd.Err() != nil {
diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go
index 65573f44856..e6194f5d9d4 100644
--- a/pkg/setting/setting_unified_alerting.go
+++ b/pkg/setting/setting_unified_alerting.go
@@ -20,6 +20,7 @@ const (
 	alertmanagerDefaultGossipInterval     = cluster.DefaultGossipInterval
 	alertmanagerDefaultPushPullInterval   = cluster.DefaultPushPullInterval
 	alertmanagerDefaultConfigPollInterval = time.Minute
+	alertmanagerRedisDefaultMaxConns      = 5
 	// To start, the alertmanager needs at least one route defined.
 	// TODO: we should move this to Grafana settings and define this as the default.
 	alertmanagerDefaultConfiguration = `{
@@ -78,6 +79,7 @@ type UnifiedAlertingSettings struct {
 	HARedisUsername                string
 	HARedisPassword                string
 	HARedisDB                      int
+	HARedisMaxConns                int
 	MaxAttempts                    int64
 	MinInterval                    time.Duration
 	EvaluationTimeout              time.Duration
@@ -240,6 +242,7 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	uaCfg.HARedisUsername = ua.Key("ha_redis_username").MustString("")
 	uaCfg.HARedisPassword = ua.Key("ha_redis_password").MustString("")
 	uaCfg.HARedisDB = ua.Key("ha_redis_db").MustInt(0)
+	uaCfg.HARedisMaxConns = ua.Key("ha_redis_max_conns").MustInt(alertmanagerRedisDefaultMaxConns)
 	peers := ua.Key("ha_peers").MustString("")
 	uaCfg.HAPeers = make([]string, 0)
 	if peers != "" {