From 513799583042ecc141a1dc9860d0fa7f924dcf3e Mon Sep 17 00:00:00 2001 From: Vadim Stepanov Date: Thu, 5 Jun 2025 15:02:40 +0100 Subject: [PATCH] Alerting: Add support for Redis Sentinel for Alerting HA (#106322) * Alerting: Add support for Redis Sentinel * docs * docs * Use minisentinel in test * Apply suggestions from code review Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com> Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com> * "address(es)" -> "address or addresses" * make update-workspace * make lint-go-diff --------- Co-authored-by: Johnny Kartheiser <140559259+JohnnyK-Grafana@users.noreply.github.com> Co-authored-by: Fayzal Ghantiwala <114010985+fayzal-g@users.noreply.github.com> --- conf/defaults.ini | 46 ++++++++----- conf/sample.ini | 65 +++++++++-------- .../configure-high-availability/_index.md | 7 +- .../setup-grafana/configure-grafana/_index.md | 61 ++++++++++++++-- go.mod | 5 +- go.sum | 15 ++++ .../ngalert/notifier/multiorg_alertmanager.go | 24 ++++--- pkg/services/ngalert/notifier/redis_peer.go | 34 ++++----- .../ngalert/notifier/redis_peer_test.go | 23 +++++++ pkg/setting/setting_unified_alerting.go | 19 +++++ pkg/setting/setting_unified_alerting_test.go | 69 +++++++++++++++++++ 11 files changed, 290 insertions(+), 78 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index 44ce0971d7b..ca4561fc4e0 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1288,39 +1288,51 @@ alertmanager_max_silences_count = # Maximum silence size in bytes. Default: 0 (no limit). alertmanager_max_silence_size_bytes = -# Set to true when using redis in cluster mode. +# Redis server address or addresses. It can be a single Redis address if using Redis standalone, +# or a list of comma-separated addresses if using Redis Cluster/Sentinel. +ha_redis_address = + +# Set to true when using Redis in Cluster mode. Mutually exclusive with ha_redis_sentinel_mode_enabled. ha_redis_cluster_mode_enabled = false -# The redis server address(es) that should be connected to. -# Can either be a single address, or if using redis in cluster mode, -# the cluster configuration address or a comma-separated list of addresses. -ha_redis_address = +# Set to true when using Redis in Sentinel mode. Mutually exclusive with ha_redis_cluster_mode_enabled. +ha_redis_sentinel_mode_enabled = false -# The username that should be used to authenticate with the redis server. +# Redis Sentinel master name. Only applicable when ha_redis_sentinel_mode_enabled is set to true. +ha_redis_sentinel_master_name = + +# The username that should be used to authenticate with Redis. ha_redis_username = -# The password that should be used to authenticate with the redis server. +# The password that should be used to authenticate with Redis. ha_redis_password = -# The redis database, by default it's 0. +# The username that should be used to authenticate with Redis Sentinel. +# Only applicable when ha_redis_sentinel_mode_enabled is set to true. +ha_redis_sentinel_username = + +# The password that should be used to authenticate with Redis Sentinel. +# Only applicable when ha_redis_sentinel_mode_enabled is set to true. +ha_redis_sentinel_password = + +# The Redis database. The default value is 0. ha_redis_db = -# A prefix that is used for every key or channel that is created on the redis server -# as part of HA for alerting. +# A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting. +# Useful if you plan to share Redis with multiple Grafana instances. ha_redis_prefix = -# The name of the cluster peer that will be used as identifier. If none is -# provided, a random one will be generated. +# The name of the cluster peer to use as an identifier. If none is provided, a random one is generated. ha_redis_peer_name = -# The maximum number of simultaneous redis connections. +# The maximum number of simultaneous Redis connections. ha_redis_max_conns = 5 -# Enable TLS on the client used to communicate with the redis server. This should be set to true +# Enable TLS on the client used to communicate with the Redis server. This should be set to true # if using any of the other ha_redis_tls_* fields. ha_redis_tls_enabled = false -# Path to the PEM-encoded TLS client certificate file used to authenticate with the redis server. +# Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server. # Required if using Mutual TLS. ha_redis_tls_cert_path = @@ -1331,10 +1343,10 @@ ha_redis_tls_key_path = # Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used. ha_redis_tls_ca_path = -# Overrides the expected name of the redis server certificate. +# Overrides the expected name of the Redis server certificate. ha_redis_tls_server_name = -# Skips validating the redis server certificate. +# Skips validating the Redis server certificate. ha_redis_tls_insecure_skip_verify = # Overrides the default TLS cipher suite list. diff --git a/conf/sample.ini b/conf/sample.ini index af873ea3213..5d50ad20207 100644 --- a/conf/sample.ini +++ b/conf/sample.ini @@ -1261,68 +1261,79 @@ # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m. ;alertmanager_config_poll_interval = 60s - # Maximum number of active and pending silences that a tenant can have at once. Default: 0 (no limit). ;alertmanager_max_silences_count = # Maximum silence size in bytes. Default: 0 (no limit). ;alertmanager_max_silence_size_bytes = -# Set to true when using redis in cluster mode. +# Redis server address or addresses. It can be a single Redis address if using Redis standalone, +# or a list of comma-separated addresses if using Redis Cluster/Sentinel. +;ha_redis_address = + +# Set to true when using Redis in Cluster mode. Mutually exclusive with ha_redis_sentinel_mode_enabled. ;ha_redis_cluster_mode_enabled = false -# The redis server address(es) that should be connected to. -# Can either be a single address, or if using redis in cluster mode, -# the cluster configuration address or a comma-separated list of addresses. -;ha_redis_address = +# Set to true when using Redis in Sentinel mode. Mutually exclusive with ha_redis_cluster_mode_enabled. +;ha_redis_sentinel_mode_enabled = false -# The username that should be used to authenticate with the redis server. +# Redis Sentinel master name. Only applicable when ha_redis_sentinel_mode_enabled is set to true. +;ha_redis_sentinel_master_name = + +# The username that should be used to authenticate with Redis. ;ha_redis_username = -# The password that should be used to authenticate with the redis server. +# The password that should be used to authenticate with Redis. ;ha_redis_password = -# The redis database, by default it's 0. +# The username that should be used to authenticate with Redis Sentinel. +# Only applicable when ha_redis_sentinel_mode_enabled is set to true. +;ha_redis_sentinel_username = + +# The password that should be used to authenticate with Redis Sentinel. +# Only applicable when ha_redis_sentinel_mode_enabled is set to true. +;ha_redis_sentinel_password = + +# The Redis database. The default value is 0. ;ha_redis_db = -# A prefix that is used for every key or channel that is created on the redis server -# as part of HA for alerting. +# A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting. +# Useful if you plan to share Redis with multiple Grafana instances. ;ha_redis_prefix = -# The name of the cluster peer that will be used as identifier. If none is -# provided, a random one will be generated. +# The name of the cluster peer to use as an identifier. If none is provided, a random one is generated. ;ha_redis_peer_name = -# The maximum number of simultaneous redis connections. -# ha_redis_max_conns = 5 +# The maximum number of simultaneous Redis connections. +;ha_redis_max_conns = 5 -# Enable TLS on the client used to communicate with the redis server. This should be set to true +# Enable TLS on the client used to communicate with the Redis server. This should be set to true # if using any of the other ha_redis_tls_* fields. -# ha_redis_tls_enabled = false +;ha_redis_tls_enabled = false -# Path to the PEM-encoded TLS client certificate file used to authenticate with the redis server. +# Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server. # Required if using Mutual TLS. -# ha_redis_tls_cert_path = +;ha_redis_tls_cert_path = # Path to the PEM-encoded TLS private key file. Also requires the client certificate to be configured. # Required if using Mutual TLS. -# ha_redis_tls_key_path = +;ha_redis_tls_key_path = # Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used. -# ha_redis_tls_ca_path = +;ha_redis_tls_ca_path = -# Overrides the expected name of the redis server certificate. -# ha_redis_tls_server_name = +# Overrides the expected name of the Redis server certificate. +;ha_redis_tls_server_name = -# Skips validating the redis server certificate. -# ha_redis_tls_insecure_skip_verify = +# Skips validating the Redis server certificate. +;ha_redis_tls_insecure_skip_verify = # Overrides the default TLS cipher suite list. -# ha_redis_tls_cipher_suites = +;ha_redis_tls_cipher_suites = # Overrides the default minimum TLS version. # Allowed values: VersionTLS10, VersionTLS11, VersionTLS12, VersionTLS13 -# ha_redis_tls_min_version = +;ha_redis_tls_min_version = # Listen address/hostname and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`. ;ha_listen_address = "0.0.0.0:9094" diff --git a/docs/sources/alerting/set-up/configure-high-availability/_index.md b/docs/sources/alerting/set-up/configure-high-availability/_index.md index 2823b2613bb..556843b4e50 100644 --- a/docs/sources/alerting/set-up/configure-high-availability/_index.md +++ b/docs/sources/alerting/set-up/configure-high-availability/_index.md @@ -67,7 +67,7 @@ For a demo, see this [example using Docker Compose](https://github.com/grafana/a ## Enable alerting high availability using Redis -As an alternative to Memberlist, you can configure Redis to enable high availability. Only **Redis Server** and **Redis Cluster** modes are supported. +As an alternative to Memberlist, you can configure Redis to enable high availability. Redis standalone, Redis Cluster and Redis Sentinel modes are supported. {{% admonition type="note" %}} @@ -77,8 +77,11 @@ Memberlist is the preferred option for high availability. Use Redis only in envi 1. Make sure you have a Redis server that supports pub/sub. If you use a proxy in front of your Redis cluster, make sure the proxy supports pub/sub. 1. In your custom configuration file ($WORKING_DIR/conf/custom.ini), go to the `[unified_alerting]` section. -1. Set `ha_redis_address` to the Redis server address Grafana should connect to. +1. Set `ha_redis_address` to the Redis server address or addresses Grafana should connect to. It can be a single Redis address if using Redis standalone, or a list of comma-separated addresses if using Redis Cluster or Sentinel. +1. Optional: Set `ha_redis_cluster_mode_enabled` to `true` if you are using Redis Cluster. +1. Optional: Set `ha_redis_sentinel_mode_enabled` to `true` if you are using Redis Sentinel. Also set `ha_redis_sentinel_master_name` to the Redis Sentinel master name. 1. Optional: Set the username and password if authentication is enabled on the Redis server using `ha_redis_username` and `ha_redis_password`. +1. Optional: Set the username and password if authentication is enabled on Redis Sentinel using `ha_redis_sentinel_username` and `ha_redis_sentinel_password`. 1. Optional: Set `ha_redis_prefix` to something unique if you plan to share the Redis server with multiple Grafana instances. 1. Optional: Set `ha_redis_tls_enabled` to `true` and configure the corresponding `ha_redis_tls_*` fields to secure communications between Grafana and Redis with Transport Layer Security (TLS). 1. Set `[ha_advertise_address]` to `ha_advertise_address = "${POD_IP}:9094"` This is required if the instance doesn't have an IP address that is part of RFC 6890 with a default route. diff --git a/docs/sources/setup-grafana/configure-grafana/_index.md b/docs/sources/setup-grafana/configure-grafana/_index.md index 3501be4b183..6db48dc9054 100644 --- a/docs/sources/setup-grafana/configure-grafana/_index.md +++ b/docs/sources/setup-grafana/configure-grafana/_index.md @@ -1771,19 +1771,40 @@ The interval string is a possibly signed sequence of decimal numbers, followed b #### `ha_redis_address` -The Redis server address that should be connected to. +Redis server address or addresses. It can be a single Redis address if using Redis standalone, +or a list of comma-separated addresses if using Redis Cluster/Sentinel. {{< admonition type="note" >}} For more information on Redis, refer to [Enable alerting high availability using Redis](https://grafana.com/docs/grafana//alerting/set-up/configure-high-availability/#enable-alerting-high-availability-using-redis). {{< /admonition >}} +#### `ha_redis_cluster_mode_enabled` + +Set to `true` when using Redis in Cluster mode. Mutually exclusive with `ha_redis_sentinel_mode_enabled`. + +#### `ha_redis_sentinel_mode_enabled` + +Set to `true` when using Redis in Sentinel mode. Mutually exclusive with `ha_redis_cluster_mode_enabled`. + +#### `ha_redis_sentinel_master_name` + +Redis Sentinel master name. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`. + #### `ha_redis_username` -The username that should be used to authenticate with the Redis server. +The username that should be used to authenticate with Redis. #### `ha_redis_password` -The password that should be used to authenticate with the Redis server. +The password that should be used to authenticate with Redis. + +#### `ha_redis_sentinel_username` + +The username that should be used to authenticate with Redis Sentinel. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`. + +#### `ha_redis_sentinel_password` + +The password that should be used to authenticate with Redis Sentinel. Only applicable when `ha_redis_sentinel_mode_enabled` is set to `true`. #### `ha_redis_db` @@ -1791,7 +1812,7 @@ The Redis database. The default value is `0`. #### `ha_redis_prefix` -A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting. +A prefix that is used for every key or channel that is created on the Redis server as part of HA for alerting. Useful if you plan to share Redis with multiple Grafana instances. #### `ha_redis_peer_name` @@ -1801,6 +1822,38 @@ The name of the cluster peer to use as an identifier. If none is provided, a ran The maximum number of simultaneous Redis connections. +#### `ha_redis_tls_enabled` + +Enable TLS on the client used to communicate with the Redis server. This should be set to `true` if using any of the other `ha_redis_tls_*` fields. + +#### `ha_redis_tls_cert_path` + +Path to the PEM-encoded TLS client certificate file used to authenticate with the Redis server. Required if using Mutual TLS. + +#### `ha_redis_tls_key_path` + +Path to the PEM-encoded TLS private key file. Also requires the client certificate to be configured. Required if using Mutual TLS. + +#### `ha_redis_tls_ca_path` + +Path to the PEM-encoded CA certificates file. If not set, the host's root CA certificates are used. + +#### `ha_redis_tls_server_name` + +Overrides the expected name of the Redis server certificate. + +#### `ha_redis_tls_insecure_skip_verify` + +Skips validating the Redis server certificate. + +#### `ha_redis_tls_cipher_suites` + +Overrides the default TLS cipher suite list. + +#### `ha_redis_tls_min_version` + +Overrides the default minimum TLS version. Allowed values: `VersionTLS10`, `VersionTLS11`, `VersionTLS12`, `VersionTLS13` + #### `ha_listen_address` Listen IP address and port to receive unified alerting messages for other Grafana instances. The port is used for both TCP and UDP. It is assumed other Grafana instances are also running on the same port. The default value is `0.0.0.0:9094`. diff --git a/go.mod b/go.mod index 3e3875c530d..0542656c481 100644 --- a/go.mod +++ b/go.mod @@ -18,6 +18,7 @@ require ( github.com/Azure/azure-storage-blob-go v0.15.0 // @grafana/grafana-backend-group github.com/Azure/go-autorest/autorest v0.11.29 // @grafana/grafana-backend-group github.com/Azure/go-autorest/autorest/adal v0.9.24 // @grafana/grafana-backend-group + github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb // @grafana/alerting-backend github.com/BurntSushi/toml v1.5.0 // @grafana/identity-access-team github.com/DATA-DOG/go-sqlmock v1.5.2 // @grafana/grafana-search-and-storage github.com/Masterminds/semver v1.5.0 // @grafana/grafana-backend-group @@ -389,6 +390,7 @@ require ( github.com/hashicorp/errwrap v1.1.0 // indirect github.com/hashicorp/go-cleanhttp v0.5.2 // indirect github.com/hashicorp/go-immutable-radix v1.3.1 // indirect + github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect github.com/hashicorp/go-rootcerts v1.0.2 // indirect github.com/hashicorp/go-sockaddr v1.0.7 // indirect github.com/hashicorp/go-uuid v1.0.3 // indirect @@ -426,6 +428,7 @@ require ( github.com/lestrrat-go/strftime v1.0.4 // indirect github.com/magefile/mage v1.15.0 // indirect github.com/mailru/easyjson v0.7.7 // indirect + github.com/matryer/is v1.4.1 // indirect github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 // indirect github.com/mattermost/xml-roundtrip-validator v0.1.0 // indirect github.com/mattetti/filebuffer v1.0.1 // indirect @@ -567,8 +570,6 @@ require ( sigs.k8s.io/yaml v1.4.0 // indirect ) -require github.com/hashicorp/go-msgpack/v2 v2.1.1 // indirect - // Use fork of crewjam/saml with fixes for some issues until changes get merged into upstream replace github.com/crewjam/saml => github.com/grafana/saml v0.4.15-0.20240917091248-ae3bbdad8a56 diff --git a/go.sum b/go.sum index be5c962595e..aa8fed04bfe 100644 --- a/go.sum +++ b/go.sum @@ -702,6 +702,8 @@ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1 h1:WJ github.com/AzureAD/microsoft-authentication-extensions-for-go/cache v0.1.1/go.mod h1:tCcJZ0uHAmvjsVYzEFivsRTN00oz5BEsRgQHu5JZ9WE= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2 h1:oygO0locgZJe7PpYPXT5A29ZkwJaPqcva7BVeemZOZs= github.com/AzureAD/microsoft-authentication-library-for-go v1.4.2/go.mod h1:wP83P5OoQ5p6ip3ScPr0BAq0BvuPAvacpEuSzyouqAI= +github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb h1:ZVN4Iat3runWOFLaBCDVU5a9X/XikSRBosye++6gojw= +github.com/Bose/minisentinel v0.0.0-20200130220412-917c5a9223bb/go.mod h1:WsAABbY4HQBgd3mGuG4KMNTbHJCPvx9IVBHzysbknss= github.com/BurntSushi/toml v0.3.1/go.mod h1:xHWCNGjB5oqiDr8zfno3MHue2Ht5sIBksp03qcyfWMU= github.com/BurntSushi/toml v1.4.0/go.mod h1:ukJfTF/6rtPPRCnwkur4qwRxa8vTRFBF0uk2lLoLwho= github.com/BurntSushi/toml v1.5.0 h1:W5quZX/G/csjUnuI8SUYlsHs9M38FC7znL0lIO+DvMg= @@ -715,6 +717,8 @@ github.com/DataDog/datadog-go v2.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3 github.com/DataDog/datadog-go v3.2.0+incompatible/go.mod h1:LButxg5PwREeZtORoXG3tL4fMGNddJ+vMq1mwgfaqoQ= github.com/FZambia/eagle v0.2.0 h1:1kQaZpJvbkvAXFRE/9K2ucBMuVqo+E29EMLYB74hIis= github.com/FZambia/eagle v0.2.0/go.mod h1:LKMYBwGYhao5sJI0TppvQ4SvvldFj9gITxrl8NvGwG0= +github.com/FZambia/sentinel v1.0.0 h1:KJ0ryjKTZk5WMp0dXvSdNqp3lFaW1fNFuEYfrkLOYIc= +github.com/FZambia/sentinel v1.0.0/go.mod h1:ytL1Am/RLlAoAXG6Kj5LNuw/TRRQrv2rt2FT26vP5gI= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0 h1:ErKg/3iS1AKcTkf3yixlZ54f9U1rljCkQyEXWUnIUxc= github.com/GoogleCloudPlatform/opentelemetry-operations-go/detectors/gcp v1.27.0/go.mod h1:yAZHSGnqScoU556rBOVkwLze6WP5N+U11RHuWaGVxwY= github.com/GoogleCloudPlatform/opentelemetry-operations-go/exporter/metric v0.51.0 h1:fYE9p3esPxA/C0rQ0AHhP0drtPXDRhaWiwg1DPqO7IU= @@ -790,8 +794,10 @@ github.com/alecthomas/units v0.0.0-20190924025748-f65c72e2690d/go.mod h1:rBZYJk5 github.com/alecthomas/units v0.0.0-20211218093645-b94a6e3cc137/go.mod h1:OMCwj8VM1Kc9e19TLln2VL61YJF0x1XFtfdL4JdbSyE= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b h1:mimo19zliBX/vSQ6PWWSL9lK8qwHozUj03+zLoEB8O0= github.com/alecthomas/units v0.0.0-20240927000941-0f3dac36c52b/go.mod h1:fvzegU4vN3H1qMT+8wDmzjAcDONcgo2/SZ/TyfdUOFs= +github.com/alicebob/gopher-json v0.0.0-20180125190556-5a6b3ba71ee6/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302 h1:uvdUDbHQHO85qeSydJtItA4T55Pw6BtAejd0APRJOCE= github.com/alicebob/gopher-json v0.0.0-20230218143504-906a9b012302/go.mod h1:SGnFV6hVsYE877CKEZ6tDNTjaSXYUk6QqoIK6PrAtcc= +github.com/alicebob/miniredis/v2 v2.11.1/go.mod h1:UA48pmi7aSazcGAvcdKcBB49z521IC9VjTTRz2nIaJE= github.com/alicebob/miniredis/v2 v2.34.0 h1:mBFWMaJSNL9RwdGRyEDoAAv8OQc5UlEhLDQggTglU/0= github.com/alicebob/miniredis/v2 v2.34.0/go.mod h1:kWShP4b58T1CW0Y5dViCd5ztzrDqRWqM3nksiyXk5s8= github.com/andybalholm/brotli v1.0.4/go.mod h1:fO7iG3H7G2nSZ7m0zPUDn85XEX2GTukHGRSepvi9Eig= @@ -1427,6 +1433,9 @@ github.com/golang/snappy v0.0.3/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEW github.com/golang/snappy v0.0.4/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= github.com/golang/snappy v1.0.0 h1:Oy607GVXHs7RtbggtPBnr2RmDArIsAefDwvrdWvRhGs= github.com/golang/snappy v1.0.0/go.mod h1:/XxbfmMg8lxefKM7IXC3fBNl/7bRcc72aCRzEWrmP2Q= +github.com/gomodule/redigo v1.7.1-0.20190322064113-39e2c31b7ca3/go.mod h1:B4C85qUVwatsJoIUNIfCRsp7qO0iAmpGFZ4EELWSbC4= +github.com/gomodule/redigo v1.8.9 h1:Sl3u+2BI/kk+VEatbj0scLdrFhjPmbxOc1myhDP41ws= +github.com/gomodule/redigo v1.8.9/go.mod h1:7ArFNvsTjH8GMMzB4uy1snslv2BwmginuMs06a1uzZE= github.com/google/btree v0.0.0-20180813153112-4030bb1f1f0c/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.0.0/go.mod h1:lNA+9X1NB3Zf8V7Ke586lFgjr2dZNuvo3lPJSGZ5JPQ= github.com/google/btree v1.1.3 h1:CVpQJjYgC4VbzxeGVHfvZrv1ctoYCAI8vbl07Fcxlyg= @@ -1919,6 +1928,9 @@ github.com/mailru/easyjson v0.0.0-20190614124828-94de47d64c63/go.mod h1:C1wdFJiN github.com/mailru/easyjson v0.0.0-20190626092158-b2ccc519800e/go.mod h1:C1wdFJiN94OJF2b5HbByQZoLdCWB1Yqtg26g4irojpc= github.com/mailru/easyjson v0.7.7 h1:UGYAvKxe3sBsEDzO8ZeWOSlIQfWFlxbzLZe7hwFURr0= github.com/mailru/easyjson v0.7.7/go.mod h1:xzfreul335JAWq5oZzymOObrkdz5UnU4kGfJJLY9Nlc= +github.com/matryer/is v1.2.0/go.mod h1:2fLPjFQM9rhQ15aVEtbuwhJinnOqrmgXPNdZsdwlWXA= +github.com/matryer/is v1.4.1 h1:55ehd8zaGABKLXQUe2awZ99BD/PTc2ls+KV/dXphgEQ= +github.com/matryer/is v1.4.1/go.mod h1:8I/i5uYgLzgsgEloJE1U6xx5HkBQpAZvepWuujKwMRU= github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38 h1:hQWBtNqRYrI7CWIaUSXXtNKR90KzcUA5uiuxFVWw7sU= github.com/mattbaird/jsonpatch v0.0.0-20240118010651-0ba75a80ca38/go.mod h1:M1qoD/MqPgTZIk0EWKB38wE28ACRfVcn+cU08jyArI0= github.com/mattermost/xml-roundtrip-validator v0.1.0 h1:RXbVD2UAl7A7nOTR4u7E3ILa4IbtvKBHw64LDsmu9hU= @@ -2467,6 +2479,8 @@ github.com/yuin/goldmark v1.2.1/go.mod h1:3hX8gzYuyVAZsxl0MRgGTJEmQBFcNTphYh9dec github.com/yuin/goldmark v1.3.5/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.1/go.mod h1:mwnBkeHKe2W/ZEtQ+71ViKU8L12m81fl3OWwC1Zlc8k= github.com/yuin/goldmark v1.4.13/go.mod h1:6yULJ656Px+3vBD8DxQVa3kxgyrAnzto9xy5taEt/CY= +github.com/yuin/gopher-lua v0.0.0-20190206043414-8bfc7677f583/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= +github.com/yuin/gopher-lua v0.0.0-20191213034115-f46add6fdb5c/go.mod h1:gqRgreBUhTSL0GeU64rtZ3Uq3wtjOa/TB2YfrtkCbVQ= github.com/yuin/gopher-lua v1.1.1 h1:kYKnWBjvbNP4XLT3+bPEwAXJx262OhaHDWDVOPjL46M= github.com/yuin/gopher-lua v1.1.1/go.mod h1:GBR0iDaNXjAgGg9zfCvksxSRnQx76gclCIb7kdAd1Pw= github.com/zclconf/go-cty v1.13.0 h1:It5dfKTTZHe9aeppbNOda3mN7Ag7sg6QkBNm6TkyFa0= @@ -2845,6 +2859,7 @@ golang.org/x/sys v0.0.0-20181122145206-62eef0e2fa9b/go.mod h1:STP8DvDyc/dI5b8T5h golang.org/x/sys v0.0.0-20181205085412-a5c9d58dba9a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20181221143128-b4a75ba826a6/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190102155601-82a175fd1598/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= +golang.org/x/sys v0.0.0-20190204203706-41f3e6584952/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190209173611-3b5209105503/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190215142949-d0b11bdaac8a/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= golang.org/x/sys v0.0.0-20190222072716-a9d3bda3a223/go.mod h1:STP8DvDyc/dI5b8T5hshtkjS+E42TnysNCUPdjciGhY= diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index 4a30c8fa95b..ae8634cf399 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -179,16 +179,20 @@ func (moa *MultiOrgAlertmanager) setupClustering(cfg *setting.Cfg) error { // Redis setup. if cfg.UnifiedAlerting.HARedisAddr != "" { redisPeer, err := newRedisPeer(redisConfig{ - addr: cfg.UnifiedAlerting.HARedisAddr, - name: cfg.UnifiedAlerting.HARedisPeerName, - prefix: cfg.UnifiedAlerting.HARedisPrefix, - password: cfg.UnifiedAlerting.HARedisPassword, - username: cfg.UnifiedAlerting.HARedisUsername, - db: cfg.UnifiedAlerting.HARedisDB, - maxConns: cfg.UnifiedAlerting.HARedisMaxConns, - tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled, - tls: cfg.UnifiedAlerting.HARedisTLSConfig, - clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled, + addr: cfg.UnifiedAlerting.HARedisAddr, + name: cfg.UnifiedAlerting.HARedisPeerName, + prefix: cfg.UnifiedAlerting.HARedisPrefix, + password: cfg.UnifiedAlerting.HARedisPassword, + username: cfg.UnifiedAlerting.HARedisUsername, + db: cfg.UnifiedAlerting.HARedisDB, + maxConns: cfg.UnifiedAlerting.HARedisMaxConns, + tlsEnabled: cfg.UnifiedAlerting.HARedisTLSEnabled, + tls: cfg.UnifiedAlerting.HARedisTLSConfig, + clusterMode: cfg.UnifiedAlerting.HARedisClusterModeEnabled, + sentinelMode: cfg.UnifiedAlerting.HARedisSentinelModeEnabled, + masterName: cfg.UnifiedAlerting.HARedisSentinelMasterName, + sentinelUsername: cfg.UnifiedAlerting.HARedisSentinelUsername, + sentinelPassword: cfg.UnifiedAlerting.HARedisSentinelPassword, }, clusterLogger, moa.metrics.Registerer, cfg.UnifiedAlerting.HAPushPullInterval) if err != nil { return fmt.Errorf("unable to initialize redis: %w", err) diff --git a/pkg/services/ngalert/notifier/redis_peer.go b/pkg/services/ngalert/notifier/redis_peer.go index 11b8c5d6d1c..3726aac9583 100644 --- a/pkg/services/ngalert/notifier/redis_peer.go +++ b/pkg/services/ngalert/notifier/redis_peer.go @@ -23,14 +23,18 @@ import ( ) type redisConfig struct { - addr string - username string - password string - db int - name string - prefix string - maxConns int - clusterMode bool + addr string + username string + password string + db int + name string + prefix string + maxConns int + clusterMode bool + sentinelMode bool + masterName string + sentinelUsername string + sentinelPassword string tlsEnabled bool tls dstls.ClientConfig @@ -111,21 +115,19 @@ func newRedisPeer(cfg redisConfig, logger log.Logger, reg prometheus.Registerer, } } - opts := &redis.UniversalOptions{ + rdb := redis.NewUniversalClient(&redis.UniversalOptions{ Addrs: addrs, Username: cfg.username, Password: cfg.password, DB: cfg.db, PoolSize: poolSize, TLSConfig: tlsClientConfig, - } - var rdb redis.UniversalClient - if cfg.clusterMode { - rdb = redis.NewClusterClient(opts.Cluster()) - } else { - rdb = redis.NewClient(opts.Simple()) - } + // Options specific to Sentinel mode. + MasterName: cfg.masterName, + SentinelUsername: cfg.sentinelUsername, + SentinelPassword: cfg.sentinelPassword, + }) cmd := rdb.Ping(context.Background()) if cmd.Err() != nil { diff --git a/pkg/services/ngalert/notifier/redis_peer_test.go b/pkg/services/ngalert/notifier/redis_peer_test.go index 9b88198cca2..7caf8deaf44 100644 --- a/pkg/services/ngalert/notifier/redis_peer_test.go +++ b/pkg/services/ngalert/notifier/redis_peer_test.go @@ -8,6 +8,7 @@ import ( "testing" "time" + "github.com/Bose/minisentinel" "github.com/alicebob/miniredis/v2" dstls "github.com/grafana/dskit/crypto/tls" "github.com/grafana/grafana/pkg/infra/log" @@ -45,6 +46,28 @@ func TestNewRedisPeerClusterMode(t *testing.T) { require.NoError(t, ping.Err()) } +func TestNewRedisPeerSentinelMode(t *testing.T) { + // Can't use RunTLS here because minisentinel does not support TLS. + mr, err := miniredis.Run() + require.NoError(t, err) + defer mr.Close() + + ms := minisentinel.NewSentinel(mr, minisentinel.WithReplica(mr)) + err = ms.Start() + require.NoError(t, err) + defer ms.Close() + + redisPeer, err := newRedisPeer(redisConfig{ + sentinelMode: true, + masterName: ms.MasterInfo().Name, + addr: ms.Addr(), + }, log.NewNopLogger(), prometheus.NewRegistry(), time.Second*60) + require.NoError(t, err) + + ping := redisPeer.redis.Ping(context.Background()) + require.NoError(t, ping.Err()) +} + func TestNewRedisPeerWithTLS(t *testing.T) { // Write client and server certificates/keys to tempDir, both issued by the same CA certPaths := createX509TestDir(t) diff --git a/pkg/setting/setting_unified_alerting.go b/pkg/setting/setting_unified_alerting.go index b875c06b184..4d26114ea75 100644 --- a/pkg/setting/setting_unified_alerting.go +++ b/pkg/setting/setting_unified_alerting.go @@ -69,6 +69,11 @@ const ( lokiDefaultMaxQuerySize = 65536 // 64kb ) +var ( + errHARedisBothClusterAndSentinel = fmt.Errorf("'ha_redis_cluster_mode_enabled' and 'ha_redis_sentinel_mode_enabled' are mutually exclusive") + errHARedisSentinelMasterNameRequired = fmt.Errorf("'ha_redis_sentinel_master_name' is required when 'ha_redis_sentinel_mode_enabled' is true") +) + type UnifiedAlertingSettings struct { AdminConfigPollInterval time.Duration AlertmanagerConfigPollInterval time.Duration @@ -83,6 +88,10 @@ type UnifiedAlertingSettings struct { HAPushPullInterval time.Duration HALabel string HARedisClusterModeEnabled bool + HARedisSentinelModeEnabled bool + HARedisSentinelMasterName string + HARedisSentinelUsername string + HARedisSentinelPassword string HARedisAddr string HARedisPeerName string HARedisPrefix string @@ -276,6 +285,16 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error { uaCfg.HAAdvertiseAddr = ua.Key("ha_advertise_address").MustString("") uaCfg.HALabel = ua.Key("ha_label").MustString("") uaCfg.HARedisClusterModeEnabled = ua.Key("ha_redis_cluster_mode_enabled").MustBool(false) + uaCfg.HARedisSentinelModeEnabled = ua.Key("ha_redis_sentinel_mode_enabled").MustBool(false) + if uaCfg.HARedisClusterModeEnabled && uaCfg.HARedisSentinelModeEnabled { + return errHARedisBothClusterAndSentinel + } + uaCfg.HARedisSentinelMasterName = ua.Key("ha_redis_sentinel_master_name").MustString("") + if uaCfg.HARedisSentinelModeEnabled && uaCfg.HARedisSentinelMasterName == "" { + return errHARedisSentinelMasterNameRequired + } + uaCfg.HARedisSentinelUsername = ua.Key("ha_redis_sentinel_username").MustString("") + uaCfg.HARedisSentinelPassword = ua.Key("ha_redis_sentinel_password").MustString("") uaCfg.HARedisAddr = ua.Key("ha_redis_address").MustString("") uaCfg.HARedisPeerName = ua.Key("ha_redis_peer_name").MustString("") uaCfg.HARedisPrefix = ua.Key("ha_redis_prefix").MustString("") diff --git a/pkg/setting/setting_unified_alerting_test.go b/pkg/setting/setting_unified_alerting_test.go index bd972e7ef54..0f6ae5663da 100644 --- a/pkg/setting/setting_unified_alerting_test.go +++ b/pkg/setting/setting_unified_alerting_test.go @@ -350,3 +350,72 @@ func TestHARedisTLSSettings(t *testing.T) { require.Equal(t, cipherSuites, cfg.UnifiedAlerting.HARedisTLSConfig.CipherSuites) require.Equal(t, minVersion, cfg.UnifiedAlerting.HARedisTLSConfig.MinVersion) } + +func TestHARedisSentinelModeSettings(t *testing.T) { + testCases := []struct { + desc string + haRedisSentinelModeEnabled bool + haRedisClusterModeEnabled bool + haRedisSentinelMasterName string + haRedisSentinelUsername string + haRedisSentinelPassword string + expectedErr error + }{ + { + desc: "should not fail when Sentinel mode is enabled and master name is set", + haRedisSentinelModeEnabled: true, + haRedisSentinelMasterName: "exampleMasterName", + }, + { + desc: "should not fail when Sentinel mode is enabled, master name is set, and Sentinel username and password are provided", + haRedisSentinelModeEnabled: true, + haRedisSentinelMasterName: "exampleMasterName", + haRedisSentinelUsername: "exampleSentinelUsername", + haRedisSentinelPassword: "exampleSentinelPassword", + }, + { + desc: "should fail when Sentinel mode is enabled but master name is not set", + haRedisSentinelModeEnabled: true, + expectedErr: errHARedisSentinelMasterNameRequired, + }, + { + desc: "should fail when both Sentinel mode and Cluster mode are enabled", + haRedisSentinelModeEnabled: true, + haRedisClusterModeEnabled: true, + expectedErr: errHARedisBothClusterAndSentinel, + }, + } + + for _, tc := range testCases { + t.Run(tc.desc, func(t *testing.T) { + f := ini.Empty() + section, err := f.NewSection("unified_alerting") + require.NoError(t, err) + + _, err = section.NewKey("ha_redis_sentinel_mode_enabled", strconv.FormatBool(tc.haRedisSentinelModeEnabled)) + require.NoError(t, err) + _, err = section.NewKey("ha_redis_cluster_mode_enabled", strconv.FormatBool(tc.haRedisClusterModeEnabled)) + require.NoError(t, err) + _, err = section.NewKey("ha_redis_sentinel_master_name", tc.haRedisSentinelMasterName) + require.NoError(t, err) + _, err = section.NewKey("ha_redis_sentinel_username", tc.haRedisSentinelUsername) + require.NoError(t, err) + _, err = section.NewKey("ha_redis_sentinel_password", tc.haRedisSentinelPassword) + require.NoError(t, err) + + cfg := NewCfg() + err = cfg.ReadUnifiedAlertingSettings(f) + if tc.expectedErr == nil { + require.NoError(t, err) + } else { + require.ErrorIs(t, err, tc.expectedErr) + return + } + + require.Equal(t, tc.haRedisSentinelModeEnabled, cfg.UnifiedAlerting.HARedisSentinelModeEnabled) + require.Equal(t, tc.haRedisSentinelMasterName, cfg.UnifiedAlerting.HARedisSentinelMasterName) + require.Equal(t, tc.haRedisSentinelUsername, cfg.UnifiedAlerting.HARedisSentinelUsername) + require.Equal(t, tc.haRedisSentinelPassword, cfg.UnifiedAlerting.HARedisSentinelPassword) + }) + } +}