operator: Change endpoints for generated liveness and readiness probes (#5576)

3 years ago · e75257f8dd
parent b4ac996b87
commit e75257f8dd
8 changed files with 54 additions and 149 deletions
--- a/operator/CHANGELOG.md
+++ b/operator/CHANGELOG.md
@ -1,5 +1,6 @@
 ## Main

+- [5576](https://github.com/grafana/loki/pull/5576) **xperimental**: Change endpoints for generated liveness and readiness probes
 - [5560](https://github.com/grafana/loki/pull/5560) **periklis**: Fix service monitor's server name for operator metrics
 - [5345](https://github.com/grafana/loki/pull/5345) **ronensc**: Add flag to create Prometheus rules
 - [4974](https://github.com/grafana/loki/pull/5432) **Red-GV**: Provide storage configuration for Azure, GCS, and Swift through common_config
--- a/operator/internal/manifests/compactor.go
+++ b/operator/internal/manifests/compactor.go
@ -61,33 +61,8 @@ func NewCompactorStatefulSet(opts Options) *appsv1.StatefulSet {
 					fmt.Sprintf("-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiConfigFileName)),
 					fmt.Sprintf("-runtime-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiRuntimeConfigFileName)),
 				},
-				ReadinessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/ready",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					PeriodSeconds:       10,
-					InitialDelaySeconds: 15,
-					TimeoutSeconds:      1,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				ReadinessProbe: lokiReadinessProbe(),
+				LivenessProbe:  lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/distributor.go
+++ b/operator/internal/manifests/distributor.go
@ -74,33 +74,8 @@ func NewDistributorDeployment(opts Options) *appsv1.Deployment {
 					fmt.Sprintf("-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiConfigFileName)),
 					fmt.Sprintf("-runtime-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiRuntimeConfigFileName)),
 				},
-				ReadinessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/ready",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					PeriodSeconds:       10,
-					InitialDelaySeconds: 15,
-					TimeoutSeconds:      1,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				ReadinessProbe: lokiReadinessProbe(),
+				LivenessProbe:  lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/indexgateway.go
+++ b/operator/internal/manifests/indexgateway.go
@ -60,33 +60,8 @@ func NewIndexGatewayStatefulSet(opts Options) *appsv1.StatefulSet {
 					fmt.Sprintf("-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiConfigFileName)),
 					fmt.Sprintf("-runtime-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiRuntimeConfigFileName)),
 				},
-				ReadinessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/ready",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					PeriodSeconds:       10,
-					InitialDelaySeconds: 15,
-					TimeoutSeconds:      1,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				ReadinessProbe: lokiReadinessProbe(),
+				LivenessProbe:  lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/ingester.go
+++ b/operator/internal/manifests/ingester.go
@ -60,33 +60,8 @@ func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet {
 					fmt.Sprintf("-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiConfigFileName)),
 					fmt.Sprintf("-runtime-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiRuntimeConfigFileName)),
 				},
-				ReadinessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/ready",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					PeriodSeconds:       10,
-					InitialDelaySeconds: 15,
-					TimeoutSeconds:      1,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				ReadinessProbe: lokiReadinessProbe(),
+				LivenessProbe:  lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/querier.go
+++ b/operator/internal/manifests/querier.go
@ -59,33 +59,8 @@ func NewQuerierDeployment(opts Options) *appsv1.Deployment {
 					fmt.Sprintf("-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiConfigFileName)),
 					fmt.Sprintf("-runtime-config.file=%s", path.Join(config.LokiConfigMountDir, config.LokiRuntimeConfigFileName)),
 				},
-				ReadinessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/ready",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					PeriodSeconds:       10,
-					InitialDelaySeconds: 15,
-					TimeoutSeconds:      1,
-					SuccessThreshold:    1,
-					FailureThreshold:    3,
-				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				ReadinessProbe: lokiReadinessProbe(),
+				LivenessProbe:  lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/query-frontend.go
+++ b/operator/internal/manifests/query-frontend.go
@ -68,7 +68,11 @@ func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment {
 				ReadinessProbe: &corev1.Probe{
 					Handler: corev1.Handler{
 						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
+							// The frontend will only return ready once a querier has connected to it.
+							// Because the service used for connecting the querier to the frontend only lists ready
+							// instances there's sequencing issue. For now, we re-use the liveness-probe path
+							// for the readiness-probe as a workaround.
+							Path:   lokiLivenessPath,
 							Port:   intstr.FromInt(httpPort),
 							Scheme: corev1.URISchemeHTTP,
 						},
@ -79,19 +83,7 @@ func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment {
 					SuccessThreshold:    1,
 					FailureThreshold:    3,
 				},
-				LivenessProbe: &corev1.Probe{
-					Handler: corev1.Handler{
-						HTTPGet: &corev1.HTTPGetAction{
-							Path:   "/metrics",
-							Port:   intstr.FromInt(httpPort),
-							Scheme: corev1.URISchemeHTTP,
-						},
-					},
-					TimeoutSeconds:   2,
-					PeriodSeconds:    30,
-					FailureThreshold: 10,
-					SuccessThreshold: 1,
-				},
+				LivenessProbe: lokiLivenessProbe(),
 				Ports: []corev1.ContainerPort{
 					{
 						Name:          lokiHTTPPortName,
--- a/operator/internal/manifests/var.go
+++ b/operator/internal/manifests/var.go
@ -7,6 +7,7 @@ import (
 	monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1"
 	corev1 "k8s.io/api/core/v1"
 	"k8s.io/apimachinery/pkg/labels"
+	"k8s.io/apimachinery/pkg/util/intstr"
 )

 const (
@ -19,6 +20,9 @@ const (
 	lokiGRPCPortName   = "grpclb"
 	lokiGossipPortName = "gossip-ring"

+	lokiLivenessPath  = "/loki/api/v1/status/buildinfo"
+	lokiReadinessPath = "/ready"
+
 	gatewayContainerName    = "gateway"
 	gatewayHTTPPort         = 8080
 	gatewayInternalPort     = 8081
@ -235,3 +239,36 @@ func serviceMonitorEndpoint(portName, serviceName, namespace string, enableTLS b
 		Scheme: "http",
 	}
 }
+
+func lokiLivenessProbe() *corev1.Probe {
+	return &corev1.Probe{
+		Handler: corev1.Handler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   lokiLivenessPath,
+				Port:   intstr.FromInt(httpPort),
+				Scheme: corev1.URISchemeHTTP,
+			},
+		},
+		TimeoutSeconds:   2,
+		PeriodSeconds:    30,
+		FailureThreshold: 10,
+		SuccessThreshold: 1,
+	}
+}
+
+func lokiReadinessProbe() *corev1.Probe {
+	return &corev1.Probe{
+		Handler: corev1.Handler{
+			HTTPGet: &corev1.HTTPGetAction{
+				Path:   lokiReadinessPath,
+				Port:   intstr.FromInt(httpPort),
+				Scheme: corev1.URISchemeHTTP,
+			},
+		},
+		PeriodSeconds:       10,
+		InitialDelaySeconds: 15,
+		TimeoutSeconds:      1,
+		SuccessThreshold:    1,
+		FailureThreshold:    3,
+	}
+}