operator: Add support for tail TLS encryption (#6663)

operator/CHANGELOG.md

@@ -1,6 +1,6 @@
 ## Main
 
-- [6646](https://github.com/grafana/loki/pull/6646) **periklis**: Update Loki operand to v2.6.0
+- [6663](https://github.com/grafana/loki/pull/6663) **aminesnow**: Generalize live tail fix to all clusters using TLS
 - [6443](https://github.com/grafana/loki/pull/6443) **aminesnow**: Fix live tail of logs not working on OpenShift-based clusters
 - [6646](https://github.com/grafana/loki/pull/6646) **periklis**: Update Loki operand to v2.6.0
 - [6594](https://github.com/grafana/loki/pull/6594) **xperimental**: Disable client certificate authentication on gateway

operator/internal/manifests/config.go

@@ -69,6 +69,11 @@ func ConfigOptions(opt Options) config.Options {
 		}
 	}
 
+	protocol := "http"
+	if opt.Gates.HTTPEncryption {
+		protocol = "https"
+	}
+
 	return config.Options{
 		Stack:     opt.Stack,
 		Namespace: opt.Namespace,
@@ -82,8 +87,9 @@ func ConfigOptions(opt Options) config.Options {
 			Port: gossipPort,
 		},
 		Querier: config.Address{
-			FQDN: fqdn(NewQuerierHTTPService(opt).GetName(), opt.Namespace),
-			Port: httpPort,
+			Protocol: protocol,
+			FQDN:     fqdn(NewQuerierHTTPService(opt).GetName(), opt.Namespace),
+			Port:     httpPort,
 		},
 		IndexGateway: config.Address{
 			FQDN: fqdn(NewIndexGatewayGRPCService(opt).GetName(), opt.Namespace),

operator/internal/manifests/internal/config/build_test.go

@@ -198,8 +198,9 @@ overrides:
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
@@ -444,8 +445,9 @@ overrides:
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
@@ -513,8 +515,9 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) {
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
@@ -795,8 +798,9 @@ overrides:
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
@@ -1125,8 +1129,9 @@ overrides:
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",
@@ -1469,8 +1474,9 @@ overrides:
 			Port: 7946,
 		},
 		Querier: Address{
-			FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local",
-			Port: 3100,
+			Protocol: "http",
+			FQDN:     "loki-querier-http-lokistack-dev.default.svc.cluster.local",
+			Port:     3100,
 		},
 		IndexGateway: Address{
 			FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local",

operator/internal/manifests/internal/config/loki-config.yaml

@@ -48,7 +48,7 @@ compactor:
   compaction_interval: 2h
   working_directory: {{ .StorageDirectory }}/compactor
 frontend:
-  tail_proxy_url: http://{{ .Querier.FQDN }}:{{ .Querier.Port }}
+  tail_proxy_url: {{ .Querier.Protocol }}://{{ .Querier.FQDN }}:{{ .Querier.Port }}
   compress_responses: true
   max_outstanding_per_tenant: 256
   log_queries_longer_than: 5s

operator/internal/manifests/internal/config/options.go

@@ -30,6 +30,8 @@ type Options struct {
 
 // Address FQDN and port for a k8s service.
 type Address struct {
+	// Protocol is optional
+	Protocol string
 	// FQDN is required
 	FQDN string
 	// Port is required

operator/internal/manifests/query-frontend.go

@@ -4,9 +4,9 @@ import (
 	"fmt"
 	"path"
 
-	lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
+	"github.com/ViaQ/logerr/v2/kverrors"
 	"github.com/grafana/loki/operator/internal/manifests/internal/config"
-	"github.com/grafana/loki/operator/internal/manifests/openshift"
+	"github.com/imdario/mergo"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -31,13 +31,6 @@ func BuildQueryFrontend(opts Options) ([]client.Object, error) {
 		}
 	}
 
-	if opts.Stack.Tenants != nil {
-		mode := opts.Stack.Tenants.Mode
-		if err := configureQueryFrontendDeploymentForMode(deployment, mode, &opts); err != nil {
-			return nil, err
-		}
-	}
-
 	return []client.Object{
 		deployment,
 		NewQueryFrontendGRPCService(opts),
@@ -219,6 +212,11 @@ func NewQueryFrontendHTTPService(opts Options) *corev1.Service {
 
 func configureQueryFrontendHTTPServicePKI(deployment *appsv1.Deployment, stackName string) error {
 	serviceName := serviceNameQueryFrontendHTTP(stackName)
+	caBundleName := signingCABundleName(stackName)
+
+	if err := configureTailCA(deployment, lokiFrontendContainerName, caBundleName, caBundleDir, caFile); err != nil {
+		return err
+	}
 	return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName)
 }
 
@@ -227,18 +225,54 @@ func configureQueryFrontendGRPCServicePKI(deployment *appsv1.Deployment, stackNa
 	return configureGRPCServicePKI(&deployment.Spec.Template.Spec, serviceName)
 }
 
-func configureQueryFrontendDeploymentForMode(deployment *appsv1.Deployment, mode lokiv1.ModeType, opts *Options) error {
-	switch mode {
-	case lokiv1.Static, lokiv1.Dynamic:
-		return nil // nothing to configure
-	case lokiv1.OpenshiftLogging:
-		url := fmt.Sprintf("https://%s:%d", fqdn(serviceNameQuerierHTTP(opts.Name), opts.Namespace), httpPort)
-		caBundleName := signingCABundleName(opts.Name)
-
-		if opts.Gates.ServiceMonitorTLSEndpoints {
-			return openshift.ConfigureQueryFrontendDeployment(deployment, url, lokiFrontendContainerName, caBundleName, caBundleDir, caFile)
+// ConfigureQueryFrontendDeployment configures CA certificate when TLS is enabled.
+func configureTailCA(d *appsv1.Deployment,
+	qfContainerName, caBundleVolumeName, caDir, caFile string,
+) error {
+	var qfIdx int
+	for i, c := range d.Spec.Template.Spec.Containers {
+		if c.Name == qfContainerName {
+			qfIdx = i
+			break
 		}
 	}
 
+	containerSpec := corev1.Container{
+		Args: []string{
+			fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caDir, caFile),
+		},
+		VolumeMounts: []corev1.VolumeMount{
+			{
+				Name:      caBundleVolumeName,
+				ReadOnly:  true,
+				MountPath: caDir,
+			},
+		},
+	}
+
+	p := corev1.PodSpec{
+		Volumes: []corev1.Volume{
+			{
+				Name: caBundleVolumeName,
+				VolumeSource: corev1.VolumeSource{
+					ConfigMap: &corev1.ConfigMapVolumeSource{
+						DefaultMode: &defaultConfigMapMode,
+						LocalObjectReference: corev1.LocalObjectReference{
+							Name: caBundleVolumeName,
+						},
+					},
+				},
+			},
+		},
+	}
+
+	if err := mergo.Merge(&d.Spec.Template.Spec.Containers[qfIdx], containerSpec, mergo.WithAppendSlice); err != nil {
+		return kverrors.Wrap(err, "failed to add tls config args")
+	}
+
+	if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithAppendSlice); err != nil {
+		return kverrors.Wrap(err, "failed to add tls volumes")
+	}
+
 	return nil
 }

operator/internal/manifests/query-frontend_test.go

@@ -1,13 +1,16 @@
 package manifests
 
 import (
+	"fmt"
+	"path"
 	"testing"
 
-	v1 "github.com/grafana/loki/operator/apis/config/v1"
 	lokiv1 "github.com/grafana/loki/operator/apis/loki/v1"
+	"github.com/grafana/loki/operator/internal/manifests/internal/config"
 	"github.com/stretchr/testify/require"
 	appsv1 "k8s.io/api/apps/v1"
 	corev1 "k8s.io/api/core/v1"
+	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 )
 
 func TestNewQueryFrontendDeployment_SelectorMatchesLabels(t *testing.T) {
@@ -48,61 +51,49 @@ func TestNewQueryFrontendDeployment_HasTemplateConfigHashAnnotation(t *testing.T
 	require.Equal(t, annotations[expected], "deadbeef")
 }
 
-func TestConfigureQueryFrontendDeploymentForMode(t *testing.T) {
-	type tt struct {
-		desc string
-		opts *Options
-		dpl  *appsv1.Deployment
-		want *appsv1.Deployment
-	}
-
-	tc := []tt{
-		{
-			desc: "static mode",
-			opts: &Options{
-				Stack: lokiv1.LokiStackSpec{
-					Tenants: &lokiv1.TenantsSpec{
-						Mode: lokiv1.Static,
-					},
+func TestConfigureQueryFrontendHTTPServicePKI(t *testing.T) {
+	opts := Options{
+		Name:      "abcd",
+		Namespace: "efgh",
+		Stack: lokiv1.LokiStackSpec{
+			Template: &lokiv1.LokiTemplateSpec{
+				QueryFrontend: &lokiv1.LokiComponentSpec{
+					Replicas: 1,
 				},
 			},
-			dpl:  &appsv1.Deployment{},
-			want: &appsv1.Deployment{},
 		},
-		{
-			desc: "dynamic mode",
-			opts: &Options{
-				Stack: lokiv1.LokiStackSpec{
-					Tenants: &lokiv1.TenantsSpec{
-						Mode: lokiv1.Dynamic,
-					},
-				},
-			},
-			dpl:  &appsv1.Deployment{},
-			want: &appsv1.Deployment{},
+	}
+	d := appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
 		},
-		{
-			desc: "openshift-logging mode",
-			opts: &Options{
-				Name:      "test",
-				Namespace: "test-ns",
-				Stack: lokiv1.LokiStackSpec{
-					Tenants: &lokiv1.TenantsSpec{
-						Mode: lokiv1.OpenshiftLogging,
-					},
-				},
-				Gates: v1.FeatureGates{
-					ServiceMonitorTLSEndpoints: true,
-				},
-			},
-			dpl: &appsv1.Deployment{
-				Spec: appsv1.DeploymentSpec{
-					Template: corev1.PodTemplateSpec{
-						Spec: corev1.PodSpec{
-							Containers: []corev1.Container{
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: lokiFrontendContainerName,
+							Args: []string{
+								"-target=query-frontend",
+							},
+							VolumeMounts: []corev1.VolumeMount{
 								{
-									Args: []string{
-										"-target=query-frontend",
+									Name:      configVolumeName,
+									ReadOnly:  false,
+									MountPath: config.LokiConfigMountDir,
+								},
+							},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: configVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									DefaultMode: &defaultConfigMapMode,
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: lokiConfigMapName(opts.Name),
 									},
 								},
 							},
@@ -110,52 +101,99 @@ func TestConfigureQueryFrontendDeploymentForMode(t *testing.T) {
 					},
 				},
 			},
-			want: &appsv1.Deployment{
-				Spec: appsv1.DeploymentSpec{
-					Template: corev1.PodTemplateSpec{
-						Spec: corev1.PodSpec{
-							Containers: []corev1.Container{
+		},
+	}
+
+	caBundleVolumeName := signingCABundleName(opts.Name)
+	serviceName := serviceNameQueryFrontendHTTP(opts.Name)
+	expected := appsv1.Deployment{
+		TypeMeta: metav1.TypeMeta{
+			Kind:       "Deployment",
+			APIVersion: appsv1.SchemeGroupVersion.String(),
+		},
+		Spec: appsv1.DeploymentSpec{
+			Template: corev1.PodTemplateSpec{
+				Spec: corev1.PodSpec{
+					Containers: []corev1.Container{
+						{
+							Name: lokiFrontendContainerName,
+							Args: []string{
+								"-target=query-frontend",
+								fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caBundleDir, caFile),
+								fmt.Sprintf("-server.http-tls-cert-path=%s", path.Join(httpTLSDir, tlsCertFile)),
+								fmt.Sprintf("-server.http-tls-key-path=%s", path.Join(httpTLSDir, tlsKeyFile)),
+							},
+							VolumeMounts: []corev1.VolumeMount{
+								{
+									Name:      configVolumeName,
+									ReadOnly:  false,
+									MountPath: config.LokiConfigMountDir,
+								},
+								{
+									Name:      caBundleVolumeName,
+									ReadOnly:  true,
+									MountPath: caBundleDir,
+								},
 								{
-									Args: []string{
-										"-target=query-frontend",
-										"-frontend.tail-proxy-url=https://test-querier-http.test-ns.svc.cluster.local:3100",
-										"-frontend.tail-tls-config.tls-ca-path=/var/run/ca/service-ca.crt",
+									Name:      serviceName,
+									ReadOnly:  false,
+									MountPath: httpTLSDir,
+								},
+							},
+							ReadinessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Scheme: corev1.URISchemeHTTPS,
 									},
-									VolumeMounts: []corev1.VolumeMount{
-										{
-											Name:      "test-ca-bundle",
-											ReadOnly:  true,
-											MountPath: "/var/run/ca",
-										},
+								},
+							},
+							LivenessProbe: &corev1.Probe{
+								ProbeHandler: corev1.ProbeHandler{
+									HTTPGet: &corev1.HTTPGetAction{
+										Scheme: corev1.URISchemeHTTPS,
 									},
 								},
 							},
-							Volumes: []corev1.Volume{
-								{
-									Name: "test-ca-bundle",
-									VolumeSource: corev1.VolumeSource{
-										ConfigMap: &corev1.ConfigMapVolumeSource{
-											DefaultMode: &defaultConfigMapMode,
-											LocalObjectReference: corev1.LocalObjectReference{
-												Name: "test-ca-bundle",
-											},
-										},
+						},
+					},
+					Volumes: []corev1.Volume{
+						{
+							Name: configVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									DefaultMode: &defaultConfigMapMode,
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: lokiConfigMapName(opts.Name),
+									},
+								},
+							},
+						},
+						{
+							Name: caBundleVolumeName,
+							VolumeSource: corev1.VolumeSource{
+								ConfigMap: &corev1.ConfigMapVolumeSource{
+									DefaultMode: &defaultConfigMapMode,
+									LocalObjectReference: corev1.LocalObjectReference{
+										Name: caBundleVolumeName,
 									},
 								},
 							},
 						},
+						{
+							Name: serviceName,
+							VolumeSource: corev1.VolumeSource{
+								Secret: &corev1.SecretVolumeSource{
+									SecretName: serviceName,
+								},
+							},
+						},
 					},
 				},
 			},
 		},
 	}
-	for _, tc := range tc {
-		tc := tc
-		t.Run(tc.desc, func(t *testing.T) {
-			t.Parallel()
-			err := configureQueryFrontendDeploymentForMode(tc.dpl, tc.opts.Stack.Tenants.Mode, tc.opts)
-			require.NoError(t, err)
-			require.Equal(t, tc.want, tc.dpl)
-		})
-	}
+
+	err := configureQueryFrontendHTTPServicePKI(&d, opts.Name)
+	require.Nil(t, err)
+	require.Equal(t, expected, d)
 }