From 65645ea8e65cb1ff5095f9c7791e968b2ec410f6 Mon Sep 17 00:00:00 2001 From: Mohamed-Amine Bouqsimi Date: Tue, 12 Jul 2022 12:29:51 +0100 Subject: [PATCH] operator: Add support for tail TLS encryption (#6663) --- operator/CHANGELOG.md | 2 +- operator/internal/manifests/config.go | 10 +- .../manifests/internal/config/build_test.go | 30 ++- .../internal/config/loki-config.yaml | 2 +- .../manifests/internal/config/options.go | 2 + operator/internal/manifests/query-frontend.go | 72 ++++-- .../internal/manifests/query-frontend_test.go | 208 +++++++++++------- 7 files changed, 206 insertions(+), 120 deletions(-) diff --git a/operator/CHANGELOG.md b/operator/CHANGELOG.md index 30ed43d79e..a95cf2d0e7 100644 --- a/operator/CHANGELOG.md +++ b/operator/CHANGELOG.md @@ -1,6 +1,6 @@ ## Main -- [6646](https://github.com/grafana/loki/pull/6646) **periklis**: Update Loki operand to v2.6.0 +- [6663](https://github.com/grafana/loki/pull/6663) **aminesnow**: Generalize live tail fix to all clusters using TLS - [6443](https://github.com/grafana/loki/pull/6443) **aminesnow**: Fix live tail of logs not working on OpenShift-based clusters - [6646](https://github.com/grafana/loki/pull/6646) **periklis**: Update Loki operand to v2.6.0 - [6594](https://github.com/grafana/loki/pull/6594) **xperimental**: Disable client certificate authentication on gateway diff --git a/operator/internal/manifests/config.go b/operator/internal/manifests/config.go index 6405564a59..25f6f9e621 100644 --- a/operator/internal/manifests/config.go +++ b/operator/internal/manifests/config.go @@ -69,6 +69,11 @@ func ConfigOptions(opt Options) config.Options { } } + protocol := "http" + if opt.Gates.HTTPEncryption { + protocol = "https" + } + return config.Options{ Stack: opt.Stack, Namespace: opt.Namespace, @@ -82,8 +87,9 @@ func ConfigOptions(opt Options) config.Options { Port: gossipPort, }, Querier: config.Address{ - FQDN: fqdn(NewQuerierHTTPService(opt).GetName(), opt.Namespace), - Port: httpPort, + Protocol: protocol, + FQDN: fqdn(NewQuerierHTTPService(opt).GetName(), opt.Namespace), + Port: httpPort, }, IndexGateway: config.Address{ FQDN: fqdn(NewIndexGatewayGRPCService(opt).GetName(), opt.Namespace), diff --git a/operator/internal/manifests/internal/config/build_test.go b/operator/internal/manifests/internal/config/build_test.go index 1a98943f98..94353f5394 100644 --- a/operator/internal/manifests/internal/config/build_test.go +++ b/operator/internal/manifests/internal/config/build_test.go @@ -198,8 +198,9 @@ overrides: Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", @@ -444,8 +445,9 @@ overrides: Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", @@ -513,8 +515,9 @@ func TestBuild_ConfigAndRuntimeConfig_CreateLokiConfigFailed(t *testing.T) { Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", @@ -795,8 +798,9 @@ overrides: Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", @@ -1125,8 +1129,9 @@ overrides: Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", @@ -1469,8 +1474,9 @@ overrides: Port: 7946, }, Querier: Address{ - FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", - Port: 3100, + Protocol: "http", + FQDN: "loki-querier-http-lokistack-dev.default.svc.cluster.local", + Port: 3100, }, IndexGateway: Address{ FQDN: "loki-index-gateway-grpc-lokistack-dev.default.svc.cluster.local", diff --git a/operator/internal/manifests/internal/config/loki-config.yaml b/operator/internal/manifests/internal/config/loki-config.yaml index 1b15added8..51a7ce3c91 100644 --- a/operator/internal/manifests/internal/config/loki-config.yaml +++ b/operator/internal/manifests/internal/config/loki-config.yaml @@ -48,7 +48,7 @@ compactor: compaction_interval: 2h working_directory: {{ .StorageDirectory }}/compactor frontend: - tail_proxy_url: http://{{ .Querier.FQDN }}:{{ .Querier.Port }} + tail_proxy_url: {{ .Querier.Protocol }}://{{ .Querier.FQDN }}:{{ .Querier.Port }} compress_responses: true max_outstanding_per_tenant: 256 log_queries_longer_than: 5s diff --git a/operator/internal/manifests/internal/config/options.go b/operator/internal/manifests/internal/config/options.go index ef07ae5103..3102e46289 100644 --- a/operator/internal/manifests/internal/config/options.go +++ b/operator/internal/manifests/internal/config/options.go @@ -30,6 +30,8 @@ type Options struct { // Address FQDN and port for a k8s service. type Address struct { + // Protocol is optional + Protocol string // FQDN is required FQDN string // Port is required diff --git a/operator/internal/manifests/query-frontend.go b/operator/internal/manifests/query-frontend.go index d887fa0d58..8b2b0e668c 100644 --- a/operator/internal/manifests/query-frontend.go +++ b/operator/internal/manifests/query-frontend.go @@ -4,9 +4,9 @@ import ( "fmt" "path" - lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/ViaQ/logerr/v2/kverrors" "github.com/grafana/loki/operator/internal/manifests/internal/config" - "github.com/grafana/loki/operator/internal/manifests/openshift" + "github.com/imdario/mergo" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" @@ -31,13 +31,6 @@ func BuildQueryFrontend(opts Options) ([]client.Object, error) { } } - if opts.Stack.Tenants != nil { - mode := opts.Stack.Tenants.Mode - if err := configureQueryFrontendDeploymentForMode(deployment, mode, &opts); err != nil { - return nil, err - } - } - return []client.Object{ deployment, NewQueryFrontendGRPCService(opts), @@ -219,6 +212,11 @@ func NewQueryFrontendHTTPService(opts Options) *corev1.Service { func configureQueryFrontendHTTPServicePKI(deployment *appsv1.Deployment, stackName string) error { serviceName := serviceNameQueryFrontendHTTP(stackName) + caBundleName := signingCABundleName(stackName) + + if err := configureTailCA(deployment, lokiFrontendContainerName, caBundleName, caBundleDir, caFile); err != nil { + return err + } return configureHTTPServicePKI(&deployment.Spec.Template.Spec, serviceName) } @@ -227,18 +225,54 @@ func configureQueryFrontendGRPCServicePKI(deployment *appsv1.Deployment, stackNa return configureGRPCServicePKI(&deployment.Spec.Template.Spec, serviceName) } -func configureQueryFrontendDeploymentForMode(deployment *appsv1.Deployment, mode lokiv1.ModeType, opts *Options) error { - switch mode { - case lokiv1.Static, lokiv1.Dynamic: - return nil // nothing to configure - case lokiv1.OpenshiftLogging: - url := fmt.Sprintf("https://%s:%d", fqdn(serviceNameQuerierHTTP(opts.Name), opts.Namespace), httpPort) - caBundleName := signingCABundleName(opts.Name) - - if opts.Gates.ServiceMonitorTLSEndpoints { - return openshift.ConfigureQueryFrontendDeployment(deployment, url, lokiFrontendContainerName, caBundleName, caBundleDir, caFile) +// ConfigureQueryFrontendDeployment configures CA certificate when TLS is enabled. +func configureTailCA(d *appsv1.Deployment, + qfContainerName, caBundleVolumeName, caDir, caFile string, +) error { + var qfIdx int + for i, c := range d.Spec.Template.Spec.Containers { + if c.Name == qfContainerName { + qfIdx = i + break } } + containerSpec := corev1.Container{ + Args: []string{ + fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caDir, caFile), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: caBundleVolumeName, + ReadOnly: true, + MountPath: caDir, + }, + }, + } + + p := corev1.PodSpec{ + Volumes: []corev1.Volume{ + { + Name: caBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: caBundleVolumeName, + }, + }, + }, + }, + }, + } + + if err := mergo.Merge(&d.Spec.Template.Spec.Containers[qfIdx], containerSpec, mergo.WithAppendSlice); err != nil { + return kverrors.Wrap(err, "failed to add tls config args") + } + + if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithAppendSlice); err != nil { + return kverrors.Wrap(err, "failed to add tls volumes") + } + return nil } diff --git a/operator/internal/manifests/query-frontend_test.go b/operator/internal/manifests/query-frontend_test.go index 66cd650fb4..92ded54dd0 100644 --- a/operator/internal/manifests/query-frontend_test.go +++ b/operator/internal/manifests/query-frontend_test.go @@ -1,13 +1,16 @@ package manifests import ( + "fmt" + "path" "testing" - v1 "github.com/grafana/loki/operator/apis/config/v1" lokiv1 "github.com/grafana/loki/operator/apis/loki/v1" + "github.com/grafana/loki/operator/internal/manifests/internal/config" "github.com/stretchr/testify/require" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" ) func TestNewQueryFrontendDeployment_SelectorMatchesLabels(t *testing.T) { @@ -48,61 +51,49 @@ func TestNewQueryFrontendDeployment_HasTemplateConfigHashAnnotation(t *testing.T require.Equal(t, annotations[expected], "deadbeef") } -func TestConfigureQueryFrontendDeploymentForMode(t *testing.T) { - type tt struct { - desc string - opts *Options - dpl *appsv1.Deployment - want *appsv1.Deployment - } - - tc := []tt{ - { - desc: "static mode", - opts: &Options{ - Stack: lokiv1.LokiStackSpec{ - Tenants: &lokiv1.TenantsSpec{ - Mode: lokiv1.Static, - }, +func TestConfigureQueryFrontendHTTPServicePKI(t *testing.T) { + opts := Options{ + Name: "abcd", + Namespace: "efgh", + Stack: lokiv1.LokiStackSpec{ + Template: &lokiv1.LokiTemplateSpec{ + QueryFrontend: &lokiv1.LokiComponentSpec{ + Replicas: 1, }, }, - dpl: &appsv1.Deployment{}, - want: &appsv1.Deployment{}, }, - { - desc: "dynamic mode", - opts: &Options{ - Stack: lokiv1.LokiStackSpec{ - Tenants: &lokiv1.TenantsSpec{ - Mode: lokiv1.Dynamic, - }, - }, - }, - dpl: &appsv1.Deployment{}, - want: &appsv1.Deployment{}, + } + d := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: appsv1.SchemeGroupVersion.String(), }, - { - desc: "openshift-logging mode", - opts: &Options{ - Name: "test", - Namespace: "test-ns", - Stack: lokiv1.LokiStackSpec{ - Tenants: &lokiv1.TenantsSpec{ - Mode: lokiv1.OpenshiftLogging, - }, - }, - Gates: v1.FeatureGates{ - ServiceMonitorTLSEndpoints: true, - }, - }, - dpl: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: lokiFrontendContainerName, + Args: []string{ + "-target=query-frontend", + }, + VolumeMounts: []corev1.VolumeMount{ { - Args: []string{ - "-target=query-frontend", + Name: configVolumeName, + ReadOnly: false, + MountPath: config.LokiConfigMountDir, + }, + }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: lokiConfigMapName(opts.Name), }, }, }, @@ -110,52 +101,99 @@ func TestConfigureQueryFrontendDeploymentForMode(t *testing.T) { }, }, }, - want: &appsv1.Deployment{ - Spec: appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ + }, + } + + caBundleVolumeName := signingCABundleName(opts.Name) + serviceName := serviceNameQueryFrontendHTTP(opts.Name) + expected := appsv1.Deployment{ + TypeMeta: metav1.TypeMeta{ + Kind: "Deployment", + APIVersion: appsv1.SchemeGroupVersion.String(), + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: lokiFrontendContainerName, + Args: []string{ + "-target=query-frontend", + fmt.Sprintf("-frontend.tail-tls-config.tls-ca-path=%s/%s", caBundleDir, caFile), + fmt.Sprintf("-server.http-tls-cert-path=%s", path.Join(httpTLSDir, tlsCertFile)), + fmt.Sprintf("-server.http-tls-key-path=%s", path.Join(httpTLSDir, tlsKeyFile)), + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: configVolumeName, + ReadOnly: false, + MountPath: config.LokiConfigMountDir, + }, + { + Name: caBundleVolumeName, + ReadOnly: true, + MountPath: caBundleDir, + }, { - Args: []string{ - "-target=query-frontend", - "-frontend.tail-proxy-url=https://test-querier-http.test-ns.svc.cluster.local:3100", - "-frontend.tail-tls-config.tls-ca-path=/var/run/ca/service-ca.crt", + Name: serviceName, + ReadOnly: false, + MountPath: httpTLSDir, + }, + }, + ReadinessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: "test-ca-bundle", - ReadOnly: true, - MountPath: "/var/run/ca", - }, + }, + }, + LivenessProbe: &corev1.Probe{ + ProbeHandler: corev1.ProbeHandler{ + HTTPGet: &corev1.HTTPGetAction{ + Scheme: corev1.URISchemeHTTPS, }, }, }, - Volumes: []corev1.Volume{ - { - Name: "test-ca-bundle", - VolumeSource: corev1.VolumeSource{ - ConfigMap: &corev1.ConfigMapVolumeSource{ - DefaultMode: &defaultConfigMapMode, - LocalObjectReference: corev1.LocalObjectReference{ - Name: "test-ca-bundle", - }, - }, + }, + }, + Volumes: []corev1.Volume{ + { + Name: configVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: lokiConfigMapName(opts.Name), + }, + }, + }, + }, + { + Name: caBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: caBundleVolumeName, }, }, }, }, + { + Name: serviceName, + VolumeSource: corev1.VolumeSource{ + Secret: &corev1.SecretVolumeSource{ + SecretName: serviceName, + }, + }, + }, }, }, }, }, } - for _, tc := range tc { - tc := tc - t.Run(tc.desc, func(t *testing.T) { - t.Parallel() - err := configureQueryFrontendDeploymentForMode(tc.dpl, tc.opts.Stack.Tenants.Mode, tc.opts) - require.NoError(t, err) - require.Equal(t, tc.want, tc.dpl) - }) - } + + err := configureQueryFrontendHTTPServicePKI(&d, opts.Name) + require.Nil(t, err) + require.Equal(t, expected, d) }