From 0706b537ffd8aa1cfdcf803f6a8066b3aff85f0e Mon Sep 17 00:00:00 2001 From: Periklis Tsirakidis Date: Tue, 19 Oct 2021 16:17:36 +0200 Subject: [PATCH] Fix upstream calls to Loki using server-side TLS when provided (#97) --- internal/manifests/gateway.go | 4 +- internal/manifests/gateway_tenants.go | 13 +- internal/manifests/gateway_tenants_test.go | 413 +++++++++++++----- .../manifests/internal/config/build_test.go | 4 +- .../internal/config/loki-config.yaml | 2 +- internal/manifests/openshift/build.go | 8 +- internal/manifests/openshift/build_test.go | 6 +- internal/manifests/openshift/configure.go | 103 +++-- internal/manifests/openshift/options.go | 33 +- internal/manifests/openshift/service_ca.go | 26 ++ .../openshift/serviceaccount_test.go | 2 +- internal/manifests/openshift/var.go | 16 +- 12 files changed, 472 insertions(+), 158 deletions(-) create mode 100644 internal/manifests/openshift/service_ca.go diff --git a/internal/manifests/gateway.go b/internal/manifests/gateway.go index 72ef62c6f9..ad93703a75 100644 --- a/internal/manifests/gateway.go +++ b/internal/manifests/gateway.go @@ -49,7 +49,7 @@ func BuildGateway(opts Options) ([]client.Object, error) { if opts.Stack.Tenants != nil { mode := opts.Stack.Tenants.Mode - if err := configureDeploymentForMode(&dpl.Spec, mode, opts.Flags); err != nil { + if err := configureDeploymentForMode(dpl, mode, opts.Flags); err != nil { return nil, err } @@ -344,7 +344,7 @@ func gatewayConfigOptions(opt Options) gateway.Options { func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName string) error { var gwIndex int for i, c := range podSpec.Containers { - if c.Name == LabelGatewayComponent { + if c.Name == gatewayContainerName { gwIndex = i break } diff --git a/internal/manifests/gateway_tenants.go b/internal/manifests/gateway_tenants.go index cd5745ed27..29a96d5d7d 100644 --- a/internal/manifests/gateway_tenants.go +++ b/internal/manifests/gateway_tenants.go @@ -31,6 +31,7 @@ func ApplyGatewayDefaultOptions(opts *Options) error { serviceNameGatewayHTTP(opts.Name), gatewayHTTPPortName, ComponentLabels(LabelGatewayComponent, opts.Name), + opts.Flags.EnableCertificateSigningService, ) if err := mergo.Merge(&opts.OpenShiftOptions, &defaults, mergo.WithOverride); err != nil { @@ -41,18 +42,22 @@ func ApplyGatewayDefaultOptions(opts *Options) error { return nil } -func configureDeploymentForMode(d *appsv1.DeploymentSpec, mode lokiv1beta1.ModeType, flags FeatureFlags) error { +func configureDeploymentForMode(d *appsv1.Deployment, mode lokiv1beta1.ModeType, flags FeatureFlags) error { switch mode { case lokiv1beta1.Static, lokiv1beta1.Dynamic: return nil // nothing to configure case lokiv1beta1.OpenshiftLogging: - return openshift.ConfigureDeployment( + return openshift.ConfigureGatewayDeployment( d, + gatewayContainerName, tlsMetricsSercetVolume, gateway.LokiGatewayTLSDir, gateway.LokiGatewayCertFile, gateway.LokiGatewayKeyFile, + gateway.LokiGatewayCABundleDir, + gateway.LokiGatewayCAFile, flags.EnableTLSServiceMonitorConfig, + flags.EnableCertificateSigningService, ) } @@ -64,7 +69,7 @@ func configureServiceForMode(s *corev1.ServiceSpec, mode lokiv1beta1.ModeType) e case lokiv1beta1.Static, lokiv1beta1.Dynamic: return nil // nothing to configure case lokiv1beta1.OpenshiftLogging: - return openshift.ConfigureService(s) + return openshift.ConfigureGatewayService(s) } return nil @@ -101,7 +106,7 @@ func configureServiceMonitorForMode(sm *monitoringv1.ServiceMonitor, mode lokiv1 case lokiv1beta1.Static, lokiv1beta1.Dynamic: return nil // nothing to configure case lokiv1beta1.OpenshiftLogging: - return openshift.ConfigureServiceMonitor(sm, flags.EnableTLSServiceMonitorConfig) + return openshift.ConfigureGatewayServiceMonitor(sm, flags.EnableTLSServiceMonitorConfig) } return nil diff --git a/internal/manifests/gateway_tenants_test.go b/internal/manifests/gateway_tenants_test.go index b3aa86b947..5de87474bf 100644 --- a/internal/manifests/gateway_tenants_test.go +++ b/internal/manifests/gateway_tenants_test.go @@ -12,6 +12,7 @@ import ( appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" "k8s.io/apimachinery/pkg/util/intstr" ) @@ -142,79 +143,106 @@ func TestConfigureDeploymentForMode(t *testing.T) { desc string mode lokiv1beta1.ModeType flags FeatureFlags - dpl *appsv1.DeploymentSpec - want *appsv1.DeploymentSpec + dpl *appsv1.Deployment + want *appsv1.Deployment } tc := []tt{ { desc: "static mode", mode: lokiv1beta1.Static, - dpl: &appsv1.DeploymentSpec{}, - want: &appsv1.DeploymentSpec{}, + dpl: &appsv1.Deployment{}, + want: &appsv1.Deployment{}, }, { desc: "dynamic mode", mode: lokiv1beta1.Dynamic, - dpl: &appsv1.DeploymentSpec{}, - want: &appsv1.DeploymentSpec{}, + dpl: &appsv1.Deployment{}, + want: &appsv1.Deployment{}, }, { desc: "openshift-logging mode", mode: lokiv1beta1.OpenshiftLogging, - dpl: &appsv1.DeploymentSpec{}, - want: &appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "opa", - Image: "quay.io/observatorium/opa-openshift:latest", - Args: []string{ - "--log.level=warn", - "--opa.package=lokistack", - "--web.listen=:8082", - "--web.internal.listen=:8083", - "--web.healthchecks.url=http://localhost:8082", - `--openshift.mappings=application=loki.openshift.io`, - `--openshift.mappings=infrastructure=loki.openshift.io`, - `--openshift.mappings=audit=loki.openshift.io`, - }, - Ports: []corev1.ContainerPort{ - { - Name: openshift.GatewayOPAHTTPPortName, - ContainerPort: openshift.GatewayOPAHTTPPort, - Protocol: corev1.ProtocolTCP, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--logs.read.endpoint=http://example.com", + "--logs.tail.endpoint=http://example.com", + "--logs.write.endpoint=http://example.com", }, - { - Name: openshift.GatewayOPAInternalPortName, - ContainerPort: openshift.GatewayOPAInternalPort, - Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--logs.read.endpoint=http://example.com", + "--logs.tail.endpoint=http://example.com", + "--logs.write.endpoint=http://example.com", }, }, - LivenessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTP, + { + Name: "opa", + Image: "quay.io/observatorium/opa-openshift:latest", + Args: []string{ + "--log.level=warn", + "--opa.package=lokistack", + "--web.listen=:8082", + "--web.internal.listen=:8083", + "--web.healthchecks.url=http://localhost:8082", + `--openshift.mappings=application=loki.openshift.io`, + `--openshift.mappings=infrastructure=loki.openshift.io`, + `--openshift.mappings=audit=loki.openshift.io`, + }, + Ports: []corev1.ContainerPort{ + { + Name: openshift.GatewayOPAHTTPPortName, + ContainerPort: openshift.GatewayOPAHTTPPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: openshift.GatewayOPAInternalPortName, + ContainerPort: openshift.GatewayOPAInternalPort, + Protocol: corev1.ProtocolTCP, }, }, - TimeoutSeconds: 2, - PeriodSeconds: 30, - FailureThreshold: 10, - }, - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTP, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTP, + }, + }, + TimeoutSeconds: 2, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTP, + }, }, + TimeoutSeconds: 1, + PeriodSeconds: 5, + FailureThreshold: 12, }, - TimeoutSeconds: 1, - PeriodSeconds: 5, - FailureThreshold: 12, }, }, }, @@ -228,67 +256,246 @@ func TestConfigureDeploymentForMode(t *testing.T) { flags: FeatureFlags{ EnableTLSServiceMonitorConfig: true, }, - dpl: &appsv1.DeploymentSpec{}, - want: &appsv1.DeploymentSpec{ - Template: corev1.PodTemplateSpec{ - Spec: corev1.PodSpec{ - Containers: []corev1.Container{ - { - Name: "opa", - Image: "quay.io/observatorium/opa-openshift:latest", - Args: []string{ - "--log.level=warn", - "--opa.package=lokistack", - "--web.listen=:8082", - "--web.internal.listen=:8083", - "--web.healthchecks.url=http://localhost:8082", - "--tls.internal.server.cert-file=/var/run/tls/tls.crt", - "--tls.internal.server.key-file=/var/run/tls/tls.key", - `--openshift.mappings=application=loki.openshift.io`, - `--openshift.mappings=infrastructure=loki.openshift.io`, - `--openshift.mappings=audit=loki.openshift.io`, + dpl: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--logs.read.endpoint=http://example.com", + "--logs.tail.endpoint=http://example.com", + "--logs.write.endpoint=http://example.com", + }, + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--logs.read.endpoint=http://example.com", + "--logs.tail.endpoint=http://example.com", + "--logs.write.endpoint=http://example.com", + }, + }, + { + Name: "opa", + Image: "quay.io/observatorium/opa-openshift:latest", + Args: []string{ + "--log.level=warn", + "--opa.package=lokistack", + "--web.listen=:8082", + "--web.internal.listen=:8083", + "--web.healthchecks.url=http://localhost:8082", + "--tls.internal.server.cert-file=/var/run/tls/tls.crt", + "--tls.internal.server.key-file=/var/run/tls/tls.key", + `--openshift.mappings=application=loki.openshift.io`, + `--openshift.mappings=infrastructure=loki.openshift.io`, + `--openshift.mappings=audit=loki.openshift.io`, + }, + Ports: []corev1.ContainerPort{ + { + Name: openshift.GatewayOPAHTTPPortName, + ContainerPort: openshift.GatewayOPAHTTPPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: openshift.GatewayOPAInternalPortName, + ContainerPort: openshift.GatewayOPAInternalPort, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 2, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 1, + PeriodSeconds: 5, + FailureThreshold: 12, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsMetricsSercetVolume, + ReadOnly: true, + MountPath: gateway.LokiGatewayTLSDir, + }, + }, }, - Ports: []corev1.ContainerPort{ - { - Name: openshift.GatewayOPAHTTPPortName, - ContainerPort: openshift.GatewayOPAHTTPPort, - Protocol: corev1.ProtocolTCP, + }, + }, + }, + }, + }, + }, + { + desc: "openshift-logging mode with-cert-signing-service", + mode: lokiv1beta1.OpenshiftLogging, + flags: FeatureFlags{ + EnableTLSServiceMonitorConfig: true, + EnableCertificateSigningService: true, + }, + dpl: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--other.args=foo-bar", + "--logs.read.endpoint=http://example.com", + "--logs.tail.endpoint=http://example.com", + "--logs.write.endpoint=http://example.com", }, - { - Name: openshift.GatewayOPAInternalPortName, - ContainerPort: openshift.GatewayOPAInternalPort, - Protocol: corev1.ProtocolTCP, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "tls-secret", + ReadOnly: true, + MountPath: "/var/run/tls", + }, }, }, - LivenessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/live", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, + }, + Volumes: []corev1.Volume{ + { + Name: "tls-secret-volume", + }, + }, + }, + }, + }, + }, + want: &appsv1.Deployment{ + ObjectMeta: metav1.ObjectMeta{ + Name: "gateway", + }, + Spec: appsv1.DeploymentSpec{ + Template: corev1.PodTemplateSpec{ + Spec: corev1.PodSpec{ + Containers: []corev1.Container{ + { + Name: gatewayContainerName, + Args: []string{ + "--other.args=foo-bar", + "--logs.read.endpoint=https://example.com", + "--logs.tail.endpoint=https://example.com", + "--logs.write.endpoint=https://example.com", + "--logs.tls.ca-file=/var/run/ca/service-ca.crt", + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: "tls-secret", + ReadOnly: true, + MountPath: "/var/run/tls", + }, + { + Name: "gateway-ca-bundle", + ReadOnly: true, + MountPath: "/var/run/ca", }, }, - TimeoutSeconds: 2, - PeriodSeconds: 30, - FailureThreshold: 10, }, - ReadinessProbe: &corev1.Probe{ - Handler: corev1.Handler{ - HTTPGet: &corev1.HTTPGetAction{ - Path: "/ready", - Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), - Scheme: corev1.URISchemeHTTPS, + { + Name: "opa", + Image: "quay.io/observatorium/opa-openshift:latest", + Args: []string{ + "--log.level=warn", + "--opa.package=lokistack", + "--web.listen=:8082", + "--web.internal.listen=:8083", + "--web.healthchecks.url=http://localhost:8082", + "--tls.internal.server.cert-file=/var/run/tls/tls.crt", + "--tls.internal.server.key-file=/var/run/tls/tls.key", + `--openshift.mappings=application=loki.openshift.io`, + `--openshift.mappings=infrastructure=loki.openshift.io`, + `--openshift.mappings=audit=loki.openshift.io`, + }, + Ports: []corev1.ContainerPort{ + { + Name: openshift.GatewayOPAHTTPPortName, + ContainerPort: openshift.GatewayOPAHTTPPort, + Protocol: corev1.ProtocolTCP, + }, + { + Name: openshift.GatewayOPAInternalPortName, + ContainerPort: openshift.GatewayOPAInternalPort, + Protocol: corev1.ProtocolTCP, + }, + }, + LivenessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/live", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 2, + PeriodSeconds: 30, + FailureThreshold: 10, + }, + ReadinessProbe: &corev1.Probe{ + Handler: corev1.Handler{ + HTTPGet: &corev1.HTTPGetAction{ + Path: "/ready", + Port: intstr.FromInt(int(openshift.GatewayOPAInternalPort)), + Scheme: corev1.URISchemeHTTPS, + }, + }, + TimeoutSeconds: 1, + PeriodSeconds: 5, + FailureThreshold: 12, + }, + VolumeMounts: []corev1.VolumeMount{ + { + Name: tlsMetricsSercetVolume, + ReadOnly: true, + MountPath: gateway.LokiGatewayTLSDir, }, }, - TimeoutSeconds: 1, - PeriodSeconds: 5, - FailureThreshold: 12, }, - VolumeMounts: []corev1.VolumeMount{ - { - Name: tlsMetricsSercetVolume, - ReadOnly: true, - MountPath: gateway.LokiGatewayTLSDir, + }, + Volumes: []corev1.Volume{ + { + Name: "tls-secret-volume", + }, + { + Name: "gateway-ca-bundle", + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: "gateway-ca-bundle", + }, + }, }, }, }, diff --git a/internal/manifests/internal/config/build_test.go b/internal/manifests/internal/config/build_test.go index e09cecf75c..3e2d0f6acf 100644 --- a/internal/manifests/internal/config/build_test.go +++ b/internal/manifests/internal/config/build_test.go @@ -10,7 +10,7 @@ import ( func TestBuild_ConfigAndRuntimeConfig_NoRuntimeConfigGenerated(t *testing.T) { expCfg := ` --- -auth_enabled: false +auth_enabled: true chunk_store_config: chunk_cache_config: enable_fifocache: yes @@ -207,7 +207,7 @@ overrides: func TestBuild_ConfigAndRuntimeConfig_BothGenerated(t *testing.T) { expCfg := ` --- -auth_enabled: false +auth_enabled: true chunk_store_config: chunk_cache_config: enable_fifocache: yes diff --git a/internal/manifests/internal/config/loki-config.yaml b/internal/manifests/internal/config/loki-config.yaml index 2ac6a66f9a..bda2007129 100644 --- a/internal/manifests/internal/config/loki-config.yaml +++ b/internal/manifests/internal/config/loki-config.yaml @@ -1,5 +1,5 @@ --- -auth_enabled: false +auth_enabled: true chunk_store_config: chunk_cache_config: enable_fifocache: yes diff --git a/internal/manifests/openshift/build.go b/internal/manifests/openshift/build.go index b12b407345..59f6936587 100644 --- a/internal/manifests/openshift/build.go +++ b/internal/manifests/openshift/build.go @@ -5,10 +5,16 @@ import "sigs.k8s.io/controller-runtime/pkg/client" // Build returns a list of auxiliary openshift/k8s objects // for lokistack gateway deployments on OpenShift. func Build(opts Options) []client.Object { - return []client.Object{ + objs := []client.Object{ BuildRoute(opts), BuildServiceAccount(opts), BuildClusterRole(opts), BuildClusterRoleBinding(opts), } + + if opts.BuildOpts.EnableCertificateSigningService { + objs = append(objs, BuildServiceCAConfigMap(opts)) + } + + return objs } diff --git a/internal/manifests/openshift/build_test.go b/internal/manifests/openshift/build_test.go index 345403e105..4df9c63c3c 100644 --- a/internal/manifests/openshift/build_test.go +++ b/internal/manifests/openshift/build_test.go @@ -12,7 +12,7 @@ import ( ) func TestBuild_ServiceAccountRefMatches(t *testing.T) { - opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}) + opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}, false) objs := Build(opts) sa := objs[1].(*corev1.ServiceAccount) @@ -24,7 +24,7 @@ func TestBuild_ServiceAccountRefMatches(t *testing.T) { } func TestBuild_ClusterRoleRefMatches(t *testing.T) { - opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}) + opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}, false) objs := Build(opts) cr := objs[2].(*rbacv1.ClusterRole) @@ -35,7 +35,7 @@ func TestBuild_ClusterRoleRefMatches(t *testing.T) { } func TestBuild_ServiceAccountAnnotationsRouteRefMatches(t *testing.T) { - opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}) + opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}, false) objs := Build(opts) rt := objs[0].(*routev1.Route) diff --git a/internal/manifests/openshift/configure.go b/internal/manifests/openshift/configure.go index ef05ad9c92..78d1ebd448 100644 --- a/internal/manifests/openshift/configure.go +++ b/internal/manifests/openshift/configure.go @@ -1,13 +1,16 @@ package openshift import ( + "fmt" + "regexp" + "strings" + "github.com/ViaQ/logerr/kverrors" "github.com/imdario/mergo" monitoringv1 "github.com/prometheus-operator/prometheus-operator/pkg/apis/monitoring/v1" appsv1 "k8s.io/api/apps/v1" corev1 "k8s.io/api/core/v1" - networkingv1 "k8s.io/api/networking/v1" ) const ( @@ -19,33 +22,93 @@ const ( tenantAudit = "audit" ) -// defaultTenants represents the slice of all supported LokiStack on OpenShift. -var defaultTenants = []string{ - tenantApplication, - tenantInfrastructure, - tenantAudit, -} +var ( + // defaultTenants represents the slice of all supported LokiStack on OpenShift. + defaultTenants = []string{ + tenantApplication, + tenantInfrastructure, + tenantAudit, + } -// ConfigureDeployment merges an OpenPolicyAgent sidecar into the deployment spec. + logsEndpointRe = regexp.MustCompile(`.*logs..*.endpoint.*`) +) + +// ConfigureGatewayDeployment merges an OpenPolicyAgent sidecar into the deployment spec. // With this, the deployment will route authorization request to the OpenShift // apiserver through the sidecar. -func ConfigureDeployment(d *appsv1.DeploymentSpec, sercretVolumeName, tlsDir, certFile, keyFile string, withTLS bool) error { +func ConfigureGatewayDeployment( + d *appsv1.Deployment, + gwContainerName string, + sercretVolumeName, tlsDir, certFile, keyFile string, + caDir, caFile string, + withTLS, withCertSigningService bool, +) error { + var gwIndex int + for i, c := range d.Spec.Template.Spec.Containers { + if c.Name == gwContainerName { + gwIndex = i + break + } + } + + gwContainer := d.Spec.Template.Spec.Containers[gwIndex].DeepCopy() + gwArgs := gwContainer.Args + gwVolumes := d.Spec.Template.Spec.Volumes + + if withCertSigningService { + for i, a := range gwArgs { + if logsEndpointRe.MatchString(a) { + gwContainer.Args[i] = strings.Replace(a, "http", "https", 1) + } + } + + gwArgs = append(gwArgs, fmt.Sprintf("--logs.tls.ca-file=%s/%s", caDir, caFile)) + + caBundleVolumeName := serviceCABundleName(Options{ + BuildOpts: BuildOptions{ + GatewayName: d.GetName(), + }, + }) + + gwContainer.VolumeMounts = append(gwContainer.VolumeMounts, corev1.VolumeMount{ + Name: caBundleVolumeName, + ReadOnly: true, + MountPath: caDir, + }) + + gwVolumes = append(gwVolumes, corev1.Volume{ + Name: caBundleVolumeName, + VolumeSource: corev1.VolumeSource{ + ConfigMap: &corev1.ConfigMapVolumeSource{ + DefaultMode: &defaultConfigMapMode, + LocalObjectReference: corev1.LocalObjectReference{ + Name: caBundleVolumeName, + }, + }, + }, + }) + } + + gwContainer.Args = gwArgs + p := corev1.PodSpec{ Containers: []corev1.Container{ + *gwContainer, newOPAOpenShiftContainer(sercretVolumeName, tlsDir, certFile, keyFile, withTLS), }, + Volumes: gwVolumes, } - if err := mergo.Merge(&d.Template.Spec, p, mergo.WithAppendSlice); err != nil { + if err := mergo.Merge(&d.Spec.Template.Spec, p, mergo.WithOverride); err != nil { return kverrors.Wrap(err, "failed to merge sidecar container spec ") } return nil } -// ConfigureService merges the OpenPolicyAgent sidecar metrics port into +// ConfigureGatewayService merges the OpenPolicyAgent sidecar metrics port into // the service spec. With this the metrics are exposed through the same service. -func ConfigureService(s *corev1.ServiceSpec) error { +func ConfigureGatewayService(s *corev1.ServiceSpec) error { spec := corev1.ServiceSpec{ Ports: []corev1.ServicePort{ { @@ -62,22 +125,10 @@ func ConfigureService(s *corev1.ServiceSpec) error { return nil } -// ConfigureIngress merges the OpenShift Route-specific annotations to -// the lokistack gateway ingress object. -func ConfigureIngress(i *networkingv1.Ingress) error { - ing := networkingv1.Ingress{} - - if err := mergo.Merge(i, ing); err != nil { - return kverrors.Wrap(err, "failed to merge ingress config") - } - - return nil -} - -// ConfigureServiceMonitor merges the OpenPolicyAgent sidecar endpoint into +// ConfigureGatewayServiceMonitor merges the OpenPolicyAgent sidecar endpoint into // the service monitor. With this cluster-monitoring prometheus can scrape // the sidecar metrics. -func ConfigureServiceMonitor(sm *monitoringv1.ServiceMonitor, withTLS bool) error { +func ConfigureGatewayServiceMonitor(sm *monitoringv1.ServiceMonitor, withTLS bool) error { var opaEndpoint monitoringv1.Endpoint if withTLS { diff --git a/internal/manifests/openshift/options.go b/internal/manifests/openshift/options.go index cf7531f8e9..a59395e895 100644 --- a/internal/manifests/openshift/options.go +++ b/internal/manifests/openshift/options.go @@ -36,16 +36,22 @@ type AuthorizationSpec struct { // extra lokistack gateway k8s objects (e.g. ServiceAccount, Route, RBAC) // on openshift. type BuildOptions struct { - LokiStackName string - GatewayName string - GatewayNamespace string - GatewaySvcName string - GatewaySvcTargetPort string - Labels map[string]string + LokiStackName string + GatewayName string + GatewayNamespace string + GatewaySvcName string + GatewaySvcTargetPort string + Labels map[string]string + EnableCertificateSigningService bool } // NewOptions returns an openshift options struct. -func NewOptions(stackName, gwName, gwNamespace, gwBaseDomain, gwSvcName, gwPortName string, gwLabels map[string]string) Options { +func NewOptions( + stackName string, + gwName, gwNamespace, gwBaseDomain, gwSvcName, gwPortName string, + gwLabels map[string]string, + enableCertSigningService bool, +) Options { host := ingressHost(stackName, gwNamespace, gwBaseDomain) var authn []AuthenticationSpec @@ -61,12 +67,13 @@ func NewOptions(stackName, gwName, gwNamespace, gwBaseDomain, gwSvcName, gwPortN return Options{ BuildOpts: BuildOptions{ - LokiStackName: stackName, - GatewayName: gwName, - GatewayNamespace: gwNamespace, - GatewaySvcName: gwSvcName, - GatewaySvcTargetPort: gwPortName, - Labels: gwLabels, + LokiStackName: stackName, + GatewayName: gwName, + GatewayNamespace: gwNamespace, + GatewaySvcName: gwSvcName, + GatewaySvcTargetPort: gwPortName, + Labels: gwLabels, + EnableCertificateSigningService: enableCertSigningService, }, Authentication: authn, Authorization: AuthorizationSpec{ diff --git a/internal/manifests/openshift/service_ca.go b/internal/manifests/openshift/service_ca.go new file mode 100644 index 0000000000..d637fe109d --- /dev/null +++ b/internal/manifests/openshift/service_ca.go @@ -0,0 +1,26 @@ +package openshift + +import ( + corev1 "k8s.io/api/core/v1" + metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" +) + +// BuildServiceCAConfigMap returns a k8s configmap for the LokiStack +// gateway serviceCA configmap. This configmap is used to configure +// the gateway to proxy server-side TLS encrypted requests to Loki. +func BuildServiceCAConfigMap(opts Options) *corev1.ConfigMap { + return &corev1.ConfigMap{ + TypeMeta: metav1.TypeMeta{ + Kind: "ConfigMap", + APIVersion: corev1.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Annotations: map[string]string{ + InjectCABundleKey: "true", + }, + Labels: opts.BuildOpts.Labels, + Name: serviceCABundleName(opts), + Namespace: opts.BuildOpts.GatewayNamespace, + }, + } +} diff --git a/internal/manifests/openshift/serviceaccount_test.go b/internal/manifests/openshift/serviceaccount_test.go index 50fd4483f0..9680c3ec73 100644 --- a/internal/manifests/openshift/serviceaccount_test.go +++ b/internal/manifests/openshift/serviceaccount_test.go @@ -8,7 +8,7 @@ import ( ) func TestBuildServiceAccount_AnnotationsMatchDefaultTenants(t *testing.T) { - opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}) + opts := NewOptions("abc", "abc", "efgh", "example.com", "abc", "abc", map[string]string{}, false) sa := BuildServiceAccount(opts) require.Len(t, sa.GetAnnotations(), len(defaultTenants)) diff --git a/internal/manifests/openshift/var.go b/internal/manifests/openshift/var.go index 9cf009aa4b..3bf984505f 100644 --- a/internal/manifests/openshift/var.go +++ b/internal/manifests/openshift/var.go @@ -20,9 +20,17 @@ var ( cookieSecretLength = 32 allowedRunes = []rune("abcdefghijklmnopqrstuvwxyzABCDEFGHIJKLMNOPQRSTUVWXYZ0123456789") - // ServingCertKey is the annotation key for using the cert-signing service - // on k8s service objects. + defaultConfigMapMode = int32(420) + + // ServingCertKey is the annotation key for services used the + // cert-signing service to create a new key/cert pair signed + // by the service CA stored in a secret with the same name + // as the annotated service. ServingCertKey = "service.beta.openshift.io/serving-cert-secret-name" + // InjectCABundleKey is the annotation key for configmaps used by the + // cert-signing service to inject the service CA into the annotated + // configmap. + InjectCABundleKey = "service.beta.openshift.io/inject-cabundle" ) func clusterRoleName(opts Options) string { @@ -41,6 +49,10 @@ func serviceAccountName(opts Options) string { return opts.BuildOpts.GatewayName } +func serviceCABundleName(opts Options) string { + return fmt.Sprintf("%s-ca-bundle", opts.BuildOpts.GatewayName) +} + func serviceAccountAnnotations(opts Options) map[string]string { a := make(map[string]string, len(opts.Authentication)) for _, auth := range opts.Authentication {