operator: Making all pods and containers compliant with restricted policy (#6514)

pull/6562/head
Gerard Vanloo 4 years ago committed by GitHub
parent a1745c7b47
commit 92cd8165af
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      operator/CHANGELOG.md
  2. 4
      operator/Makefile
  3. 1
      operator/apis/config/v1/projectconfig_types.go
  4. 1
      operator/bundle/manifests/loki-operator-manager-config_v1_configmap.yaml
  5. 10
      operator/bundle/manifests/loki-operator.clusterserviceversion.yaml
  6. 2
      operator/cmd/loki-broker/main.go
  7. 5
      operator/config/manager/manager.yaml
  8. 1
      operator/config/overlays/development/controller_manager_config.yaml
  9. 1
      operator/config/overlays/development/kustomization.yaml
  10. 10
      operator/config/overlays/development/manager_security_context_patch.yaml
  11. 1
      operator/config/overlays/openshift/controller_manager_config.yaml
  12. 7
      operator/config/overlays/openshift/manager_auth_proxy_patch.yaml
  13. 1
      operator/config/overlays/production/controller_manager_config.yaml
  14. 7
      operator/config/overlays/production/manager_auth_proxy_patch.yaml
  15. 87
      operator/hack/addons_dev.yaml
  16. 59
      operator/hack/addons_ocp.yaml
  17. 133
      operator/internal/manifests/build_test.go
  18. 2
      operator/internal/manifests/compactor.go
  19. 2
      operator/internal/manifests/distributor.go
  20. 13
      operator/internal/manifests/gateway.go
  21. 2
      operator/internal/manifests/indexgateway.go
  22. 2
      operator/internal/manifests/ingester.go
  23. 2
      operator/internal/manifests/options.go
  24. 2
      operator/internal/manifests/querier.go
  25. 2
      operator/internal/manifests/query-frontend.go
  26. 2
      operator/internal/manifests/ruler.go
  27. 24
      operator/internal/manifests/var.go
  28. 1
      operator/main.go

@ -1,5 +1,6 @@
## Main
- [6514](https://github.com/grafana/loki/pull/6514) **Red-GV** Update all pods and containers to be compliant with restricted Pod Security Standard
- [6531](https://github.com/grafana/loki/pull/6531) **periklis**: Use default interface_names for lokistack clusters (IPv6 Support)
- [6411](https://github.com/grafana/loki/pull/6478) **aminesnow**: Support TLS enabled lokistack-gateway for vanilla kubernetes deployments
- [6504](https://github.com/grafana/loki/pull/6504) **periklis**: Disable usage report on OpenShift

@ -194,6 +194,10 @@ quickstart: oci-build oci-push $(KIND)
@./quickstart.sh $(filter-out $@,$(MAKECMDGOALS))
endif
.PHONY: quickstart-cleanup
quickstart-cleanup: $(KIND) ## Cleanup for quickstart set up
$(KIND) delete cluster
.PHONY: run
run: generate manifests ## Run against the configured Kubernetes cluster in ~/.kube/config
go run ./main.go

@ -19,6 +19,7 @@ type FeatureFlags struct {
EnableLokiStackWebhook bool `json:"enableLokiStackWebhook,omitempty"`
EnableAlertingRuleWebhook bool `json:"enableAlertingRuleWebhook,omitempty"`
EnableRecordingRuleWebhook bool `json:"enableRecordingRuleWebhook,omitempty"`
EnableRuntimeSeccompProfile bool `json:"enableRuntimeSeccompProfile,omitempty"`
}
//+kubebuilder:object:root=true

@ -27,6 +27,7 @@ data:
enableLokiStackWebhook: true
enableAlertingRuleWebhook: true
enableRecordingRuleWebhook: true
enableRuntimeSeccompProfile: false
kind: ConfigMap
metadata:
labels:

@ -1210,6 +1210,9 @@ spec:
resources: {}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- mountPath: /tmp/k8s-webhook-server/serving-certs
name: webhook-cert
@ -1230,11 +1233,18 @@ spec:
- containerPort: 8443
name: https
resources: {}
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumeMounts:
- mountPath: /var/run/secrets/serving-cert
name: loki-operator-metrics-cert
nodeSelector:
kubernetes.io/os: linux
securityContext:
runAsNonRoot: true
terminationGracePeriodSeconds: 10
volumes:
- name: webhook-cert

@ -42,6 +42,8 @@ func (c *config) registerFlags(f *flag.FlagSet) {
f.BoolVar(&c.featureFlags.EnableTLSServiceMonitorConfig, "with-tls-service-monitors", false, "Enable TLS endpoint for service monitors.")
f.BoolVar(&c.featureFlags.EnablePrometheusAlerts, "with-prometheus-alerts", false, "Enables prometheus alerts")
f.BoolVar(&c.featureFlags.EnableGateway, "with-lokistack-gateway", false, "Enables the manifest creation for the entire lokistack-gateway.")
f.BoolVar(&c.featureFlags.EnableRuntimeSeccompProfile, "with-runtime-seccomp-profile", false, "Enables the usage of the runtime/default seccomp profile for pods and containers.")
// Object storage options
c.objectStorage = storage.Options{
S3: &storage.S3StorageConfig{},

@ -27,6 +27,9 @@ spec:
name: metrics
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
livenessProbe:
httpGet:
path: /healthz
@ -42,3 +45,5 @@ spec:
nodeSelector:
kubernetes.io/os: linux
terminationGracePeriodSeconds: 10
securityContext:
runAsNonRoot: true

@ -15,3 +15,4 @@ featureFlags:
enableLokiStackWebhook: false
enableAlertingRuleWebhook: false
enableRecordingRuleWebhook: false
enableRuntimeSeccompProfile: true

@ -29,3 +29,4 @@ patchesStrategicMerge:
- manager_run_flags_patch.yaml
- manager_related_image_patch.yaml
- manager_image_pull_policy_patch.yaml
- manager_security_context_patch.yaml

@ -0,0 +1,10 @@
apiVersion: apps/v1
kind: Deployment
metadata:
name: controller-manager
spec:
template:
spec:
securityContext:
seccompProfile:
type: RuntimeDefault

@ -24,3 +24,4 @@ featureFlags:
enableLokiStackWebhook: true
enableAlertingRuleWebhook: true
enableRecordingRuleWebhook: true
enableRuntimeSeccompProfile: false

@ -21,9 +21,16 @@ spec:
volumeMounts:
- mountPath: /var/run/secrets/serving-cert
name: loki-operator-metrics-cert
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumes:
- name: loki-operator-metrics-cert
secret:
defaultMode: 420
optional: true
secretName: loki-operator-metrics
securityContext:
runAsNonRoot: true

@ -11,3 +11,4 @@ leaderElection:
resourceName: e3716011.grafana.com
featureFlags:
enableLokiStackGateway: true
enableRuntimeSeccompProfile: false

@ -23,9 +23,16 @@ spec:
volumeMounts:
- mountPath: /var/run/secrets/serving-cert
name: loki-operator-metrics-cert
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
volumes:
- name: loki-operator-metrics-cert
secret:
defaultMode: 420
optional: true
secretName: loki-operator-metrics
securityContext:
runAsNonRoot: true

@ -29,7 +29,7 @@ spec:
spec:
containers:
- name: logcli
image: docker.io/grafana/logcli:2.4.2-amd64
image: docker.io/grafana/logcli:2.5.0-amd64
imagePullPolicy: IfNotPresent
command:
- /bin/sh
@ -43,7 +43,16 @@ spec:
args:
- -c
- while true; do logcli query '{job="systemd-journal"}'; sleep 30; done
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
serviceAccountName: lokistack-dev-addons-logcli
securityContext:
runAsNonRoot: true
seccompProfile:
type: RuntimeDefault
---
apiVersion: apps/v1
kind: DaemonSet
@ -63,7 +72,7 @@ spec:
spec:
containers:
- name: promtail
image: docker.io/grafana/promtail:2.4.2
image: docker.io/grafana/promtail:2.5.0
args:
- -config.file=/etc/promtail/promtail.yaml
- -log.level=info
@ -84,31 +93,33 @@ spec:
name: journal
readOnly: true
securityContext:
privileged: true
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
serviceAccountName: lokistack-dev-addons-promtail
securityContext:
runAsNonRoot: true
volumes:
- configMap:
defaultMode: 420
name: lokistack-dev-addons-promtail
name: config
- hostPath:
path: /run/promtail
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: run
- hostPath:
path: /var/lib/docker/containers
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: docker
- hostPath:
path: /var/log/pods
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: pods
- hostPath:
path: /var/log/journal
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: journal
---
apiVersion: v1
@ -397,46 +408,6 @@ data:
- __meta_kubernetes_pod_container_name
target_label: __path__
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: lokistack-dev-addons-policy
spec:
allowPrivilegeEscalation: false
fsGroup:
rule: RunAsAny
hostIPC: false
hostNetwork: false
hostPID: false
privileged: false
readOnlyRootFilesystem: true
requiredDropCapabilities:
- ALL
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
- configMap
- hostPath
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: lokistack-dev-addons-writer
rules:
- apiGroups:
- extensions
resourceNames:
- lokistack-dev-addons-policy
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:

@ -29,7 +29,7 @@ spec:
spec:
containers:
- name: logcli
image: docker.io/grafana/logcli:2.4.2-amd64
image: docker.io/grafana/logcli:2.5.0-amd64
imagePullPolicy: IfNotPresent
command:
- /bin/sh
@ -43,7 +43,14 @@ spec:
args:
- -c
- while true; do logcli --ca-cert="/var/run/secrets/kubernetes.io/serviceaccount/service-ca.crt" query '{job="systemd-journal"}'; sleep 30; done
securityContext:
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
serviceAccountName: lokistack-dev-addons-logcli
securityContext:
runAsNonRoot: true
---
apiVersion: apps/v1
kind: DaemonSet
@ -63,7 +70,7 @@ spec:
spec:
containers:
- name: promtail
image: docker.io/grafana/promtail:2.4.2
image: docker.io/grafana/promtail:2.5.0
args:
- -config.file=/etc/promtail/promtail.yaml
- -log.level=info
@ -84,31 +91,33 @@ spec:
name: journal
readOnly: true
securityContext:
privileged: true
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
allowPrivilegeEscalation: false
capabilities:
drop:
- ALL
serviceAccountName: lokistack-dev-addons-promtail
securityContext:
runAsNonRoot: true
volumes:
- configMap:
defaultMode: 420
name: lokistack-dev-addons-promtail
name: config
- hostPath:
path: /run/promtail
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: run
- hostPath:
path: /var/lib/docker/containers
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: docker
- hostPath:
path: /var/log/pods
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: pods
- hostPath:
path: /var/log/journal
type: ""
- emptyDir:
medium: ""
sizeLimit: 10G
name: journal
---
apiVersion: v1
@ -400,20 +409,6 @@ data:
target_label: __path__
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: lokistack-dev-addons-writer
rules:
- apiGroups:
- security.openshift.io
resourceNames:
- privileged
resources:
- securitycontextconstraints
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: lokistack-dev-addons-writer

@ -539,6 +539,139 @@ func TestBuildAll_WithFeatureFlags_EnableTLSGRPCServices(t *testing.T) {
}
}
func TestBuildAll_WithFeatureFlags_EnableRuntimeSeccompProfile(t *testing.T) {
type test struct {
desc string
BuildOptions Options
}
table := []test{
{
desc: "disabled default/runtime seccomp profile",
BuildOptions: Options{
Name: "test",
Namespace: "test",
Stack: lokiv1beta1.LokiStackSpec{
Size: lokiv1beta1.SizeOneXSmall,
Rules: &lokiv1beta1.RulesSpec{
Enabled: true,
},
Template: &lokiv1beta1.LokiTemplateSpec{
Compactor: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Distributor: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Ingester: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Querier: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
QueryFrontend: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Gateway: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
IndexGateway: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Ruler: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
},
},
Flags: FeatureFlags{
EnableRuntimeSeccompProfile: false,
},
},
},
{
desc: "enabled default/runtime seccomp profile",
BuildOptions: Options{
Name: "test",
Namespace: "test",
Stack: lokiv1beta1.LokiStackSpec{
Size: lokiv1beta1.SizeOneXSmall,
Rules: &lokiv1beta1.RulesSpec{
Enabled: true,
},
Template: &lokiv1beta1.LokiTemplateSpec{
Compactor: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Distributor: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Ingester: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Querier: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
QueryFrontend: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Gateway: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
IndexGateway: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
Ruler: &lokiv1beta1.LokiComponentSpec{
Replicas: 1,
},
},
},
Flags: FeatureFlags{
EnableRuntimeSeccompProfile: true,
},
},
},
}
for _, tst := range table {
tst := tst
t.Run(tst.desc, func(t *testing.T) {
t.Parallel()
err := ApplyDefaultSettings(&tst.BuildOptions)
require.NoError(t, err)
objs, err := BuildAll(tst.BuildOptions)
require.NoError(t, err)
for _, o := range objs {
var (
name string
spec *corev1.PodSpec
)
switch obj := o.(type) {
case *appsv1.Deployment:
name = obj.Name
spec = &obj.Spec.Template.Spec
case *appsv1.StatefulSet:
name = obj.Name
spec = &obj.Spec.Template.Spec
default:
continue
}
t.Run(name, func(t *testing.T) {
if tst.BuildOptions.Flags.EnableRuntimeSeccompProfile {
require.NotNil(t, spec.SecurityContext.SeccompProfile)
require.Equal(t, spec.SecurityContext.SeccompProfile.Type, corev1.SeccompProfileTypeRuntimeDefault)
} else {
require.Nil(t, spec.SecurityContext.SeccompProfile)
}
})
}
})
}
}
func TestBuildAll_WithFeatureFlags_EnableGateway(t *testing.T) {
type test struct {
desc string

@ -101,8 +101,10 @@ func NewCompactorStatefulSet(opts Options) *appsv1.StatefulSet {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.Compactor != nil {

@ -96,8 +96,10 @@ func NewDistributorDeployment(opts Options) *appsv1.Deployment {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.Distributor != nil {

@ -60,6 +60,8 @@ func BuildGateway(opts Options) ([]client.Object, error) {
objs = configureGatewayObjsForMode(objs, opts)
}
configureDeploymentForRestrictedPolicy(dpl, opts.Flags)
return objs, nil
}
@ -407,3 +409,14 @@ func configureGatewayMetricsPKI(podSpec *corev1.PodSpec, serviceName string) err
return nil
}
func configureDeploymentForRestrictedPolicy(d *appsv1.Deployment, flags FeatureFlags) {
podSpec := d.Spec.Template.Spec
podSpec.SecurityContext = podSecurityContext(flags.EnableRuntimeSeccompProfile)
for i := range podSpec.Containers {
podSpec.Containers[i].SecurityContext = containerSecurityContext()
}
d.Spec.Template.Spec = podSpec
}

@ -100,8 +100,10 @@ func NewIndexGatewayStatefulSet(opts Options) *appsv1.StatefulSet {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.IndexGateway != nil {

@ -114,8 +114,10 @@ func NewIngesterStatefulSet(opts Options) *appsv1.StatefulSet {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.Ingester != nil {

@ -44,6 +44,8 @@ type FeatureFlags struct {
EnableGateway bool
EnableGatewayRoute bool
EnableGrafanaLabsStats bool
EnableRuntimeSeccompProfile bool
EnableNonRootUser bool
}
// Tenants contains the configuration per tenant and secrets for authn/authz.

@ -102,8 +102,10 @@ func NewQuerierDeployment(opts Options) *appsv1.Deployment {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.Querier != nil {

@ -106,8 +106,10 @@ func NewQueryFrontendDeployment(opts Options) *appsv1.Deployment {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.QueryFrontend != nil {

@ -124,8 +124,10 @@ func NewRulerStatefulSet(opts Options) *appsv1.StatefulSet {
TerminationMessagePath: "/dev/termination-log",
TerminationMessagePolicy: "File",
ImagePullPolicy: "IfNotPresent",
SecurityContext: containerSecurityContext(),
},
},
SecurityContext: podSecurityContext(opts.Flags.EnableRuntimeSeccompProfile),
}
if opts.Stack.Template != nil && opts.Stack.Template.Ruler != nil {

@ -9,6 +9,7 @@ import (
corev1 "k8s.io/api/core/v1"
"k8s.io/apimachinery/pkg/labels"
"k8s.io/apimachinery/pkg/util/intstr"
"k8s.io/utils/pointer"
)
const (
@ -328,3 +329,26 @@ func lokiReadinessProbe() *corev1.Probe {
FailureThreshold: 3,
}
}
func containerSecurityContext() *corev1.SecurityContext {
return &corev1.SecurityContext{
AllowPrivilegeEscalation: pointer.Bool(false),
Capabilities: &corev1.Capabilities{
Drop: []corev1.Capability{"ALL"},
},
}
}
func podSecurityContext(withSeccompProfile bool) *corev1.PodSecurityContext {
context := corev1.PodSecurityContext{
RunAsNonRoot: pointer.Bool(true),
}
if withSeccompProfile {
context.SeccompProfile = &corev1.SeccompProfile{
Type: corev1.SeccompProfileTypeRuntimeDefault,
}
}
return &context
}

@ -104,6 +104,7 @@ func main() {
EnableGateway: ctrlCfg.Flags.EnableGateway,
EnableGatewayRoute: ctrlCfg.Flags.EnableGatewayRoute,
EnableGrafanaLabsStats: ctrlCfg.Flags.EnableGrafanaLabsStats,
EnableRuntimeSeccompProfile: ctrlCfg.Flags.EnableRuntimeSeccompProfile,
}
if err = (&lokictrl.LokiStackReconciler{

Loading…
Cancel
Save