operator: Update developer addons (#5599)

pull/5672/head
Gerard Vanloo 3 years ago committed by GitHub
parent 27479484f8
commit 8154f3f2b4
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 8
      operator/docs/forwarding_logs_to_gateway.md
  2. 16
      operator/docs/hack_loki_operator.md
  3. 531
      operator/hack/addons_dev.yaml
  4. 222
      operator/hack/addons_ocp.yaml

@ -71,13 +71,13 @@ To configure Promtail to send application, audit, and infrastructure logs, add t
clients:
- # ...
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
url: http://lokistack-gateway-http-lokistack-dev.openshift-logging.svc:8080/api/logs/v1/audit/loki/api/v1/push
url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/audit/loki/api/v1/push
- # ...
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
url: http://lokistack-gateway-http-lokistack-dev.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push
url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push
- # ...
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
url: http://lokistack-gateway-http-lokistack-dev.openshift-logging.svc:8080/api/logs/v1/infrastructure/loki/api/v1/push
url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/infrastructure/loki/api/v1/push
```
The rest of the configuration can be configured to the developer's desire.
@ -93,7 +93,7 @@ The Fluentd configuration can be overrided to target the `application` endpoint
@type loki
# ...
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
url: http://lokistack-gateway-http-lokistack-dev.openshift-logging.svc:8080/api/logs/v1/application
url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application
</match>
```

@ -196,6 +196,22 @@ Each tenant Secret is required to match:
* `metadata.name` with `TenantsSecretsSpec.Name`.
* `metadata.namespace` with `LokiStack.metadata.namespace`.
## Development Add-Ons
To help with testing and development, a [Promtail](https://grafana.com/docs/loki/latest/clients/promtail/) and [logcli](https://grafana.com/docs/loki/latest/getting-started/logcli/) deployment are available. The example file has been configured to work with the [lokistack-gateway](./forwarding_logs_to_gateway.md). In order to work without this component, change the URLs to use the `distributor` and `query-frontend` service respectively.
In order to deploy these resources, follow the above steps to deploy the operator and instance. Then, do the following command:
```console
kubectl apply -f ./hack/addons_dev.yaml
```
### Notes
[1] When using an OpenShift cluster, the `addons_ocp.yaml` should be used. In a native K8s cluster the `addons_dev.yaml` should be used. The OpenShift environment uses `SecurityContextConstraints` in order to limit or enable pod capabilities.
[2] When deploying on a native K8s cluster, ensure that the namespaces of the `ServiceAccount` in the `ClusterRoleBinding` objects are changed accordingly.
## Basic Troubleshooting on Hacking on Loki Operator
### New changes are not detected by Loki Operator

@ -0,0 +1,531 @@
# This file is used to create additional objects to help development of the operator
# within a cluster. logcli pod helps write queries, promtail writes logs, etc
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: lokistack-dev-addons-logcli
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: lokistack-dev-addons-promtail
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: lokistack-dev-addons-logcli
spec:
selector:
matchLabels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
template:
metadata:
name: lokistack-dev-addons-logcli
labels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
spec:
containers:
- name: logcli
image: docker.io/grafana/logcli:2.4.1-amd64
imagePullPolicy: IfNotPresent
command:
- /bin/sh
env:
- name: LOKI_ORG_ID
value: application
- name: LOKI_ADDR
value: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application
- name: LOKI_BEARER_TOKEN_FILE
value: /var/run/secrets/kubernetes.io/serviceaccount/token
args:
- -c
- while true; do logcli query '{job="systemd-journal"}'; sleep 30; done
serviceAccountName: lokistack-dev-addons-logcli
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: lokistack-dev-addons-promtail
spec:
selector:
matchLabels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
template:
metadata:
name: lokistack-dev-addons-promtail
labels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
spec:
containers:
- name: promtail
image: docker.io/grafana/promtail:2.4.1
args:
- -config.file=/etc/promtail/promtail.yaml
- -log.level=info
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
- mountPath: /etc/promtail
name: config
- mountPath: /run/promtail
name: run
- mountPath: /var/lib/docker/containers
name: docker
readOnly: true
- mountPath: /var/log/pods
name: pods
readOnly: true
- mountPath: /var/log/journal
name: journal
readOnly: true
securityContext:
privileged: true
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
serviceAccountName: lokistack-dev-addons-promtail
volumes:
- configMap:
defaultMode: 420
name: lokistack-dev-addons-promtail
name: config
- hostPath:
path: /run/promtail
type: ""
name: run
- hostPath:
path: /var/lib/docker/containers
type: ""
name: docker
- hostPath:
path: /var/log/pods
type: ""
name: pods
- hostPath:
path: /var/log/journal
type: ""
name: journal
---
apiVersion: v1
kind: ConfigMap
metadata:
name: lokistack-dev-addons-promtail
data:
promtail.yaml: |
clients:
- url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push
tenant_id: application
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
backoff_config:
max_period: 5m
max_retries: 20
min_period: 1s
batchsize: 100
batchwait: 10s
timeout: 10s
positions:
filename: /run/promtail/positions.yaml
server:
http_listen_port: 3100
grpc_listen_port: 9095
target_config:
sync_period: 10s
scrape_configs:
- job_name: journal
journal:
max_age: 12h
path: /var/log/journal
labels:
job: systemd-journal
relabel_configs:
- source_labels:
- __journal__systemd_unit
target_label: unit
- source_labels:
- __journal__hostname
target_label: hostname
- job_name: kubernetes-pods-name
pipeline_stages:
- docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- source_labels:
- __meta_kubernetes_pod_label_name
target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __service__
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- job_name: kubernetes-pods-app
pipeline_stages:
- docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: drop
regex: .+
source_labels:
- __meta_kubernetes_pod_label_name
- source_labels:
- __meta_kubernetes_pod_label_app
target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __service__
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- job_name: kubernetes-pods-direct-controllers
pipeline_stages:
- docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: drop
regex: .+
separator: ''
source_labels:
- __meta_kubernetes_pod_label_name
- __meta_kubernetes_pod_label_app
- action: drop
regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
source_labels:
- __meta_kubernetes_pod_controller_name
- source_labels:
- __meta_kubernetes_pod_controller_name
target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __service__
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label:
container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- job_name: kubernetes-pods-indirect-controller
pipeline_stages:
- docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: drop
regex: .+
separator: ''
source_labels:
- __meta_kubernetes_pod_label_name
- __meta_kubernetes_pod_label_app
- action: keep
regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
source_labels:
- __meta_kubernetes_pod_controller_name
- action: replace
regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$
source_labels:
- __meta_kubernetes_pod_controller_name
target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __service__
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_uid
- __meta_kubernetes_pod_container_name
target_label: __path__
- job_name: kubernetes-pods-static
pipeline_stages:
- docker: {}
kubernetes_sd_configs:
- role: pod
relabel_configs:
- action: drop
regex: ^$
source_labels:
- __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
- action: replace
source_labels:
- __meta_kubernetes_pod_label_component
target_label: __service__
- source_labels:
- __meta_kubernetes_pod_node_name
target_label: __host__
- action: drop
regex: ^$
source_labels:
- __service__
- action: replace
replacement: $1
separator: /
source_labels:
- __meta_kubernetes_namespace
- __service__
target_label: job
- action: replace
source_labels:
- __meta_kubernetes_namespace
target_label: namespace
- action: replace
source_labels:
- __meta_kubernetes_pod_name
target_label: instance
- action: replace
source_labels:
- __meta_kubernetes_pod_container_name
target_label: container_name
- action: labelmap
regex: __meta_kubernetes_pod_label_(.+)
- replacement: /var/log/pods/*$1/*.log
separator: /
source_labels:
- __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
- __meta_kubernetes_pod_container_name
target_label: __path__
---
apiVersion: policy/v1beta1
kind: PodSecurityPolicy
metadata:
name: lokistack-dev-addons-policy
spec:
allowPrivilegeEscalation: false
fsGroup:
rule: RunAsAny
hostIPC: false
hostNetwork: false
hostPID: false
privileged: false
readOnlyRootFilesystem: true
requiredDropCapabilities:
- ALL
runAsUser:
rule: RunAsAny
seLinux:
rule: RunAsAny
supplementalGroups:
rule: RunAsAny
volumes:
- secret
- configMap
- hostPath
---
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: lokistack-dev-addons-writer
rules:
- apiGroups:
- extensions
resourceNames:
- lokistack-dev-addons-policy
resources:
- podsecuritypolicies
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: lokistack-dev-addons-writer
labels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: lokistack-dev-addons-writer
subjects:
- kind: ServiceAccount
name: lokistack-dev-addons-promtail
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lokistack-dev-addons-reader
labels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
rules:
- apiGroups:
- loki.grafana.com
resources:
- application
resourceNames:
- logs
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lokistack-dev-addons-writer
labels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
rules:
- apiGroups:
- ""
resources:
- nodes
- nodes/proxy
- services
- endpoints
- pods
verbs:
- get
- watch
- list
- apiGroups:
- loki.grafana.com
resources:
- application
resourceNames:
- logs
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: lokistack-dev-addons-reader-clusterrolebinding
labels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: lokistack-dev-addons-reader
subjects:
- kind: ServiceAccount
name: lokistack-dev-addons-logcli
namespace: openshift-logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: lokistack-dev-addons-writer-clusterrolebinding
labels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: lokistack-dev-addons-writer
subjects:
- kind: ServiceAccount
name: lokistack-dev-addons-promtail
namespace: openshift-logging

@ -2,76 +2,71 @@
# within a cluster. logcli pod helps write queries, promtail writes logs, etc
---
apiVersion: v1
kind: Pod
kind: ServiceAccount
metadata:
name: lokistack-dev-addons-logcli
---
apiVersion: v1
kind: ServiceAccount
metadata:
name: lokistack-dev-addons-promtail
---
apiVersion: apps/v1
kind: Deployment
metadata:
name: logcli
namespace: loki
labels:
app.kubernetes.io/name: logcli
name: lokistack-dev-addons-logcli
spec:
containers:
- name: logcli
image: docker.io/grafana/logcli:2.2.0-amd64
env:
- name: LOKI_ADDR
value: http://loki-querier-http-lokistack-sample.loki.svc.cluster.local:3100
command: [ "/bin/sh", "-c", "--" ]
args: [ "while true; do sleep 30; done;" ]
selector:
matchLabels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
template:
metadata:
name: lokistack-dev-addons-logcli
labels:
app.kubernetes.io/name: logcli
app.kubernetes.io/instance: developer-addons
spec:
containers:
- name: logcli
image: docker.io/grafana/logcli:2.4.1-amd64
imagePullPolicy: IfNotPresent
command:
- /bin/sh
env:
- name: LOKI_ORG_ID
value: application
- name: LOKI_ADDR
value: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application
- name: LOKI_BEARER_TOKEN_FILE
value: /var/run/secrets/kubernetes.io/serviceaccount/token
args:
- -c
- while true; do logcli query '{job="systemd-journal"}'; sleep 30; done
serviceAccountName: lokistack-dev-addons-logcli
---
apiVersion: apps/v1
kind: DaemonSet
metadata:
name: loki-promtail
namespace: loki
labels:
app.kubernetes.io/name: promtail
name: lokistack-dev-addons-promtail
spec:
selector:
matchLabels:
app.kubernetes.io/name: promtail
app.kubernetes.io/instance: developer-addons
template:
metadata:
name: lokistack-dev-addons-promtail
labels:
app.kubernetes.io/name: promtail
annotations:
prometheus.io/port: metrics
prometheus.io/scrape: "true"
app.kubernetes.io/instance: developer-addons
spec:
containers:
- args:
- name: promtail
image: docker.io/grafana/promtail:2.4.1
args:
- -config.file=/etc/promtail/promtail.yaml
- -client.url=http://loki-distributor-http-lokistack-sample.loki.svc.cluster.local:3100/api/prom/push
- -log.level=info
env:
- name: HOSTNAME
valueFrom:
fieldRef:
apiVersion: v1
fieldPath: spec.nodeName
image: docker.io/grafana/promtail:2.1.0
imagePullPolicy: IfNotPresent
name: promtail
ports:
- containerPort: 3101
name: metrics
protocol: TCP
readinessProbe:
failureThreshold: 5
httpGet:
path: /ready
port: metrics
scheme: HTTP
initialDelaySeconds: 10
periodSeconds: 10
successThreshold: 1
timeoutSeconds: 1
resources: {}
securityContext:
procMount: Default
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
terminationMessagePath: /dev/termination-log
terminationMessagePolicy: File
volumeMounts:
@ -88,14 +83,16 @@ spec:
- mountPath: /var/log/journal
name: journal
readOnly: true
serviceAccountName: loki-promtail
tolerations:
- effect: NoSchedule
key: node-role.kubernetes.io/master
securityContext:
privileged: true
readOnlyRootFilesystem: true
runAsGroup: 0
runAsUser: 0
serviceAccountName: lokistack-dev-addons-promtail
volumes:
- configMap:
defaultMode: 420
name: loki-promtail
name: lokistack-dev-addons-promtail
name: config
- hostPath:
path: /run/promtail
@ -113,30 +110,29 @@ spec:
path: /var/log/journal
type: ""
name: journal
---
apiVersion: v1
kind: ConfigMap
metadata:
name: loki-promtail
namespace: loki
labels:
app.kubernetes.io/name: promtail
name: lokistack-dev-addons-promtail
data:
promtail.yaml: |
client:
backoff_config:
min_period: 100ms
max_period: 5s
max_retries: 5
batchsize: 102400
batchwait: 1s
external_labels: {}
timeout: 10s
clients:
- url: http://lokistack-dev-gateway-http.openshift-logging.svc:8080/api/logs/v1/application/loki/api/v1/push
tenant_id: application
bearer_token_file: /var/run/secrets/kubernetes.io/serviceaccount/token
backoff_config:
max_period: 5m
max_retries: 20
min_period: 1s
batchsize: 100
batchwait: 10s
timeout: 10s
positions:
filename: /run/promtail/positions.yaml
server:
http_listen_port: 3101
http_listen_port: 3100
grpc_listen_port: 9095
target_config:
sync_period: 10s
scrape_configs:
@ -400,23 +396,51 @@ data:
- __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror
- __meta_kubernetes_pod_container_name
target_label: __path__
---
apiVersion: v1
kind: ServiceAccount
apiVersion: rbac.authorization.k8s.io/v1
kind: Role
metadata:
name: lokistack-dev-addons-writer
rules:
- apiGroups:
- security.openshift.io
resourceNames:
- privileged
resources:
- securitycontextconstraints
verbs:
- use
---
apiVersion: rbac.authorization.k8s.io/v1
kind: RoleBinding
metadata:
name: loki-promtail
namespace: loki
labels:
app.kubernetes.io/name: promtail
name: lokistack-dev-addons-writer
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: Role
name: lokistack-dev-addons-writer
subjects:
- kind: ServiceAccount
name: lokistack-dev-addons-promtail
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: loki-promtail-clusterrole
labels:
app.kubernetes.io/name: promtail
name: lokistack-dev-addons-reader
rules:
- apiGroups:
- loki.grafana.com
resources:
- application
resourceNames:
- logs
verbs:
- get
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRole
metadata:
name: lokistack-dev-addons-writer
rules:
- apiGroups:
- ""
@ -430,19 +454,37 @@ rules:
- get
- watch
- list
- apiGroups:
- loki.grafana.com
resources:
- application
resourceNames:
- logs
verbs:
- create
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: lokistack-dev-addons-reader-clusterrolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: lokistack-dev-addons-reader
subjects:
- kind: ServiceAccount
name: lokistack-dev-addons-logcli
namespace: openshift-logging
---
apiVersion: rbac.authorization.k8s.io/v1
kind: ClusterRoleBinding
metadata:
name: loki-promtail-clusterrolebinding
labels:
app.kubernetes.io/name: promtail
name: lokistack-dev-addons-writer-clusterrolebinding
roleRef:
apiGroup: rbac.authorization.k8s.io
kind: ClusterRole
name: loki-promtail-clusterrole
name: lokistack-dev-addons-writer
subjects:
- kind: ServiceAccount
name: loki-promtail
namespace: loki
name: lokistack-dev-addons-promtail
namespace: openshift-logging
Loading…
Cancel
Save