From bc62e4b3411f5d1c29d27ba566bf4fe31af9e247 Mon Sep 17 00:00:00 2001 From: Brett Jones Date: Thu, 18 Mar 2021 11:13:49 -0500 Subject: [PATCH] add hack/addons.yaml for promtail and logcli --- config/manager/kustomization.yaml | 2 +- hack/addons.yaml | 448 +++++++++++++++++++++ internal/manifests/build.go | 1 - internal/manifests/config/loki-config.yaml | 10 +- internal/manifests/config/options.go | 21 +- internal/manifests/querier.go | 52 ++- internal/manifests/query-frontend.go | 4 +- internal/manifests/shared.go | 14 +- internal/manifests/var.go | 8 + 9 files changed, 534 insertions(+), 26 deletions(-) create mode 100644 hack/addons.yaml diff --git a/config/manager/kustomization.yaml b/config/manager/kustomization.yaml index f8e09d3db5..1aa30aa666 100644 --- a/config/manager/kustomization.yaml +++ b/config/manager/kustomization.yaml @@ -13,4 +13,4 @@ kind: Kustomization images: - name: controller newName: quay.io/blockloop/loki-operator - newTag: "1616013530" + newTag: "1616076328" diff --git a/hack/addons.yaml b/hack/addons.yaml new file mode 100644 index 0000000000..27853b6570 --- /dev/null +++ b/hack/addons.yaml @@ -0,0 +1,448 @@ +# This file is used to create additional objects to help development of the operator +# within a cluster. logcli pod helps write queries, promtail writes logs, etc +--- +apiVersion: v1 +kind: Pod +metadata: + name: logcli + namespace: loki + labels: + app.kubernetes.io/name: logcli +spec: + containers: + - name: logcli + image: docker.io/grafana/logcli:2.2.0-amd64 + env: + - name: LOKI_ADDR + value: http://loki-querier-http-lokistack-sample.loki.svc.cluster.local:3100 + command: [ "/bin/sh", "-c", "--" ] + args: [ "while true; do sleep 30; done;" ] + +--- +apiVersion: apps/v1 +kind: DaemonSet +metadata: + name: loki-promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail +spec: + selector: + matchLabels: + app.kubernetes.io/name: promtail + template: + metadata: + labels: + app.kubernetes.io/name: promtail + annotations: + prometheus.io/port: metrics + prometheus.io/scrape: "true" + spec: + containers: + - args: + - -config.file=/etc/promtail/promtail.yaml + - -client.url=http://loki-distributor-http-lokistack-sample.loki.svc.cluster.local:3100/api/prom/push + - -log.level=info + env: + - name: HOSTNAME + valueFrom: + fieldRef: + apiVersion: v1 + fieldPath: spec.nodeName + image: docker.io/grafana/promtail:2.1.0 + imagePullPolicy: IfNotPresent + name: promtail + ports: + - containerPort: 3101 + name: metrics + protocol: TCP + readinessProbe: + failureThreshold: 5 + httpGet: + path: /ready + port: metrics + scheme: HTTP + initialDelaySeconds: 10 + periodSeconds: 10 + successThreshold: 1 + timeoutSeconds: 1 + resources: {} + securityContext: + procMount: Default + readOnlyRootFilesystem: true + runAsGroup: 0 + runAsUser: 0 + terminationMessagePath: /dev/termination-log + terminationMessagePolicy: File + volumeMounts: + - mountPath: /etc/promtail + name: config + - mountPath: /run/promtail + name: run + - mountPath: /var/lib/docker/containers + name: docker + readOnly: true + - mountPath: /var/log/pods + name: pods + readOnly: true + - mountPath: /var/log/journal + name: journal + readOnly: true + serviceAccountName: loki-promtail + tolerations: + - effect: NoSchedule + key: node-role.kubernetes.io/master + volumes: + - configMap: + defaultMode: 420 + name: loki-promtail + name: config + - hostPath: + path: /run/promtail + type: "" + name: run + - hostPath: + path: /var/lib/docker/containers + type: "" + name: docker + - hostPath: + path: /var/log/pods + type: "" + name: pods + - hostPath: + path: /var/log/journal + type: "" + name: journal + +--- +apiVersion: v1 +kind: ConfigMap +metadata: + name: loki-promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail +data: + promtail.yaml: | + client: + backoff_config: + min_period: 100ms + max_period: 5s + max_retries: 5 + batchsize: 102400 + batchwait: 1s + external_labels: {} + timeout: 10s + positions: + filename: /run/promtail/positions.yaml + server: + http_listen_port: 3101 + target_config: + sync_period: 10s + scrape_configs: + - job_name: journal + journal: + max_age: 12h + path: /var/log/journal + labels: + job: systemd-journal + relabel_configs: + - source_labels: + - __journal__systemd_unit + target_label: unit + - source_labels: + - __journal__hostname + target_label: hostname + - job_name: kubernetes-pods-name + pipeline_stages: + - docker: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - source_labels: + - __meta_kubernetes_pod_label_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-app + pipeline_stages: + - docker: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + source_labels: + - __meta_kubernetes_pod_label_name + - source_labels: + - __meta_kubernetes_pod_label_app + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-direct-controllers + pipeline_stages: + - docker: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: drop + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: + container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-indirect-controller + pipeline_stages: + - docker: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: .+ + separator: '' + source_labels: + - __meta_kubernetes_pod_label_name + - __meta_kubernetes_pod_label_app + - action: keep + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + - action: replace + regex: ^([0-9a-z-.]+)(-[0-9a-f]{8,10})$ + source_labels: + - __meta_kubernetes_pod_controller_name + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_uid + - __meta_kubernetes_pod_container_name + target_label: __path__ + - job_name: kubernetes-pods-static + pipeline_stages: + - docker: {} + kubernetes_sd_configs: + - role: pod + relabel_configs: + - action: drop + regex: ^$ + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - action: replace + source_labels: + - __meta_kubernetes_pod_label_component + target_label: __service__ + - source_labels: + - __meta_kubernetes_pod_node_name + target_label: __host__ + - action: drop + regex: ^$ + source_labels: + - __service__ + - action: replace + replacement: $1 + separator: / + source_labels: + - __meta_kubernetes_namespace + - __service__ + target_label: job + - action: replace + source_labels: + - __meta_kubernetes_namespace + target_label: namespace + - action: replace + source_labels: + - __meta_kubernetes_pod_name + target_label: instance + - action: replace + source_labels: + - __meta_kubernetes_pod_container_name + target_label: container_name + - action: labelmap + regex: __meta_kubernetes_pod_label_(.+) + - replacement: /var/log/pods/*$1/*.log + separator: / + source_labels: + - __meta_kubernetes_pod_annotation_kubernetes_io_config_mirror + - __meta_kubernetes_pod_container_name + target_label: __path__ + +--- +apiVersion: v1 +kind: ServiceAccount +metadata: + name: loki-promtail + namespace: loki + labels: + app.kubernetes.io/name: promtail + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRole +metadata: + name: loki-promtail-clusterrole + labels: + app.kubernetes.io/name: promtail +rules: +- apiGroups: + - "" + resources: + - nodes + - nodes/proxy + - services + - endpoints + - pods + verbs: + - get + - watch + - list + +--- +apiVersion: rbac.authorization.k8s.io/v1 +kind: ClusterRoleBinding +metadata: + name: loki-promtail-clusterrolebinding + labels: + app.kubernetes.io/name: promtail +roleRef: + apiGroup: rbac.authorization.k8s.io + kind: ClusterRole + name: loki-promtail-clusterrole +subjects: +- kind: ServiceAccount + name: loki-promtail + namespace: loki diff --git a/internal/manifests/build.go b/internal/manifests/build.go index 79cdb22b6c..899839e173 100644 --- a/internal/manifests/build.go +++ b/internal/manifests/build.go @@ -14,7 +14,6 @@ func BuildAll(stackName, namespace string) ([]client.Object, error) { return nil, err } res = append(res, cm) - res = append(res, BuildDistributor(stackName)...) res = append(res, BuildIngester(stackName)...) res = append(res, BuildQuerier(stackName)...) diff --git a/internal/manifests/config/loki-config.yaml b/internal/manifests/config/loki-config.yaml index 975706775d..630e729ba7 100644 --- a/internal/manifests/config/loki-config.yaml +++ b/internal/manifests/config/loki-config.yaml @@ -8,12 +8,14 @@ distributor: kvstore: store: memberlist frontend: + downstream_url: {{ .Querier.FQDN }}:{{ .Querier.Port }} compress_responses: true max_outstanding_per_tenant: 200 + log_queries_longer_than: 5s frontend_worker: - frontend_address: {{ .FrontendWorker.FQDN }}:{{ .FrontendWorker.Port }} - grpc_client_config: - max_send_msg_size: 104857600 +# frontend_address: {{ .FrontendWorker.FQDN }}:{{ .FrontendWorker.Port }} +# grpc_client_config: +# max_send_msg_size: 104857600 parallelism: 32 ingester: chunk_block_size: 262144 @@ -87,7 +89,7 @@ server: http_listen_port: 3100 http_server_idle_timeout: 120s http_server_write_timeout: 1m - log_level: debug + log_level: info storage_config: boltdb: directory: {{ .StorageDirectory }}/index diff --git a/internal/manifests/config/options.go b/internal/manifests/config/options.go index bf0645ad9b..ee9a136c03 100644 --- a/internal/manifests/config/options.go +++ b/internal/manifests/config/options.go @@ -3,9 +3,11 @@ package config // lokiConfigOptions is used to render the loki-config.yaml file template type Options struct { // FrontendWorker is required - FrontendWorker FrontendWorker + FrontendWorker Address // GossipRing is required - GossipRing GossipRing + GossipRing Address + // Querier is required + Querier Address // Storage is required StorageDirectory string @@ -13,18 +15,9 @@ type Options struct { Namespace string } -type FrontendWorker struct { - // FQDN is the required name of the service or fqdn WITHOUT the port +type Address struct { + // FQDN is required FQDN string - - // Port is the required service port - Port int -} - -type GossipRing struct { - // FQDN is the required name of the gossip ring service or fqdn WITHOUT the port - FQDN string - - // Port is the required gossip ring service port + // Port is required Port int } diff --git a/internal/manifests/querier.go b/internal/manifests/querier.go index 495008a4ef..2e2a5f10f1 100644 --- a/internal/manifests/querier.go +++ b/internal/manifests/querier.go @@ -18,6 +18,8 @@ import ( func BuildQuerier(stackName string) []client.Object { return []client.Object{ NewQuerierDeployment(stackName), + NewQuerierGRPCService(stackName), + NewQuerierHTTPService(stackName), } } @@ -141,4 +143,52 @@ func NewQuerierDeployment(stackName string) *apps.Deployment { }, }, } -} \ No newline at end of file +} + +func NewQuerierGRPCService(stackName string) *core.Service { + l := ComponentLabels("querier", stackName) + return &core.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: apps.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: serviceNameQuerierGRPC(stackName), + Labels: l, + }, + Spec: core.ServiceSpec{ + ClusterIP: "None", + Ports: []core.ServicePort{ + { + Name: "grpc", + Port: grpcPort, + }, + }, + Selector: l, + }, + } +} + +func NewQuerierHTTPService(stackName string) *core.Service { + l := ComponentLabels("querier", stackName) + return &core.Service{ + TypeMeta: metav1.TypeMeta{ + Kind: "Service", + APIVersion: apps.SchemeGroupVersion.String(), + }, + ObjectMeta: metav1.ObjectMeta{ + Name: serviceNameQuerierHTTP(stackName), + Labels: l, + }, + Spec: core.ServiceSpec{ + ClusterIP: "None", + Ports: []core.ServicePort{ + { + Name: "http", + Port: httpPort, + }, + }, + Selector: l, + }, + } +} diff --git a/internal/manifests/query-frontend.go b/internal/manifests/query-frontend.go index 900500bee3..fc7193ba89 100644 --- a/internal/manifests/query-frontend.go +++ b/internal/manifests/query-frontend.go @@ -180,8 +180,8 @@ func NewQueryFrontendHTTPService(stackName string) *core.Service { ClusterIP: "None", Ports: []core.ServicePort{ { - Name: "grpc", - Port: grpcPort, + Name: "http", + Port: httpPort, }, }, Selector: l, diff --git a/internal/manifests/shared.go b/internal/manifests/shared.go index 95e5a903d2..51289cf583 100644 --- a/internal/manifests/shared.go +++ b/internal/manifests/shared.go @@ -12,12 +12,20 @@ import ( func LokiConfigMap(stackName, namespace string) (*core.ConfigMap, error) { b, err := config.Build(config.Options{ - Namespace: namespace, - StorageDirectory: strings.TrimRight(dataDirectory, "/"), - GossipRing: config.GossipRing{ + FrontendWorker: config.Address{ + FQDN: "", + Port: 0, + }, + GossipRing: config.Address{ FQDN: fqdn(LokiGossipRingService(stackName).GetName(), namespace), Port: gossipPort, }, + Querier: config.Address{ + FQDN: serviceNameQuerierHTTP(stackName), + Port: httpPort, + }, + StorageDirectory: strings.TrimRight(dataDirectory, "/"), + Namespace: namespace, }) if err != nil { return nil, err diff --git a/internal/manifests/var.go b/internal/manifests/var.go index 1f71684e32..bf534a83ba 100644 --- a/internal/manifests/var.go +++ b/internal/manifests/var.go @@ -33,6 +33,14 @@ func GossipLabels() map[string]string { } } +func serviceNameQuerierHTTP(stackName string) string { + return fmt.Sprintf("loki-querier-http-%s", stackName) +} + +func serviceNameQuerierGRPC(stackName string) string { + return fmt.Sprintf("loki-querier-grpc-%s", stackName) +} + func serviceNameGossipRing(stackName string) string { return fmt.Sprintf("loki-gossip-ring-%s", stackName) }