Helm: Allow users to define remote write for metrics. (#7399)

production/helm/loki/README.md

@@ -308,6 +308,7 @@ monitoring:
 | monitoring.selfMonitoring.grafanaAgent.labels | object | `{}` | Additional Grafana Agent labels |
 | monitoring.selfMonitoring.grafanaAgent.namespace | string | `nil` | Alternative namespace for Grafana Agent resources |
 | monitoring.selfMonitoring.logsInstance.annotations | object | `{}` | LogsInstance annotations |
+| monitoring.selfMonitoring.logsInstance.clients | string | `nil` | Additional clients for remote write |
 | monitoring.selfMonitoring.logsInstance.labels | object | `{}` | Additional LogsInstance labels |
 | monitoring.selfMonitoring.logsInstance.namespace | string | `nil` | Alternative namespace for LogsInstance resources |
 | monitoring.selfMonitoring.lokiCanary.annotations | object | `{}` | Additional annotations for the `loki-canary` Daemonset |
@@ -332,6 +333,10 @@ monitoring:
 | monitoring.serviceMonitor.enabled | bool | `true` | If enabled, ServiceMonitor resources for Prometheus Operator are created |
 | monitoring.serviceMonitor.interval | string | `nil` | ServiceMonitor scrape interval |
 | monitoring.serviceMonitor.labels | object | `{}` | Additional ServiceMonitor labels |
+| monitoring.serviceMonitor.metricsInstance | object | `{"annotations":{},"labels":{},"remoteWrite":null}` | If defined, will create a MetricsInstance for the Grafana Agent Operator. |
+| monitoring.serviceMonitor.metricsInstance.annotations | object | `{}` | MerticsInstance annotations |
+| monitoring.serviceMonitor.metricsInstance.labels | object | `{}` | Additional MatricsInstance labels |
+| monitoring.serviceMonitor.metricsInstance.remoteWrite | string | `nil` | If defined a MetricsInstance will be created to remote write metrics. |
 | monitoring.serviceMonitor.namespace | string | `nil` | Alternative namespace for ServiceMonitor resources |
 | monitoring.serviceMonitor.namespaceSelector | object | `{}` | Namespace selector for ServiceMonitor resources |
 | monitoring.serviceMonitor.relabelings | list | `[]` | ServiceMonitor relabel configs to apply to samples before scraping https://github.com/prometheus-operator/prometheus-operator/blob/master/Documentation/api.md#relabelconfig |

production/helm/loki/templates/monitoring/_helpers-monitoring.tpl

@@ -8,7 +8,7 @@ Client definition for LogsInstance
   {{- $url = printf "http://%s.%s.svc.%s:3100/loki/api/v1/push" (include "loki.singleBinaryFullname" .) .Release.Namespace .Values.global.clusterDomain }}
 {{- else if .Values.gateway.enabled -}}
   {{- $url = printf "http://%s.%s.svc.%s/loki/api/v1/push" (include "loki.gatewayFullname" .) .Release.Namespace .Values.global.clusterDomain }}
-{{- end }}
+{{- end -}}
 - url: {{ $url }}
   externalLabels:
     cluster: {{ include "loki.fullname" . }}

production/helm/loki/templates/monitoring/grafana-agent.yaml

@@ -21,7 +21,14 @@ spec:
     instanceSelector:
       matchLabels:
         {{- include "loki.selectorLabels" $ | nindent 8 }}
-      # cluster label for logs is added in the LogsInstance
+  {{- with $.Values.monitoring.serviceMonitor}}
+  {{- if .metricsInstance.remoteWrite}}
+  metrics:
+    instanceSelector:
+      matchLabels:
+        {{- include "loki.selectorLabels" $ | nindent 8 }}
+  {{- end }}
+  {{- end }}
 
 ---
 

production/helm/loki/templates/monitoring/logs-instance.yaml

@@ -17,13 +17,14 @@ metadata:
 spec:
   clients:
     {{- include "loki.logsInstanceClient" $ | nindent 4}}
+    {{- with .clients}}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
 
-  # Supply an empty namespace selector to look in all namespaces. Remove
-  # this to only look in the same namespace as the LogsInstance CR
-  podLogsNamespaceSelector: { }
+  podLogsNamespaceSelector: {}
 
   podLogsSelector:
     matchLabels:
-      instance: primary
+      {{- include "loki.selectorLabels" $ | nindent 6 }}
 {{- end -}}
 {{- end -}}

production/helm/loki/templates/monitoring/metrics-instance.yaml

@@ -0,0 +1,29 @@
+{{- if .Values.monitoring.serviceMonitor.enabled }}
+{{- with .Values.monitoring.serviceMonitor.metricsInstance }}
+apiVersion: monitoring.grafana.com/v1alpha1
+kind: MetricsInstance
+metadata:
+  name: {{ include "loki.fullname" $ }}
+  namespace: {{ .namespace | default $.Release.Namespace }}
+  {{- with .annotations }}
+  annotations:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+  labels:
+    {{- include "loki.labels" $ | nindent 4 }}
+    {{- with .labels }}
+    {{- toYaml . | nindent 4 }}
+    {{- end }}
+spec:
+  {{- with .remoteWrite}}
+  remoteWrite:
+    {{- toYaml . | nindent 4 }}
+  {{- end }}
+
+  serviceMonitorNamespaceSelector: {}
+
+  serviceMonitorSelector:
+    matchLabels:
+      {{- include "loki.selectorLabels" $ | nindent 6 }}
+{{- end -}}
+{{- end -}}

production/helm/loki/templates/monitoring/pod-logs.yaml

@@ -11,7 +11,6 @@ metadata:
     {{- toYaml . | nindent 4 }}
   {{- end }}
   labels:
-    instance: primary
     {{- include "loki.labels" $ | nindent 4 }}
     {{- with .labels }}
     {{- toYaml . | nindent 4 }}

production/helm/loki/values.yaml

@@ -616,6 +616,14 @@ monitoring:
     scheme: http
     # -- ServiceMonitor will use these tlsConfig settings to make the health check requests
     tlsConfig: null
+    # -- If defined, will create a MetricsInstance for the Grafana Agent Operator.
+    metricsInstance:
+      # -- MerticsInstance annotations
+      annotations: {}
+      # -- Additional MatricsInstance labels
+      labels: {}
+      # -- If defined a MetricsInstance will be created to remote write metrics.
+      remoteWrite: null
 
   # Self monitoring determines whether Loki should scrape it's own logs.
   # This feature currently relies on the Grafana Agent Operator being installed,
@@ -663,6 +671,8 @@ monitoring:
       annotations: {}
       # -- Additional LogsInstance labels
       labels: {}
+      # -- Additional clients for remote write
+      clients: null
 
     # The Loki canary pushes logs to and queries from this loki installation to test
     # that it's working correctly

tools/dev/k3d/Makefile

@@ -63,5 +63,5 @@ prepare: create-registry update-repos secrets
 
 build-latest-image:
 	make -C $(CURDIR)/../../.. loki-image
-	docker tag grafana/loki:$(IMAGE_TAG) k4d-grafana:$(REGISTRY_PORT)/loki:latest
-	docker push k3d-grafana:$(REGISTRY_PORT)/loki:latest
+	docker tag grafana/loki:$(IMAGE_TAG) grafana.k3d.localhost:$(REGISTRY_PORT)/loki:latest
+	docker push grafana.k3d.localhost:$(REGISTRY_PORT)/loki:latest

tools/dev/k3d/environments/loki-distributed/spec.json

@@ -6,7 +6,7 @@
     "namespace": "environments/loki-distributed/main.jsonnet"
   },
   "spec": {
-    "apiServer": "https://0.0.0.0:42281",
+    "apiServer": "https://0.0.0.0:38655",
     "namespace": "k3d-loki-distributed",
     "resourceDefaults": {},
     "expectVersions": {}