fix: incorrect pod matcher for compactor in mixin when using ssd mode (#12846)

pull/13573/head
Quentin Bisson 10 months ago committed by GitHub
parent 29110feb99
commit 515e13cc6c
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 10
      production/loki-mixin-compiled-ssd/dashboards/loki-deletion.json
  2. 10
      production/loki-mixin-compiled/dashboards/loki-deletion.json
  3. 6
      production/loki-mixin/dashboards/loki-deletion.libsonnet

@ -379,7 +379,7 @@
"span": 4, "span": 4,
"targets": [ "targets": [
{ {
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"}", "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"format": "time_series", "format": "time_series",
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null "legendLink": null
@ -426,7 +426,7 @@
"span": 4, "span": 4,
"targets": [ "targets": [
{ {
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"format": "time_series", "format": "time_series",
"legendFormat": " {{pod}} ", "legendFormat": " {{pod}} ",
"legendLink": null "legendLink": null
@ -579,7 +579,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"}[$__rate_interval])) by (user)", "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"format": "time_series", "format": "time_series",
"legendFormat": "{{user}}", "legendFormat": "{{user}}",
"legendLink": null "legendLink": null
@ -606,7 +606,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"refId": "A" "refId": "A"
} }
], ],
@ -619,7 +619,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"loki\", pod=~\"(loki.*|enterprise-logs)-backend.*\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"refId": "A" "refId": "A"
} }
], ],

@ -379,7 +379,7 @@
"span": 4, "span": 4,
"targets": [ "targets": [
{ {
"expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}", "expr": "node_namespace_pod_container:container_cpu_usage_seconds_total:sum_irate{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}",
"format": "time_series", "format": "time_series",
"legendFormat": "{{pod}}", "legendFormat": "{{pod}}",
"legendLink": null "legendLink": null
@ -426,7 +426,7 @@
"span": 4, "span": 4,
"targets": [ "targets": [
{ {
"expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} / 1024 / 1024 ", "expr": "go_memstats_heap_inuse_bytes{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} / 1024 / 1024 ",
"format": "time_series", "format": "time_series",
"legendFormat": " {{pod}} ", "legendFormat": " {{pod}} ",
"legendLink": null "legendLink": null
@ -579,7 +579,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"}[$__rate_interval])) by (user)", "expr": "sum(rate(loki_compactor_deleted_lines{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"}[$__rate_interval])) by (user)",
"format": "time_series", "format": "time_series",
"legendFormat": "{{user}}", "legendFormat": "{{user}}",
"legendLink": null "legendLink": null
@ -606,7 +606,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ", "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"Started processing delete request|delete request for user marked as processed\" | logfmt | line_format \"{{.ts}} user={{.user}} delete_request_id={{.delete_request_id}} msg={{.msg}}\" ",
"refId": "A" "refId": "A"
} }
], ],
@ -619,7 +619,7 @@
"span": 6, "span": 6,
"targets": [ "targets": [
{ {
"expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", container=\"compactor\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"", "expr": "{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(compactor|(loki.*|enterprise-logs)-backend.*|loki-single-binary)\"} |~ \"delete request for user added\" | logfmt | line_format \"{{.ts}} user={{.user}} query='{{.query}}'\"",
"refId": "A" "refId": "A"
} }
], ],

@ -2,9 +2,7 @@ local g = import 'grafana-builder/grafana.libsonnet';
local utils = import 'mixin-utils/utils.libsonnet'; local utils = import 'mixin-utils/utils.libsonnet';
(import 'dashboard-utils.libsonnet') { (import 'dashboard-utils.libsonnet') {
local compactor_matcher = if $._config.meta_monitoring.enabled local compactor_matcher = 'pod=~"(compactor|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher,
then 'pod=~"(compactor|%s-backend.*|loki-single-binary)"' % $._config.ssd.pod_prefix_matcher
else if $._config.ssd.enabled then 'container="loki", pod=~"%s-backend.*"' % $._config.ssd.pod_prefix_matcher else 'container="compactor"',
grafanaDashboards+:: grafanaDashboards+::
{ {
'loki-deletion.json': 'loki-deletion.json':
@ -50,7 +48,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
) )
.addPanel( .addPanel(
$.newQueryPanel('Compactor memory usage (MiB)') + $.newQueryPanel('Compactor memory usage (MiB)') +
g.queryPanel('go_memstats_heap_inuse_bytes{%s, container="compactor"} / 1024 / 1024 ' % $.namespaceMatcher(), ' {{pod}} '), g.queryPanel('go_memstats_heap_inuse_bytes{%s, %s} / 1024 / 1024 ' % [$.namespaceMatcher(), compactor_matcher], ' {{pod}} '),
) )
.addPanel( .addPanel(
$.newQueryPanel('Compaction run duration (seconds)') + $.newQueryPanel('Compaction run duration (seconds)') +

Loading…
Cancel
Save