apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

fix(mixins): add backend path section in loki-operational for single scalable deployment (#13023)

Browse Source 
Signed-off-by: QuentinBisson <quentin@giantswarm.io>
pull/14173/head
Quentin Bisson 8 months ago committed by GitHub
parent f2d349924c
commit 16881ab0d3
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
3 changed files with 876 additions and 36 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 411
      production/loki-mixin-compiled-ssd/dashboards/loki-operational.json
  2. 405
      production/loki-mixin/dashboards/dashboard-loki-operational.json
  3. 96
      production/loki-mixin/dashboards/loki-operational.libsonnet

411
production/loki-mixin-compiled-ssd/dashboards/loki-operational.json
Unescape View File

@ -3429,6 +3429,417 @@
"title": "Read Path", "title": "Read Path",
"type": "row" "type": "row"
}, },
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 31
},
"id": 64,
"panels": [
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 32
},
"hiddenSeries": false,
"id": 68,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": [ ]
},
"panels": [ ],
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"(loki.*|enterprise-logs)-read.*\"}[$__rate_interval]))",
"intervalFactor": 3,
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeRegions": [ ],
"timeShift": null,
"title": "CPU Usage",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"unit": "binBps"
}
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 39
},
"hiddenSeries": false,
"id": 69,
"legend": {
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": [ ]
},
"panels": [ ],
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"(loki.*|enterprise-logs)-backend.*\"}",
"instant": false,
"intervalFactor": 3,
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeRegions": [ ],
"timeShift": null,
"title": "Memory Usage",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": { },
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "$loki_datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 3,
"w": 18,
"x": 12,
"y": 32
},
"hiddenSeries": false,
"id": 65,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": [ ]
},
"panels": [ ],
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "{}",
"color": "#F2495C"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} | logfmt | level=\"error\"[$__auto]))",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeRegions": [ ],
"timeShift": null,
"title": "Error Log Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": false,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"datasource": "$loki_datasource",
"gridPos": {
"h": 18,
"w": 18,
"x": 12,
"y": 35
},
"id": 66,
"options": {
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"panels": [ ],
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"} |= \"level=error\"",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Logs",
"type": "logs"
},
{
"aliasColors": { },
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 46
},
"hiddenSeries": false,
"id": 70,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": [ ]
},
"panels": [ ],
"percentage": false,
"pointradius": 1,
"points": false,
"renderer": "flot",
"seriesOverrides": [ ],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=~\"($namespace)/(loki.*|enterprise-logs)-backend\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{route}}",
"refId": "A"
}
],
"thresholds": [ ],
"timeFrom": null,
"timeRegions": [ ],
"timeShift": null,
"title": "Success Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": [ ]
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"targets": [ ],
"title": "Backend Path",
"type": "row"
},
{ {
"collapsed": true, "collapsed": true,
"datasource": null, "datasource": null,

405
production/loki-mixin/dashboards/dashboard-loki-operational.json
Unescape View File

@ -3894,6 +3894,411 @@
"title": "Querier", "title": "Querier",
"type": "row" "type": "row"
}, },
{
"collapsed": true,
"datasource": null,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 31
},
"id": 64,
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 32
},
"hiddenSeries": false,
"id": 68,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum by(pod) (rate(container_cpu_usage_seconds_total{cluster=~\"$cluster\", namespace=~\"$namespace\", pod=~\"querier.*\"}[$__rate_interval]))",
"intervalFactor": 3,
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "CPU Usage",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fieldConfig": {
"defaults": {
"unit": "binBps"
}
},
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 39
},
"hiddenSeries": false,
"id": 69,
"legend": {
"avg": false,
"current": false,
"hideEmpty": false,
"hideZero": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 1,
"points": true,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "go_memstats_heap_inuse_bytes{cluster=\"$cluster\", namespace=\"$namespace\", pod=~\"backend.*\"}",
"instant": false,
"intervalFactor": 3,
"legendFormat": "{{pod}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Memory Usage",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "bytes",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "$loki_datasource",
"fill": 1,
"fillGradient": 0,
"gridPos": {
"h": 3,
"w": 18,
"x": 12,
"y": 32
},
"hiddenSeries": false,
"id": 65,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "{}",
"color": "#F2495C"
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate({cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/backend\"} | logfmt | level=\"error\"[$__auto]))",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Error Log Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": false,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": false
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"datasource": "$loki_datasource",
"gridPos": {
"h": 18,
"w": 18,
"x": 12,
"y": 35
},
"id": 66,
"options": {
"showLabels": false,
"showTime": false,
"sortOrder": "Descending",
"wrapLogMessage": true
},
"targets": [
{
"expr": "{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/backend\"} |= \"level=error\"",
"refId": "A"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Logs",
"type": "logs"
},
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "$datasource",
"fill": 0,
"fillGradient": 0,
"gridPos": {
"h": 7,
"w": 6,
"x": 0,
"y": 46
},
"hiddenSeries": false,
"id": 70,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": false,
"total": false,
"values": false
},
"lines": true,
"linewidth": 1,
"nullPointMode": "null",
"options": {
"dataLinks": []
},
"percentage": false,
"pointradius": 1,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"expr": "sum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/backend\", status_code!~\"5[0-9]{2}\"}[$__rate_interval])) by (route)\n/\nsum(rate(loki_request_duration_seconds_count{cluster=\"$cluster\", namespace=\"$namespace\", job=\"$namespace/backend\"}[$__rate_interval])) by (route) > 0",
"interval": "",
"intervalFactor": 1,
"legendFormat": "{{route}}",
"refId": "A"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Success Rate",
"tooltip": {
"shared": true,
"sort": 2,
"value_type": "individual"
},
"type": "timeseries",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"title": "Backend Path",
"type": "row"
},
{ {
"collapsed": true, "collapsed": true,
"datasource": null, "datasource": null,

96
production/loki-mixin/dashboards/loki-operational.libsonnet
Unescape View File

@ -15,6 +15,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
hiddenRows:: [ hiddenRows:: [
'Cassandra', 'Cassandra',
if $._config.ssd.enabled then 'Ingester', if $._config.ssd.enabled then 'Ingester',
if !$._config.ssd.enabled then 'Backend Path',
if !$._config.operational.memcached then 'Memcached', if !$._config.operational.memcached then 'Memcached',
if !$._config.operational.consul then 'Consul', if !$._config.operational.consul then 'Consul',
if !$._config.operational.bigTable then 'Big Table', if !$._config.operational.bigTable then 'Big Table',
@ -43,6 +44,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
queryFrontend: if $._config.meta_monitoring.enabled queryFrontend: if $._config.meta_monitoring.enabled
then [utils.selector.re('job', '($namespace)/(query-frontend|%s-read|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)] then [utils.selector.re('job', '($namespace)/(query-frontend|%s-read|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))], else [utils.selector.re('job', '($namespace)/%s' % (if $._config.ssd.enabled then '%s-read' % $._config.ssd.pod_prefix_matcher else 'query-frontend'))],
backend: [utils.selector.re('job', '($namespace)/%s-backend' % $._config.ssd.pod_prefix_matcher)],
}, },
podMatchers:: { podMatchers:: {
@ -56,6 +58,7 @@ local utils = import 'mixin-utils/utils.libsonnet';
querier: if $._config.meta_monitoring.enabled querier: if $._config.meta_monitoring.enabled
then [utils.selector.re('pod', '(querier|%s-read|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)] then [utils.selector.re('pod', '(querier|%s-read|loki-single-binary)' % $._config.ssd.pod_prefix_matcher)]
else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else 'querier.*'))], else [utils.selector.re('pod', '%s' % (if $._config.ssd.enabled then '%s-read.*' % $._config.ssd.pod_prefix_matcher else 'querier.*'))],
backend: [utils.selector.re('pod', '%s-backend.*' % $._config.ssd.pod_prefix_matcher)],
}, },
} }
+ lokiOperational + { + lokiOperational + {
@ -113,6 +116,36 @@ local utils = import 'mixin-utils/utils.libsonnet';
'' ''
), ),
local replaceBackendMatchers(expr) =
std.strReplace(
std.strReplace(
std.strReplace(
expr,
'pod=~"backend.*"',
matcherStr('backend', matcher='pod', sep='')
),
'job="$namespace/backend",',
matcherStr('backend')
),
'job="$namespace/backend"',
std.rstripChars(matcherStr('backend'), ',')
),
local replaceQuerierMatchers(expr) =
std.strReplace(
std.strReplace(
std.strReplace(
expr,
'pod=~"querier.*"',
matcherStr('querier', matcher='pod', sep='')
),
'job="$namespace/querier",',
matcherStr('querier')
),
'job="$namespace/querier"',
std.rstripChars(matcherStr('querier'), ',')
),
local replaceMatchers(expr) = local replaceMatchers(expr) =
std.strReplace( std.strReplace(
std.strReplace( std.strReplace(
@ -126,59 +159,50 @@ local utils = import 'mixin-utils/utils.libsonnet';
std.strReplace( std.strReplace(
std.strReplace( std.strReplace(
std.strReplace( std.strReplace(
std.strReplace( expr,
std.strReplace( 'pod=~"ingester.*"',
std.strReplace( matcherStr('ingester', matcher='pod', sep='')
expr,
'pod=~"querier.*"',
matcherStr('querier', matcher='pod', sep='')
),
'pod=~"ingester.*"',
matcherStr('ingester', matcher='pod', sep='')
),
'pod=~"distributor.*"',
matcherStr('distributor', matcher='pod', sep='')
),
'job="$namespace/cortex-gw",',
matcherStr('cortexgateway')
), ),
'job="$namespace/cortex-gw"', 'pod=~"distributor.*"',
std.rstripChars(matcherStr('cortexgateway'), ',') matcherStr('distributor', matcher='pod', sep='')
), ),
'job=~"($namespace)/cortex-gw",', 'job="$namespace/cortex-gw",',
matcherStr('cortexgateway') matcherStr('cortexgateway')
), ),
'job="$namespace/distributor",', 'job="$namespace/cortex-gw"',
matcherStr('distributor') std.rstripChars(matcherStr('cortexgateway'), ',')
), ),
'job="$namespace/distributor"', 'job=~"($namespace)/cortex-gw",',
std.rstripChars(matcherStr('distributor'), ',') matcherStr('cortexgateway')
), ),
'job=~"($namespace)/distributor",', 'job="$namespace/distributor",',
matcherStr('distributor') matcherStr('distributor')
), ),
'job=~"($namespace)/distributor"', 'job="$namespace/distributor"',
std.rstripChars(matcherStr('distributor'), ',') std.rstripChars(matcherStr('distributor'), ',')
), ),
'job="$namespace/ingester",', 'job=~"($namespace)/distributor",',
matcherStr('ingester') matcherStr('distributor')
), ),
'job="$namespace/ingester"', 'job=~"($namespace)/distributor"',
std.rstripChars(matcherStr('ingester'), ',') std.rstripChars(matcherStr('distributor'), ',')
), ),
'job=~"($namespace)/ingester",', 'job="$namespace/ingester",',
matcherStr('ingester'), matcherStr('ingester')
), ),
'job="$namespace/querier",', 'job="$namespace/ingester"',
matcherStr('querier') std.rstripChars(matcherStr('ingester'), ',')
), ),
'job="$namespace/querier"', 'job=~"($namespace)/ingester",',
std.rstripChars(matcherStr('querier'), ',') matcherStr('ingester'),
), ),
local replaceAllMatchers(expr) = local replaceAllMatchers(expr) =
replaceMatchers(expr), replaceBackendMatchers(
replaceQuerierMatchers(
replaceMatchers(expr)
)
),
local selectDatasource(ds) = local selectDatasource(ds) =
if ds == null || ds == '' then ds if ds == null || ds == '' then ds

Write Preview
Loading…
Cancel
Save