From e18b4fca6f78c63020b7831b5986fb7d123366fa Mon Sep 17 00:00:00 2001 From: Kaviraj Kanagaraj Date: Fri, 20 Jan 2023 14:40:25 +0100 Subject: [PATCH] ingester: Add `ingester_memory_streams_labels_bytes` metric (#8212) Signed-off-by: Kaviraj This is to give visibility on `labels` size of in-memory streams on the ingesters. It caculate the size of labels via `len(labels.String())`. Meaning if you have ```go lbs := labels.Labels{ {Name: "foo", Value: "bar"}, {Name: "instance", Value: "loki"}, } ``` size would be number of bytes in `{foo="bar", instance="loki"}` which is `28`. NOTE: `len(string)` gives [bytes length in Go](https://pkg.go.dev/builtin#len) --- CHANGELOG.md | 1 + pkg/ingester/instance.go | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index 02a9bc4803..9271cfec72 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -26,6 +26,7 @@ * [8047](https://github.com/grafana/loki/pull/8047) **bboreham**: Dashboards: add k8s resource requests to CPU and memory panels. * [8061](https://github.com/grafana/loki/pull/8061) **kavirajk**: Remove circle from Loki OSS * [8131](https://github.com/grafana/loki/pull/8131) **jeschkies**: Compile Promtail ARM and ARM64 with journald support. +* [8212](https://github.com/grafana/loki/pull/8212) **kavirajk**: ingester: Add `ingester_memory_streams_labels_bytes metric` for more visibility of size of metadata of in-memory streams. ##### Fixes diff --git a/pkg/ingester/instance.go b/pkg/ingester/instance.go index 6134353d2a..387d3da415 100644 --- a/pkg/ingester/instance.go +++ b/pkg/ingester/instance.go @@ -53,6 +53,11 @@ var ( Name: "ingester_memory_streams", Help: "The total number of streams in memory per tenant.", }, []string{"tenant"}) + memoryStreamsLabelsBytes = promauto.NewGauge(prometheus.GaugeOpts{ + Namespace: "loki", + Name: "ingester_memory_streams_labels_bytes", + Help: "Total bytes of labels of the streams in memory.", + }) streamsCreatedTotal = promauto.NewCounterVec(prometheus.CounterOpts{ Namespace: "loki", Name: "ingester_streams_created_total", @@ -279,6 +284,7 @@ func (i *instance) createStream(pushReqStream logproto.Stream, record *WALRecord } memoryStreams.WithLabelValues(i.instanceID).Inc() + memoryStreamsLabelsBytes.Add(float64(len(s.labels.String()))) i.streamsCreatedTotal.Inc() i.addTailersToNewStream(s) streamsCountStats.Add(1) @@ -300,6 +306,7 @@ func (i *instance) createStreamByFP(ls labels.Labels, fp model.Fingerprint) *str i.streamsCreatedTotal.Inc() memoryStreams.WithLabelValues(i.instanceID).Inc() + memoryStreamsLabelsBytes.Add(float64(len(s.labels.String()))) i.addTailersToNewStream(s) return s @@ -322,6 +329,7 @@ func (i *instance) removeStream(s *stream) { i.index.Delete(s.labels, s.fp) i.streamsRemovedTotal.Inc() memoryStreams.WithLabelValues(i.instanceID).Dec() + memoryStreamsLabelsBytes.Sub(float64(len(s.labels.String()))) streamsCountStats.Add(-1) } }