loki/pkg/bloombuild/planner/metrics.go

package planner

import (
	"time"

	"github.com/prometheus/client_golang/prometheus"
	"github.com/prometheus/client_golang/prometheus/promauto"

	"github.com/grafana/loki/v3/pkg/queue"
)

const (
	metricsNamespace = "loki"
	metricsSubsystem = "bloomplanner"

	statusSuccess = "success"
	statusFailure = "failure"

	phasePlanning = "planning"
	phaseBuilding = "building"
)

type Metrics struct {
	running prometheus.Gauge

	// Extra Queue metrics
	connectedBuilders prometheus.GaugeFunc
	queueDuration     prometheus.Histogram
	inflightRequests  prometheus.Summary
	tasksRequeued     prometheus.Counter
	taskLost          prometheus.Counter

	planningTime     prometheus.Histogram
	buildStarted     prometheus.Counter
	buildCompleted   *prometheus.CounterVec
	buildTime        *prometheus.HistogramVec
	buildLastSuccess prometheus.Gauge

	blocksDeleted *prometheus.CounterVec
	metasDeleted  *prometheus.CounterVec

	tenantsDiscovered    prometheus.Counter
	tenantTasksPlanned   *prometheus.GaugeVec
	tenantTasksCompleted *prometheus.GaugeVec
}

func NewMetrics(
	r prometheus.Registerer,
	getConnectedBuilders func() float64,
) *Metrics {
	return &Metrics{
		running: promauto.With(r).NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "running",
			Help:      "Value will be 1 if bloom planner is currently running on this instance",
		}),
		connectedBuilders: promauto.With(r).NewGaugeFunc(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "connected_builders",
			Help:      "Number of builders currently connected to the planner.",
		}, getConnectedBuilders),
		queueDuration: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "queue_duration_seconds",
			Help:      "Time spend by tasks in queue before getting picked up by a builder.",
			Buckets:   prometheus.DefBuckets,
		}),
		inflightRequests: promauto.With(r).NewSummary(prometheus.SummaryOpts{
			Namespace:  metricsNamespace,
			Subsystem:  metricsSubsystem,
			Name:       "inflight_tasks",
			Help:       "Number of inflight tasks (either queued or processing) sampled at a regular interval. Quantile buckets keep track of inflight tasks over the last 60s.",
			Objectives: map[float64]float64{0.5: 0.05, 0.75: 0.02, 0.8: 0.02, 0.9: 0.01, 0.95: 0.01, 0.99: 0.001},
			MaxAge:     time.Minute,
			AgeBuckets: 6,
		}),
		tasksRequeued: promauto.With(r).NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "tasks_requeued_total",
			Help:      "Total number of tasks requeued due to not being picked up by a builder.",
		}),
		taskLost: promauto.With(r).NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "tasks_lost_total",
			Help:      "Total number of tasks lost due to not being picked up by a builder and failed to be requeued.",
		}),

		planningTime: promauto.With(r).NewHistogram(prometheus.HistogramOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "planning_time_seconds",
			Help:      "Time spent planning a build cycle.",
			// 1s --> 1h (steps of 1 minute)
			Buckets: prometheus.LinearBuckets(1, 60, 60),
		}),
		buildStarted: promauto.With(r).NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "build_started_total",
			Help:      "Total number of builds started",
		}),
		buildCompleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "build_completed_total",
			Help:      "Total number of builds completed",
		}, []string{"status"}),
		buildTime: promauto.With(r).NewHistogramVec(prometheus.HistogramOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "build_time_seconds",
			Help:      "Time spent during a builds cycle.",
			// Buckets in seconds:
			Buckets: append(
				// 1s --> 1h (steps of 10 minutes)
				prometheus.LinearBuckets(1, 600, 6),
				// 1h --> 24h (steps of 1 hour)
				prometheus.LinearBuckets(3600, 3600, 24)...,
			),
		}, []string{"status"}),
		buildLastSuccess: promauto.With(r).NewGauge(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "build_last_successful_run_timestamp_seconds",
			Help:      "Unix timestamp of the last successful build cycle.",
		}),

		blocksDeleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "blocks_deleted_total",
			Help:      "Number of blocks deleted",
		}, []string{"phase"}),
		metasDeleted: promauto.With(r).NewCounterVec(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "metas_deleted_total",
			Help:      "Number of metas deleted",
		}, []string{"phase"}),

		tenantsDiscovered: promauto.With(r).NewCounter(prometheus.CounterOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "tenants_discovered_total",
			Help:      "Number of tenants discovered during the current build iteration",
		}),
		tenantTasksPlanned: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "tenant_tasks_planned",
			Help:      "Number of tasks planned for a tenant during the current build iteration.",
		}, []string{"tenant"}),
		tenantTasksCompleted: promauto.With(r).NewGaugeVec(prometheus.GaugeOpts{
			Namespace: metricsNamespace,
			Subsystem: metricsSubsystem,
			Name:      "tenant_tasks_completed",
			Help:      "Number of tasks completed for a tenant during the current build iteration.",
		}, []string{"tenant", "status"}),
	}
}

func NewQueueMetrics(r prometheus.Registerer) *queue.Metrics {
	return queue.NewMetrics(r, metricsNamespace, metricsSubsystem)
}