Merge pull request #18194 from rexagod/11726

discovery: Introduce `prometheus_sd_last_update_timestamp_seconds`

discovery/manager.go

@@ -475,6 +475,7 @@ func (m *Manager) allGroups() map[string][]*targetgroup.Group {
 
 	for setName, v := range n {
 		m.metrics.DiscoveredTargets.WithLabelValues(setName).Set(float64(v))
+		m.metrics.LastUpdated.WithLabelValues(setName).SetToCurrentTime()
 	}
 
 	return tSets

discovery/manager_test.go

@@ -1595,3 +1595,30 @@ func TestConfigReloadAndShutdownRace(t *testing.T) {
 	cancel()
 	wgBg.Wait()
 }
+
+func TestGaugeLastUpdateTimestamp(t *testing.T) {
+	ctx := t.Context()
+
+	reg := prometheus.NewRegistry()
+	_, sdMetrics := NewTestMetrics(t, reg)
+
+	discoveryManager := NewManager(ctx, promslog.NewNopLogger(), reg, sdMetrics)
+	require.NotNil(t, discoveryManager)
+	discoveryManager.updatert = 100 * time.Millisecond
+	go discoveryManager.Run()
+
+	c := map[string]Configs{
+		"prometheus": {
+			staticConfig("foo:9090"),
+		},
+	}
+	discoveryManager.ApplyConfig(c)
+
+	before := time.Now()
+	<-discoveryManager.SyncCh()
+	after := time.Now()
+
+	ts := client_testutil.ToFloat64(discoveryManager.metrics.LastUpdated.WithLabelValues("prometheus"))
+	require.GreaterOrEqual(t, ts, float64(before.UnixNano())/1e9, "last update timestamp should be >= time before sync")
+	require.LessOrEqual(t, ts, float64(after.UnixNano())/1e9, "last update timestamp should be <= time after sync")
+}

discovery/metrics.go

@@ -26,6 +26,7 @@ type Metrics struct {
 	ReceivedUpdates   prometheus.Counter
 	DelayedUpdates    prometheus.Counter
 	SentUpdates       prometheus.Counter
+	LastUpdated       *prometheus.GaugeVec
 }
 
 func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (*Metrics, error) {
@@ -72,12 +73,22 @@ func NewManagerMetrics(registerer prometheus.Registerer, sdManagerName string) (
 		},
 	)
 
+	m.LastUpdated = prometheus.NewGaugeVec(
+		prometheus.GaugeOpts{
+			Name:        "prometheus_sd_last_update_timestamp_seconds",
+			Help:        "Timestamp of the last update sent to the SD consumers.",
+			ConstLabels: prometheus.Labels{"name": sdManagerName},
+		},
+		[]string{"config"},
+	)
+
 	metrics := []prometheus.Collector{
 		m.FailedConfigs,
 		m.DiscoveredTargets,
 		m.ReceivedUpdates,
 		m.DelayedUpdates,
 		m.SentUpdates,
+		m.LastUpdated,
 	}
 
 	for _, collector := range metrics {
@@ -97,4 +108,5 @@ func (m *Metrics) Unregister(registerer prometheus.Registerer) {
 	registerer.Unregister(m.ReceivedUpdates)
 	registerer.Unregister(m.DelayedUpdates)
 	registerer.Unregister(m.SentUpdates)
+	registerer.Unregister(m.LastUpdated)
 }