Added dropped targets API to targets endpoint (#3870)

docs/querying/api.md

@@ -315,9 +315,6 @@ String results are returned as result type `string`. The corresponding
 
 ## Targets
 
-> This API is experimental as it is intended to be extended with targets
-> dropped due to relabelling in the future.
-
 The following endpoint returns an overview of the current state of the
 Prometheus target discovery:
 
@@ -325,7 +322,9 @@ Prometheus target discovery:
 GET /api/v1/targets
 ```
 
-Currently only the active targets are part of the response.
+Both the active and dropped targets are part of the response.
+`labels` represents the label set after relabelling has occurred.
+`discoveredLabels` represent the unmodified labels retrieved during service discovery before relabelling has occurred.
 
 ```json
 $ curl http://localhost:9090/api/v1/targets
@@ -349,6 +348,16 @@ $ curl http://localhost:9090/api/v1/targets
         "lastScrape": "2017-01-17T15:07:44.723715405+01:00",
         "health": "up"
       }
+    ],
+    "droppedTargets": [
+      {
+        "discoveredLabels": {
+          "__address__": "127.0.0.1:9100",
+          "__metrics_path__": "/metrics",
+          "__scheme__": "http",
+          "job": "node"
+        },
+      }
     ]
   }
 }

scrape/manager.go

@@ -132,6 +132,21 @@ func (m *Manager) Targets() []*Target {
 	return targets
 }
 
+// DroppedTargets returns the targets dropped during relabelling.
+func (m *Manager) DroppedTargets() []*Target {
+	m.mtx.Lock()
+	defer m.mtx.Unlock()
+	var droppedTargets []*Target
+	for _, p := range m.scrapePools {
+		p.mtx.RLock()
+		for _, tt := range p.droppedTargets {
+			droppedTargets = append(droppedTargets, tt)
+		}
+		p.mtx.RUnlock()
+	}
+	return droppedTargets
+}
+
 func (m *Manager) reload(t map[string][]*targetgroup.Group) {
 	for tsetName, tgroup := range t {
 		scrapeConfig, ok := m.scrapeConfigs[tsetName]

web/api/v1/api.go

@@ -83,6 +83,7 @@ func (e *apiError) Error() string {
 
 type targetRetriever interface {
 	Targets() []*scrape.Target
+	DroppedTargets() []*scrape.Target
 }
 
 type alertmanagerRetriever interface {
@@ -438,14 +439,22 @@ type Target struct {
 	Health     scrape.TargetHealth `json:"health"`
 }
 
+// DroppedTarget has the information for one target that was dropped during relabelling.
+type DroppedTarget struct {
+	// Labels before any processing.
+	DiscoveredLabels map[string]string `json:"discoveredLabels"`
+}
+
 // TargetDiscovery has all the active targets.
 type TargetDiscovery struct {
-	ActiveTargets []*Target `json:"activeTargets"`
+	ActiveTargets  []*Target        `json:"activeTargets"`
+	DroppedTargets []*DroppedTarget `json:"droppedTargets"`
 }
 
 func (api *API) targets(r *http.Request) (interface{}, *apiError) {
 	targets := api.targetRetriever.Targets()
-	res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets))}
+	droppedTargets := api.targetRetriever.DroppedTargets()
+	res := &TargetDiscovery{ActiveTargets: make([]*Target, len(targets)), DroppedTargets: make([]*DroppedTarget, len(droppedTargets))}
 
 	for i, t := range targets {
 		lastErrStr := ""
@@ -464,6 +473,12 @@ func (api *API) targets(r *http.Request) (interface{}, *apiError) {
 		}
 	}
 
+	for i, t := range droppedTargets {
+		res.DroppedTargets[i] = &DroppedTarget{
+			DiscoveredLabels: t.DiscoveredLabels().Map(),
+		}
+	}
+
 	return res, nil
 }
 

web/api/v1/api_test.go

@@ -42,10 +42,34 @@ import (
 	"github.com/prometheus/prometheus/storage/remote"
 )
 
-type targetRetrieverFunc func() []*scrape.Target
-
-func (f targetRetrieverFunc) Targets() []*scrape.Target {
-	return f()
+type testTargetRetriever struct{}
+
+func (t testTargetRetriever) Targets() []*scrape.Target {
+	return []*scrape.Target{
+		scrape.NewTarget(
+			labels.FromMap(map[string]string{
+				model.SchemeLabel:      "http",
+				model.AddressLabel:     "example.com:8080",
+				model.MetricsPathLabel: "/metrics",
+			}),
+			nil,
+			url.Values{},
+		),
+	}
+}
+func (t testTargetRetriever) DroppedTargets() []*scrape.Target {
+	return []*scrape.Target{
+		scrape.NewTarget(
+			nil,
+			labels.FromMap(map[string]string{
+				model.AddressLabel:     "http://dropped.example.com:9115",
+				model.MetricsPathLabel: "/probe",
+				model.SchemeLabel:      "http",
+				model.JobLabel:         "blackbox",
+			}),
+			url.Values{},
+		),
+	}
 }
 
 type testAlertmanagerRetriever struct{}
@@ -102,19 +126,7 @@ func TestEndpoints(t *testing.T) {
 
 	now := time.Now()
 
-	tr := targetRetrieverFunc(func() []*scrape.Target {
-		return []*scrape.Target{
-			scrape.NewTarget(
-				labels.FromMap(map[string]string{
-					model.SchemeLabel:      "http",
-					model.AddressLabel:     "example.com:8080",
-					model.MetricsPathLabel: "/metrics",
-				}),
-				nil,
-				url.Values{},
-			),
-		}
-	})
+	var tr testTargetRetriever
 
 	var ar testAlertmanagerRetriever
 
@@ -447,6 +459,16 @@ func TestEndpoints(t *testing.T) {
 						Health:           "unknown",
 					},
 				},
+				DroppedTargets: []*DroppedTarget{
+					{
+						DiscoveredLabels: map[string]string{
+							"__address__":      "http://dropped.example.com:9115",
+							"__metrics_path__": "/probe",
+							"__scheme__":       "http",
+							"job":              "blackbox",
+						},
+					},
+				},
 			},
 		},
 		{