Add __meta_kubernetes_pod_phase label in discovery (#4824)

This lets you add a relabel rule to drop scrapes for pods which are
not running.

Signed-off-by: Bryan Boreham <bjboreham@gmail.com>

discovery/kubernetes/endpoints_test.go

@@ -197,6 +197,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) {
 						"__meta_kubernetes_pod_name":                     "testpod",
 						"__meta_kubernetes_pod_ip":                       "1.2.3.4",
 						"__meta_kubernetes_pod_ready":                    "unknown",
+						"__meta_kubernetes_pod_phase":                    "",
 						"__meta_kubernetes_pod_node_name":                "testnode",
 						"__meta_kubernetes_pod_host_ip":                  "2.3.4.5",
 						"__meta_kubernetes_pod_container_name":           "c1",
@@ -210,6 +211,7 @@ func TestEndpointsDiscoveryAdd(t *testing.T) {
 						"__meta_kubernetes_pod_name":                    "testpod",
 						"__meta_kubernetes_pod_ip":                      "1.2.3.4",
 						"__meta_kubernetes_pod_ready":                   "unknown",
+						"__meta_kubernetes_pod_phase":                   "",
 						"__meta_kubernetes_pod_node_name":               "testnode",
 						"__meta_kubernetes_pod_host_ip":                 "2.3.4.5",
 						"__meta_kubernetes_pod_container_name":          "c2",
@@ -585,6 +587,7 @@ func TestEndpointsDiscoveryNamespaces(t *testing.T) {
 						"__meta_kubernetes_pod_name":                     "testpod",
 						"__meta_kubernetes_pod_ip":                       "4.3.2.1",
 						"__meta_kubernetes_pod_ready":                    "unknown",
+						"__meta_kubernetes_pod_phase":                    "",
 						"__meta_kubernetes_pod_node_name":                "testnode",
 						"__meta_kubernetes_pod_host_ip":                  "2.3.4.5",
 						"__meta_kubernetes_pod_container_name":           "c1",

discovery/kubernetes/pod.go

@@ -142,6 +142,7 @@ const (
 	podContainerPortNumberLabel   = metaLabelPrefix + "pod_container_port_number"
 	podContainerPortProtocolLabel = metaLabelPrefix + "pod_container_port_protocol"
 	podReadyLabel                 = metaLabelPrefix + "pod_ready"
+	podPhaseLabel                 = metaLabelPrefix + "pod_phase"
 	podLabelPrefix                = metaLabelPrefix + "pod_label_"
 	podAnnotationPrefix           = metaLabelPrefix + "pod_annotation_"
 	podNodeNameLabel              = metaLabelPrefix + "pod_node_name"
@@ -167,6 +168,7 @@ func podLabels(pod *apiv1.Pod) model.LabelSet {
 		podNameLabel:     lv(pod.ObjectMeta.Name),
 		podIPLabel:       lv(pod.Status.PodIP),
 		podReadyLabel:    podReady(pod),
+		podPhaseLabel:    lv(string(pod.Status.Phase)),
 		podNodeNameLabel: lv(pod.Spec.NodeName),
 		podHostIPLabel:   lv(pod.Status.HostIP),
 		podUID:           lv(string(pod.ObjectMeta.UID)),

discovery/kubernetes/pod_test.go

@@ -70,6 +70,7 @@ func makeMultiPortPods() *v1.Pod {
 		Status: v1.PodStatus{
 			PodIP:  "1.2.3.4",
 			HostIP: "2.3.4.5",
+			Phase:  "Running",
 			Conditions: []v1.PodCondition{
 				{
 					Type:   v1.PodReady,
@@ -105,6 +106,7 @@ func makePods() *v1.Pod {
 		Status: v1.PodStatus{
 			PodIP:  "1.2.3.4",
 			HostIP: "2.3.4.5",
+			Phase:  "Running",
 			Conditions: []v1.PodCondition{
 				{
 					Type:   v1.PodReady,
@@ -135,6 +137,7 @@ func expectedPodTargetGroups(ns string) map[string]*targetgroup.Group {
 				"__meta_kubernetes_pod_ip":        "1.2.3.4",
 				"__meta_kubernetes_pod_host_ip":   "2.3.4.5",
 				"__meta_kubernetes_pod_ready":     "true",
+				"__meta_kubernetes_pod_phase":     "Running",
 				"__meta_kubernetes_pod_uid":       "abc123",
 			},
 			Source: key,
@@ -184,6 +187,7 @@ func TestPodDiscoveryBeforeRun(t *testing.T) {
 					"__meta_kubernetes_pod_ip":                        "1.2.3.4",
 					"__meta_kubernetes_pod_host_ip":                   "2.3.4.5",
 					"__meta_kubernetes_pod_ready":                     "true",
+					"__meta_kubernetes_pod_phase":                     "Running",
 					"__meta_kubernetes_pod_uid":                       "abc123",
 					"__meta_kubernetes_pod_controller_kind":           "testcontrollerkind",
 					"__meta_kubernetes_pod_controller_name":           "testcontrollername",

docs/configuration/configuration.md

@@ -717,6 +717,8 @@ Available meta labels:
 * `__meta_kubernetes_pod_container_port_number`: Number of the container port.
 * `__meta_kubernetes_pod_container_port_protocol`: Protocol of the container port.
 * `__meta_kubernetes_pod_ready`: Set to `true` or `false` for the pod's ready state.
+* `__meta_kubernetes_pod_phase`: Set to `Pending`, `Running`, `Succeeded`, `Failed` or `Unknown`
+  in the [lifecycle](https://kubernetes.io/docs/concepts/workloads/pods/pod-lifecycle/#pod-phase).
 * `__meta_kubernetes_pod_node_name`: The name of the node the pod is scheduled onto.
 * `__meta_kubernetes_pod_host_ip`: The current host IP of the pod object.
 * `__meta_kubernetes_pod_uid`: The UID of the pod object.