Fix honor_labels for empty labels, prune empty labels.

The semantics of honor_labels are that if a target exposes
and empty label it will override the target labels. This PR
fixes that by once again distinguishing between empty labels
and missing labels in this one use case.

Beyond that empty labels should be pruned and not added to storage,
which this also fixes.

Fixes #3841

pkg/labels/labels.go

@@ -112,6 +112,16 @@ func (ls Labels) Get(name string) string {
 	return ""
 }
 
+// Has returns if the label with the given name is present.
+func (ls Labels) Has(name string) bool {
+	for _, l := range ls {
+		if l.Name == name {
+			return true
+		}
+	}
+	return false
+}
+
 // Equal returns whether the two label sets are equal.
 func Equal(ls, o Labels) bool {
 	if len(ls) != len(o) {

scrape/scrape.go

@@ -336,7 +336,7 @@ func (sp *scrapePool) mutateSampleLabels(lset labels.Labels, target *Target) lab
 
 	if sp.config.HonorLabels {
 		for _, l := range target.Labels() {
-			if lv := lset.Get(l.Name); lv == "" {
+			if !lset.Has(l.Name) {
 				lb.Set(l.Name, l.Value)
 			}
 		}
@@ -350,6 +350,12 @@ func (sp *scrapePool) mutateSampleLabels(lset labels.Labels, target *Target) lab
 		}
 	}
 
+	for _, l := range lb.Labels() {
+		if l.Value == "" {
+			lb.Del(l.Name)
+		}
+	}
+
 	res := lb.Labels()
 
 	if mrc := sp.config.MetricRelabelConfigs; len(mrc) > 0 {