Fix bug with new promtail stream lag metric where labels might not be (#5675)

applied if they don't exist on the source stream

Signed-off-by: Callum Styan <callumstyan@gmail.com>

clients/pkg/promtail/client/client.go

@@ -334,12 +334,16 @@ func (c *client) sendBatch(tenantID string, batch *batch) {
 					return
 				}
 				lblSet := make(prometheus.Labels)
-				for i := range lbls {
-					for _, lbl := range c.streamLagLabels {
+				for _, lbl := range c.streamLagLabels {
+					// label from streamLagLabels may not be found but we still need an empty value
+					// so that the prometheus client library doesn't panic on inconsistent label cardinality
+					value := ""
+					for i := range lbls {
 						if lbls[i].Name == lbl {
-							lblSet[lbl] = lbls[i].Value
+							value = lbls[i].Value
 						}
 					}
+					lblSet[lbl] = value
 				}
 				if lblSet != nil {
 					// always set host