Prometheus: Fix inconsistent labels in exemplars resulting in marshal json error. (#46135)

like "frame has different field lengths, field 0 is len 5 but field 14 is len 2"

Signed-off-by: Jimmie Han <hanjinming@outlook.com>

pkg/tsdb/prometheus/buffered/time_series_query.go

@@ -438,7 +438,16 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
 	// TODO: this preallocation is very naive.
 	// We should figure out a better approximation here.
 	events := make([]ExemplarEvent, 0, len(response)*2)
-
+	// Prometheus treats empty value as same as null, so `event.Labels` may not be consistent across `events`,
+	// leading errors like "frame has different field lengths, field 0 is len 5 but field 14 is len 2", need a fix.
+	eventLabels := make(map[string]struct{})
+	for _, exemplarData := range response {
+		for _, exemplar := range exemplarData.Exemplars {
+			for label := range exemplar.Labels {
+				eventLabels[string(label)] = struct{}{}
+			}
+		}
+	}
 	for _, exemplarData := range response {
 		for _, exemplar := range exemplarData.Exemplars {
 			event := ExemplarEvent{}
@@ -455,6 +464,15 @@ func exemplarToDataFrames(response []apiv1.ExemplarQueryResult, query *Prometheu
 				event.Labels[string(seriesLabel)] = string(seriesValue)
 			}
 
+			if len(event.Labels) != len(eventLabels) {
+				// Fill event labels with empty value.
+				for label := range eventLabels {
+					if _, ok := event.Labels[label]; !ok {
+						event.Labels[label] = ""
+					}
+				}
+			}
+
 			events = append(events, event)
 		}
 	}

pkg/tsdb/prometheus/buffered/time_series_query_test.go

@@ -594,6 +594,53 @@ func TestPrometheus_parseTimeSeriesResponse(t *testing.T) {
 		require.Equal(t, res[0].Fields[1].At(1), 0.003535405)
 	})
 
+	t.Run("exemplars response with inconsistent labels should marshal json ok", func(t *testing.T) {
+		value := make(map[TimeSeriesQueryType]interface{})
+		exemplars := []apiv1.ExemplarQueryResult{
+			{
+				SeriesLabels: p.LabelSet{
+					"__name__": "tns_request_duration_seconds_bucket",
+					"instance": "app:80",
+					"job":      "tns/app",
+					"service":  "example",
+				},
+				Exemplars: []apiv1.Exemplar{
+					{
+						Labels:    p.LabelSet{"traceID": "test1"},
+						Value:     0.003535405,
+						Timestamp: p.TimeFromUnixNano(time.Now().Add(-2 * time.Minute).UnixNano()),
+					},
+				},
+			},
+			{
+				SeriesLabels: p.LabelSet{
+					"__name__": "tns_request_duration_seconds_bucket",
+					"instance": "app:80",
+					"job":      "tns/app",
+					"service":  "example",
+				},
+				Exemplars: []apiv1.Exemplar{
+					{
+						Labels:    p.LabelSet{"traceID": "test2", "userID": "test3"},
+						Value:     0.003535405,
+						Timestamp: p.TimeFromUnixNano(time.Now().Add(-2 * time.Minute).UnixNano()),
+					},
+				},
+			},
+		}
+
+		value[ExemplarQueryType] = exemplars
+		query := &PrometheusQuery{
+			LegendFormat: "legend {{app}}",
+		}
+		res, err := parseTimeSeriesResponse(value, query)
+		require.NoError(t, err)
+
+		// Test frame marshal json no error.
+		_, err = res[0].MarshalJSON()
+		require.NoError(t, err)
+	})
+
 	t.Run("matrix response should be parsed normally", func(t *testing.T) {
 		values := []p.SamplePair{
 			{Value: 1, Timestamp: 1000},