Alerting: Add alert instance labels to Loki log lines in addition to stream labels (#65403)

Add instance labels to log line

pkg/services/ngalert/state/historian/loki.go

@@ -262,12 +262,13 @@ func statesToStreams(rule history_model.RuleMeta, states []state.StateTransition
 		repr := string(lblJsn)
 
 		entry := lokiEntry{
-			SchemaVersion: 1,
-			Previous:      state.PreviousFormatted(),
-			Current:       state.Formatted(),
-			Values:        valuesAsDataBlob(state.State),
-			DashboardUID:  rule.DashboardUID,
-			PanelID:       rule.PanelID,
+			SchemaVersion:  1,
+			Previous:       state.PreviousFormatted(),
+			Current:        state.Formatted(),
+			Values:         valuesAsDataBlob(state.State),
+			DashboardUID:   rule.DashboardUID,
+			PanelID:        rule.PanelID,
+			InstanceLabels: state.Labels,
 		}
 		if state.State.State == eval.Error {
 			entry.Error = state.Error.Error()
@@ -319,6 +320,9 @@ type lokiEntry struct {
 	Values        *simplejson.Json `json:"values"`
 	DashboardUID  string           `json:"dashboardUID"`
 	PanelID       int64            `json:"panelID"`
+	// InstanceLabels is exactly the set of labels associated with the alert instance in Alertmanager.
+	// These should not be conflated with labels associated with log streams.
+	InstanceLabels map[string]string `json:"labels"`
 }
 
 func valuesAsDataBlob(state *state.State) *simplejson.Json {

pkg/services/ngalert/state/historian/loki_test.go

@@ -130,6 +130,38 @@ func TestRemoteLokiBackend(t *testing.T) {
 			require.NotContains(t, res[0].Stream, "__private__")
 		})
 
+		t.Run("includes instance labels in log line", func(t *testing.T) {
+			rule := createTestRule()
+			l := log.NewNopLogger()
+			states := singleFromNormal(&state.State{
+				State:  eval.Alerting,
+				Labels: data.Labels{"statelabel": "labelvalue"},
+			})
+
+			res := statesToStreams(rule, states, nil, l)
+
+			entry := requireSingleEntry(t, res)
+			require.Contains(t, entry.InstanceLabels, "statelabel")
+		})
+
+		t.Run("does not include labels other than instance labels in log line", func(t *testing.T) {
+			rule := createTestRule()
+			l := log.NewNopLogger()
+			states := singleFromNormal(&state.State{
+				State: eval.Alerting,
+				Labels: data.Labels{
+					"statelabel": "labelvalue",
+					"labeltwo":   "labelvalue",
+					"labelthree": "labelvalue",
+				},
+			})
+
+			res := statesToStreams(rule, states, nil, l)
+
+			entry := requireSingleEntry(t, res)
+			require.Len(t, entry.InstanceLabels, 3)
+		})
+
 		t.Run("serializes values when regular", func(t *testing.T) {
 			rule := createTestRule()
 			l := log.NewNopLogger()