Alerting: Move rule UID from Loki stream labels into log lines (#70637)

Move rule uid into log line to reduce cardinality

pkg/services/ngalert/state/historian/loki.go

@@ -151,15 +151,6 @@ func buildSelectors(query models.HistoryQuery) ([]Selector, error) {
 	}
 	selectors[1] = selector
 
-	// Set the optional special selector rule_id
-	if query.RuleUID != "" {
-		rsel, err := NewSelector(RuleUIDLabel, "=", query.RuleUID)
-		if err != nil {
-			return nil, err
-		}
-		selectors = append(selectors, rsel)
-	}
-
 	return selectors, nil
 }
 
@@ -245,7 +236,6 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
 	// System-defined labels take precedence over user-defined external labels.
 	labels[StateHistoryLabelKey] = StateHistoryLabelValue
 	labels[OrgIDLabel] = fmt.Sprint(rule.OrgID)
-	labels[RuleUIDLabel] = fmt.Sprint(rule.UID)
 	labels[GroupLabel] = fmt.Sprint(rule.Group)
 	labels[FolderUIDLabel] = fmt.Sprint(rule.NamespaceUID)
 
@@ -265,6 +255,7 @@ func statesToStream(rule history_model.RuleMeta, states []state.StateTransition,
 			DashboardUID:   rule.DashboardUID,
 			PanelID:        rule.PanelID,
 			Fingerprint:    labelFingerprint(sanitizedLabels),
+			RuleUID:        rule.UID,
 			InstanceLabels: sanitizedLabels,
 		}
 		if state.State.State == eval.Error {
@@ -309,6 +300,7 @@ type lokiEntry struct {
 	DashboardUID  string           `json:"dashboardUID"`
 	PanelID       int64            `json:"panelID"`
 	Fingerprint   string           `json:"fingerprint"`
+	RuleUID       string           `json:"ruleUID"`
 	// InstanceLabels is exactly the set of labels associated with the alert instance in Alertmanager.
 	// These should not be conflated with labels associated with log streams.
 	InstanceLabels map[string]string `json:"labels"`
@@ -378,6 +370,14 @@ func buildLogQuery(query models.HistoryQuery) (string, error) {
 
 	logQL := selectorString(selectors)
 
+	if queryHasLogFilters(query) {
+		logQL = fmt.Sprintf("%s | json", logQL)
+	}
+
+	if query.RuleUID != "" {
+		logQL = fmt.Sprintf("%s | ruleUID=%q", logQL, query.RuleUID)
+	}
+
 	labelFilters := ""
 	labelKeys := make([]string, 0, len(query.Labels))
 	for k := range query.Labels {
@@ -388,10 +388,11 @@ func buildLogQuery(query models.HistoryQuery) (string, error) {
 	for _, k := range labelKeys {
 		labelFilters += fmt.Sprintf(" | labels_%s=%q", k, query.Labels[k])
 	}
-
-	if labelFilters != "" {
-		logQL = fmt.Sprintf("%s | json%s", logQL, labelFilters)
-	}
+	logQL += labelFilters
 
 	return logQL, nil
 }
+
+func queryHasLogFilters(query models.HistoryQuery) bool {
+	return query.RuleUID != "" || len(query.Labels) > 0
+}

pkg/services/ngalert/state/historian/loki_test.go

@@ -72,7 +72,6 @@ func TestRemoteLokiBackend(t *testing.T) {
 				"folderUID":          rule.NamespaceUID,
 				"group":              rule.Group,
 				"orgID":              fmt.Sprint(rule.OrgID),
-				"ruleUID":            rule.UID,
 			}
 			require.Equal(t, exp, res.Stream)
 		})
@@ -90,6 +89,20 @@ func TestRemoteLokiBackend(t *testing.T) {
 			require.NotContains(t, res.Stream, "__private__")
 		})
 
+		t.Run("includes ruleUID in log line", func(t *testing.T) {
+			rule := createTestRule()
+			l := log.NewNopLogger()
+			states := singleFromNormal(&state.State{
+				State:  eval.Alerting,
+				Labels: data.Labels{"a": "b"},
+			})
+
+			res := statesToStream(rule, states, nil, l)
+
+			entry := requireSingleEntry(t, res)
+			require.Equal(t, rule.UID, entry.RuleUID)
+		})
+
 		t.Run("includes instance labels in log line", func(t *testing.T) {
 			rule := createTestRule()
 			l := log.NewNopLogger()
@@ -210,12 +223,19 @@ func TestRemoteLokiBackend(t *testing.T) {
 				exp:   `{orgID="0",from="state-history"}`,
 			},
 			{
-				name: "adds stream label filter for ruleUID and orgID",
+				name: "adds stream label filter for orgID",
+				query: models.HistoryQuery{
+					OrgID: 123,
+				},
+				exp: `{orgID="123",from="state-history"}`,
+			},
+			{
+				name: "filters ruleUID in log line",
 				query: models.HistoryQuery{
-					RuleUID: "rule-uid",
 					OrgID:   123,
+					RuleUID: "rule-uid",
 				},
-				exp: `{orgID="123",from="state-history",ruleUID="rule-uid"}`,
+				exp: `{orgID="123",from="state-history"} | json | ruleUID="rule-uid"`,
 			},
 			{
 				name: "filters instance labels in log line",
@@ -228,6 +248,17 @@ func TestRemoteLokiBackend(t *testing.T) {
 				},
 				exp: `{orgID="123",from="state-history"} | json | labels_customlabel="customvalue" | labels_labeltwo="labelvaluetwo"`,
 			},
+			{
+				name: "filters both instance labels + ruleUID",
+				query: models.HistoryQuery{
+					OrgID:   123,
+					RuleUID: "rule-uid",
+					Labels: map[string]string{
+						"customlabel": "customvalue",
+					},
+				},
+				exp: `{orgID="123",from="state-history"} | json | ruleUID="rule-uid" | labels_customlabel="customvalue"`,
+			},
 		}
 
 		for _, tc := range cases {