Alerting: Add Ref ID to DatasourceNoData and DatasourceError alerts (#42630)

pkg/services/ngalert/eval/eval.go

@@ -8,6 +8,7 @@ import (
 	"runtime/debug"
 	"sort"
 	"strconv"
+	"strings"
 	"time"
 
 	"github.com/grafana/grafana/pkg/expr/classic"
@@ -273,6 +274,48 @@ func executeQueriesAndExpressions(ctx AlertExecCtx, data []models.AlertQuery, no
 	return exprService.TransformData(ctx.Ctx, queryDataReq)
 }
 
+// datasourceUIDsToRefIDs returns a sorted slice of Ref IDs for each Datasource UID.
+//
+// If refIDsToDatasourceUIDs is nil then this function also returns nil. Likewise,
+// if it is an empty map then it too returns an empty map.
+//
+// For example, given the following:
+//
+//		map[string]string{
+//			"ref1": "datasource1",
+//			"ref2": "datasource1",
+//			"ref3": "datasource2",
+//		}
+//
+// we would expect:
+//
+//  	map[string][]string{
+// 			"datasource1": []string{"ref1", "ref2"},
+//			"datasource2": []string{"ref3"},
+//		}
+func datasourceUIDsToRefIDs(refIDsToDatasourceUIDs map[string]string) map[string][]string {
+	if refIDsToDatasourceUIDs == nil {
+		return nil
+	}
+
+	// The ref IDs must be sorted. However, instead of sorting them once
+	// for each Datasource UID we can append them all to a slice and then
+	// sort them once
+	refIDs := make([]string, 0, len(refIDsToDatasourceUIDs))
+	for refID := range refIDsToDatasourceUIDs {
+		refIDs = append(refIDs, refID)
+	}
+	sort.Strings(refIDs)
+
+	result := make(map[string][]string)
+	for _, refID := range refIDs {
+		datasourceUID := refIDsToDatasourceUIDs[refID]
+		result[datasourceUID] = append(result[datasourceUID], refID)
+	}
+
+	return result
+}
+
 // evaluateExecutionResult takes the ExecutionResult which includes data.Frames returned
 // from SSE (Server Side Expressions). It will create Results (slice of Result) with a State
 // extracted from each Frame.
@@ -317,12 +360,12 @@ func evaluateExecutionResult(execResults ExecutionResults, ts time.Time) Results
 	}
 
 	if len(execResults.NoData) > 0 {
-		m := make(map[string]struct{})
-		for _, datasourceUID := range execResults.NoData {
-			if _, ok := m[datasourceUID]; !ok {
-				appendNoData(data.Labels{"datasource_uid": datasourceUID})
-				m[datasourceUID] = struct{}{}
-			}
+		noData := datasourceUIDsToRefIDs(execResults.NoData)
+		for datasourceUID, refIDs := range noData {
+			appendNoData(data.Labels{
+				"datasource_uid": datasourceUID,
+				"ref_id":         strings.Join(refIDs, ","),
+			})
 		}
 		return evalResults
 	}

pkg/services/ngalert/eval/eval_test.go

@@ -304,11 +304,11 @@ func TestEvaluateExecutionResultsNoData(t *testing.T) {
 		}
 		v := evaluateExecutionResult(results, time.Time{})
 		require.Len(t, v, 1)
-		require.Equal(t, data.Labels{"datasource_uid": "1"}, v[0].Instance)
+		require.Equal(t, data.Labels{"datasource_uid": "1", "ref_id": "A"}, v[0].Instance)
 		require.Equal(t, NoData, v[0].State)
 	})
 
-	t.Run("no data for Ref IDs will produce NoData result for each data source", func(t *testing.T) {
+	t.Run("no data for Ref IDs will produce NoData result for each Ref ID", func(t *testing.T) {
 		results := ExecutionResults{
 			NoData: map[string]string{
 				"A": "1",
@@ -318,16 +318,25 @@ func TestEvaluateExecutionResultsNoData(t *testing.T) {
 		}
 		v := evaluateExecutionResult(results, time.Time{})
 		require.Len(t, v, 2)
-		require.Equal(t, NoData, v[0].State)
-		require.Equal(t, NoData, v[1].State)
 
 		datasourceUIDs := make([]string, 0, len(v))
+		refIDs := make([]string, 0, len(v))
+
 		for _, next := range v {
+			require.Equal(t, NoData, next.State)
+
 			datasourceUID, ok := next.Instance["datasource_uid"]
 			require.True(t, ok)
 			require.NotEqual(t, "", datasourceUID)
 			datasourceUIDs = append(datasourceUIDs, datasourceUID)
+
+			refID, ok := next.Instance["ref_id"]
+			require.True(t, ok)
+			require.NotEqual(t, "", refID)
+			refIDs = append(refIDs, refID)
 		}
+
 		require.ElementsMatch(t, []string{"1", "2"}, datasourceUIDs)
+		require.ElementsMatch(t, []string{"A,B", "C"}, refIDs)
 	})
 }

pkg/services/ngalert/state/manager_test.go

@@ -1247,6 +1247,7 @@ func TestProcessEvalResults(t *testing.T) {
 						"label":                        "test",
 						"instance_label":               "test",
 						"datasource_uid":               "datasource_uid_1",
+						"ref_id":                       "A",
 					},
 					State: eval.Error,
 					Error: expr.QueryError{

pkg/services/ngalert/state/state.go

@@ -111,6 +111,7 @@ func (a *State) resultError(alertRule *ngModels.AlertRule, result eval.Result) {
 		if errors.As(a.Error, &queryError) {
 			for _, next := range alertRule.Data {
 				if next.RefID == queryError.RefID {
+					a.Labels["ref_id"] = next.RefID
 					a.Labels["datasource_uid"] = next.DatasourceUID
 					break
 				}