Alerting: Handle NoData and Error evaluation results (#33194)

* set processing time

* merge labels and set on response

* use state cache for adding alerts to rules

* minor cleanup

* add support for NoData and Error results

* rename test

* bring in changes from other PRs tha have been merged

* pr feedback

* add integration test

* close state tracker cleanup on context.Done

* fixup test

* not those annotations

pkg/services/ngalert/api/api_prometheus.go

@@ -51,7 +51,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
 		DiscoveryBase: apimodels.DiscoveryBase{
 			Status: "success",
 		},
-		Data: apimodels.RuleDiscovery{},
+		Data: apimodels.RuleDiscovery{
+			RuleGroups: []*apimodels.RuleGroup{},
+		},
 	}
 
 	ruleGroupQuery := ngmodels.ListOrgRuleGroupsQuery{
@@ -99,7 +101,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
 			newRule := apimodels.Rule{
 				Name:           rule.Title,
 				Labels:         rule.Labels,
-				Health:         "ok", // TODO: update this in the future when error and noData states are being evaluated and set
+				Health:         "ok",
 				Type:           apiv1.RuleTypeAlerting,
 				LastEvaluation: time.Time{},
 			}
@@ -131,9 +133,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res
 				case eval.Alerting:
 					alertingRule.State = "firing"
 				case eval.Error:
-					// handle Error case based on configuration in alertRule
+					newRule.Health = "error"
 				case eval.NoData:
-					// handle NoData case based on configuration in alertRule
+					newRule.Health = "nodata"
 				}
 				alertingRule.Alerts = append(alertingRule.Alerts, alert)
 			}

pkg/services/ngalert/eval/eval.go

@@ -78,7 +78,7 @@ const (
 
 	// Pending is the eval state for an alert instance condition
 	// that evaluated to true (Alerting) but has not yet met
-	// the For duration defined in AlertRule
+	// the For duration defined in AlertRule.
 	Pending
 
 	// NoData is the eval state for an alert rule condition

pkg/services/ngalert/schedule/schedule.go

@@ -307,6 +307,7 @@ func (sch *schedule) Ticker(grafanaCtx context.Context, stateTracker *state.Stat
 		case <-grafanaCtx.Done():
 			err := dispatcherGroup.Wait()
 			sch.saveAlertStates(stateTracker.GetAll())
+			stateTracker.Close()
 			return err
 		}
 	}

pkg/services/ngalert/state/state_tracker.go

@@ -56,6 +56,10 @@ func NewStateTracker(logger log.Logger) *StateTracker {
 	return tracker
 }
 
+func (st *StateTracker) Close() {
+	st.quit <- struct{}{}
+}
+
 func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState {
 	st.cache.mtxStates.Lock()
 	defer st.cache.mtxStates.Unlock()
@@ -76,13 +80,8 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R
 		annotations = alertRule.Annotations
 	}
 
-	newResults := []StateEvaluation{
-		{
-			EvaluationTime:  result.EvaluatedAt,
-			EvaluationState: result.State,
-		},
-	}
-
+	// If the first result we get is alerting, set StartsAt to EvaluatedAt because we
+	// do not have data for determining StartsAt otherwise
 	st.Log.Debug("adding new alert state cache entry", "cacheId", id, "state", result.State.String(), "evaluatedAt", result.EvaluatedAt.String())
 	newState := AlertState{
 		AlertRuleUID:       alertRule.UID,
@@ -90,7 +89,6 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R
 		CacheId:            id,
 		Labels:             lbs,
 		State:              result.State,
-		Results:            newResults,
 		Annotations:        annotations,
 		EvaluationDuration: evaluationDuration,
 	}
@@ -136,57 +134,111 @@ func (st *StateTracker) ProcessEvalResults(alertRule *ngModels.AlertRule, result
 //Set the current state based on evaluation results
 func (st *StateTracker) setNextState(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState {
 	currentState := st.getOrCreate(alertRule, result, evaluationDuration)
+
+	currentState.LastEvaluationTime = result.EvaluatedAt
+	currentState.EvaluationDuration = evaluationDuration
+	currentState.Results = append(currentState.Results, StateEvaluation{
+		EvaluationTime:  result.EvaluatedAt,
+		EvaluationState: result.State,
+	})
+
 	st.Log.Debug("setting alert state", "uid", alertRule.UID)
-	switch {
-	case currentState.State == result.State:
-		st.Log.Debug("no state transition", "cacheId", currentState.CacheId, "state", currentState.State.String())
-		currentState.LastEvaluationTime = result.EvaluatedAt
-		currentState.EvaluationDuration = evaluationDuration
-		currentState.Results = append(currentState.Results, StateEvaluation{
-			EvaluationTime:  result.EvaluatedAt,
-			EvaluationState: result.State,
-		})
-		if currentState.State == eval.Alerting {
-			//TODO: Move me and unify me with the top level constant
-			// 10 seconds is the base evaluation interval. We use 2 times that interval to make sure we send an alert
-			// that would expire after at least 2 iterations and avoid flapping.
-			resendDelay := 10 * 2 * time.Second
-			if alertRule.For > resendDelay {
-				resendDelay = alertRule.For * 2
-			}
-			currentState.EndsAt = result.EvaluatedAt.Add(resendDelay)
+	switch result.State {
+	case eval.Normal:
+		currentState = resultNormal(currentState, result)
+	case eval.Alerting:
+		currentState = currentState.resultAlerting(alertRule, result)
+	case eval.Error:
+		currentState = currentState.resultError(alertRule, result)
+	case eval.NoData:
+		currentState = currentState.resultNoData(alertRule, result)
+	case eval.Pending: // we do not emit results with this state
+	}
+
+	st.set(currentState)
+	return currentState
+}
+
+func resultNormal(alertState AlertState, result eval.Result) AlertState {
+	newState := alertState
+	if alertState.State != eval.Normal {
+		newState.EndsAt = result.EvaluatedAt
+	}
+	newState.State = eval.Normal
+	return newState
+}
+
+func (a AlertState) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
+	switch a.State {
+	case eval.Alerting:
+		if !(alertRule.For > 0) {
+			// If there is not For set, we will set EndsAt to be twice the evaluation interval
+			// to avoid flapping with every evaluation
+			a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
+			return a
+		}
+		a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
+	case eval.Pending:
+		if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
+			a.State = eval.Alerting
+			a.StartsAt = result.EvaluatedAt
+			a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
 		}
-		st.set(currentState)
-		return currentState
-	case currentState.State == eval.Normal && result.State == eval.Alerting:
-		st.Log.Debug("state transition from normal to alerting", "cacheId", currentState.CacheId)
-		currentState.State = eval.Alerting
-		currentState.LastEvaluationTime = result.EvaluatedAt
-		currentState.StartsAt = result.EvaluatedAt
-		currentState.EndsAt = result.EvaluatedAt.Add(alertRule.For * time.Second)
-		currentState.EvaluationDuration = evaluationDuration
-		currentState.Results = append(currentState.Results, StateEvaluation{
-			EvaluationTime:  result.EvaluatedAt,
-			EvaluationState: result.State,
-		})
-		currentState.Annotations["alerting"] = result.EvaluatedAt.String()
-		st.set(currentState)
-		return currentState
-	case currentState.State == eval.Alerting && result.State == eval.Normal:
-		st.Log.Debug("state transition from alerting to normal", "cacheId", currentState.CacheId)
-		currentState.State = eval.Normal
-		currentState.LastEvaluationTime = result.EvaluatedAt
-		currentState.EndsAt = result.EvaluatedAt
-		currentState.EvaluationDuration = evaluationDuration
-		currentState.Results = append(currentState.Results, StateEvaluation{
-			EvaluationTime:  result.EvaluatedAt,
-			EvaluationState: result.State,
-		})
-		st.set(currentState)
-		return currentState
 	default:
-		return currentState
+		a.StartsAt = result.EvaluatedAt
+		if !(alertRule.For > 0) {
+			a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
+			a.State = eval.Alerting
+		} else {
+			a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
+			if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For {
+				a.State = eval.Alerting
+			} else {
+				a.State = eval.Pending
+			}
+		}
+	}
+	return a
+}
+
+func (a AlertState) resultError(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
+	if a.StartsAt.IsZero() {
+		a.StartsAt = result.EvaluatedAt
+	}
+	if !(alertRule.For > 0) {
+		a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
+	} else {
+		a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
+	}
+
+	switch alertRule.ExecErrState {
+	case ngModels.AlertingErrState:
+		a.State = eval.Alerting
+	case ngModels.KeepLastStateErrState:
+	}
+	return a
+}
+
+func (a AlertState) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) AlertState {
+	if a.StartsAt.IsZero() {
+		a.StartsAt = result.EvaluatedAt
+	}
+	if !(alertRule.For > 0) {
+		a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second)
+	} else {
+		a.EndsAt = result.EvaluatedAt.Add(alertRule.For)
+	}
+
+	switch alertRule.NoDataState {
+	case ngModels.Alerting:
+		a.State = eval.Alerting
+	case ngModels.NoData:
+		a.State = eval.NoData
+	case ngModels.KeepLastState:
+	case ngModels.OK:
+		a.State = eval.Normal
 	}
+	return a
 }
 
 func (st *StateTracker) GetAll() []AlertState {

pkg/services/ngalert/tests/state_tracker_test.go

@@ -1,7 +1,6 @@
 package tests
 
 import (
-	"fmt"
 	"testing"
 	"time"
 
@@ -16,276 +15,748 @@ import (
 )
 
 func TestProcessEvalResults(t *testing.T) {
-	t.Skip()
 	evaluationTime, err := time.Parse("2006-01-02", "2021-03-25")
 	if err != nil {
 		t.Fatalf("error parsing date format: %s", err.Error())
 	}
-	cacheId := "map[__alert_rule_namespace_uid__:test_namespace __alert_rule_uid__:test_uid alertname:test_title label1:value1 label2:value2 rule_label:rule_value]"
+	evaluationDuration := 10 * time.Millisecond
 
-	ruleLabels := map[string]string{
-		"rule_label": "rule_value",
-	}
-	alertRule := models.AlertRule{
-		ID:           1,
-		OrgID:        123,
-		Title:        "test_title",
-		Condition:    "A",
-		UID:          "test_uid",
-		NamespaceUID: "test_namespace",
-		For:          10 * time.Second,
-		Labels:       ruleLabels,
-	}
-	processingTime := 10 * time.Millisecond
-	expectedLabels := data.Labels{
-		"label1":                       "value1",
-		"label2":                       "value2",
-		"rule_label":                   "rule_value",
-		"__alert_rule_uid__":           "test_uid",
-		"__alert_rule_namespace_uid__": "test_namespace",
-		"alertname":                    "test_title",
-	}
 	testCases := []struct {
-		desc                       string
-		uid                        string
-		evalResults                eval.Results
-		expectedState              eval.State
-		expectedReturnedStateCount int
-		expectedResultCount        int
-		expectedCacheEntries       []state.AlertState
+		desc           string
+		alertRule      *models.AlertRule
+		evalResults    []eval.Results
+		expectedStates map[string]state.AlertState
 	}{
 		{
-			desc: "given a single evaluation result",
-			uid:  "test_uid",
-			evalResults: eval.Results{
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Normal,
-					EvaluatedAt: evaluationTime,
-				},
-			},
-			expectedState:              eval.Normal,
-			expectedReturnedStateCount: 0,
-			expectedResultCount:        1,
-			expectedCacheEntries: []state.AlertState{
+			desc: "a cache entry is correctly created",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+			},
+			evalResults: []eval.Results{
 				{
-					AlertRuleUID: "test_uid",
-					OrgID:        123,
-					CacheId:      cacheId,
-					Labels:       expectedLabels,
-					State:        eval.Normal,
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Normal,
 					Results: []state.StateEvaluation{
-						{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
 					},
-					StartsAt:           time.Time{},
-					EndsAt:             time.Time{},
 					LastEvaluationTime: evaluationTime,
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
 				},
 			},
 		},
 		{
-			desc: "given a state change from normal to alerting for a single entity",
-			uid:  "test_uid",
-			evalResults: eval.Results{
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Normal,
-					EvaluatedAt: evaluationTime,
-				},
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Alerting,
-					EvaluatedAt: evaluationTime.Add(1 * time.Minute),
-				},
-			},
-			expectedState:              eval.Alerting,
-			expectedReturnedStateCount: 1,
-			expectedResultCount:        2,
-			expectedCacheEntries: []state.AlertState{
+			desc: "two results create two correct cache entries",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+			},
+			evalResults: []eval.Results{
 				{
-					AlertRuleUID: "test_uid",
-					OrgID:        123,
-					CacheId:      cacheId,
-					Labels:       expectedLabels,
-					State:        eval.Alerting,
+					eval.Result{
+						Instance:    data.Labels{"instance_label_1": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+					eval.Result{
+						Instance:    data.Labels{"instance_label_2": "test"},
+						State:       eval.Alerting,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label_1":             "test",
+					},
+					State: eval.Normal,
 					Results: []state.StateEvaluation{
-						{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
-						{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting},
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
 					},
-					StartsAt:           evaluationTime.Add(1 * time.Minute),
-					EndsAt:             evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute),
-					LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime,
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label_2":             "test",
+					},
+					State: eval.Alerting,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Alerting,
+						},
+					},
+					StartsAt:           evaluationTime,
+					EndsAt:             evaluationTime.Add(20 * time.Second),
+					LastEvaluationTime: evaluationTime,
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
 				},
 			},
 		},
 		{
-			desc: "given a state change from alerting to normal for a single entity",
-			uid:  "test_uid",
-			evalResults: eval.Results{
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Alerting,
-					EvaluatedAt: evaluationTime,
-				},
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Normal,
-					EvaluatedAt: evaluationTime.Add(1 * time.Minute),
-				},
-			},
-			expectedState:              eval.Normal,
-			expectedReturnedStateCount: 1,
-			expectedResultCount:        2,
-			expectedCacheEntries: []state.AlertState{
+			desc: "state is maintained",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_1",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
 				{
-					AlertRuleUID: "test_uid",
-					OrgID:        123,
-					CacheId:      cacheId,
-					Labels:       expectedLabels,
-					State:        eval.Normal,
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime.Add(1 * time.Minute),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_1",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_1",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Normal,
 					Results: []state.StateEvaluation{
-						{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
-						{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal},
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(1 * time.Minute),
+							EvaluationState: eval.Normal,
+						},
 					},
-					StartsAt:           evaluationTime,
-					EndsAt:             evaluationTime.Add(1 * time.Minute),
 					LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
 				},
 			},
 		},
 		{
-			desc: "given a constant alerting state for a single entity",
-			uid:  "test_uid",
-			evalResults: eval.Results{
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Alerting,
-					EvaluatedAt: evaluationTime,
-				},
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Alerting,
-					EvaluatedAt: evaluationTime.Add(1 * time.Minute),
-				},
-			},
-			expectedState:              eval.Alerting,
-			expectedReturnedStateCount: 0,
-			expectedResultCount:        2,
-			expectedCacheEntries: []state.AlertState{
+			desc: "normal -> alerting transition when For is unset",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+			},
+			evalResults: []eval.Results{
 				{
-					AlertRuleUID: "test_uid",
-					OrgID:        123,
-					CacheId:      cacheId,
-					Labels:       expectedLabels,
-					State:        eval.Alerting,
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Alerting,
+						EvaluatedAt: evaluationTime.Add(1 * time.Minute),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Alerting,
 					Results: []state.StateEvaluation{
-						{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
-						{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting},
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(1 * time.Minute),
+							EvaluationState: eval.Alerting,
+						},
 					},
-					StartsAt:           evaluationTime,
-					EndsAt:             evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute),
+					StartsAt:           evaluationTime.Add(1 * time.Minute),
+					EndsAt:             evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second),
 					LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
 				},
 			},
 		},
 		{
-			desc: "given a constant normal state for a single entity",
-			uid:  "test_uid",
-			evalResults: eval.Results{
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Normal,
-					EvaluatedAt: evaluationTime,
-				},
-				eval.Result{
-					Instance:    data.Labels{"label1": "value1", "label2": "value2"},
-					State:       eval.Normal,
-					EvaluatedAt: evaluationTime.Add(1 * time.Minute),
-				},
-			},
-			expectedState:              eval.Normal,
-			expectedReturnedStateCount: 0,
-			expectedResultCount:        2,
-			expectedCacheEntries: []state.AlertState{
+			desc: "normal -> alerting when For is set",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				For:             1 * time.Minute,
+			},
+			evalResults: []eval.Results{
 				{
-					AlertRuleUID: "test_uid",
-					OrgID:        123,
-					CacheId:      cacheId,
-					Labels:       expectedLabels,
-					State:        eval.Normal,
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Alerting,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Alerting,
+						EvaluatedAt: evaluationTime.Add(80 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Alerting,
 					Results: []state.StateEvaluation{
-						{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
-						{EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal},
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.Alerting,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(80 * time.Second),
+							EvaluationState: eval.Alerting,
+						},
+					},
+					StartsAt:           evaluationTime.Add(80 * time.Second),
+					EndsAt:             evaluationTime.Add(80 * time.Second).Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime.Add(80 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> pending when For is set but not exceeded",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				For:             1 * time.Minute,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Alerting,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Pending,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.Alerting,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> alerting when result is NoData and NoDataState is alerting",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				NoDataState:     models.Alerting,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Alerting,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> nodata when result is NoData and NoDataState is nodata",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				NoDataState:     models.NoData,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.NoData,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> normal when result is NoData and NoDataState is ok",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				NoDataState:     models.OK,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
 					},
-					StartsAt:           time.Time{},
-					EndsAt:             time.Time{},
-					LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Normal,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(20 * time.Second),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "EndsAt set correctly. normal -> alerting when result is NoData and NoDataState is alerting and For is set and For is breached",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				For:             1 * time.Minute,
+				NoDataState:     models.Alerting,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Alerting,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> normal when result is NoData and NoDataState is KeepLastState",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				For:             1 * time.Minute,
+				NoDataState:     models.KeepLastState,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Normal,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
+				},
+			},
+		},
+		{
+			desc: "normal -> normal when result is NoData and NoDataState is KeepLastState",
+			alertRule: &models.AlertRule{
+				OrgID:           1,
+				Title:           "test_title",
+				UID:             "test_alert_rule_uid_2",
+				NamespaceUID:    "test_namespace_uid",
+				Annotations:     map[string]string{"annotation": "test"},
+				Labels:          map[string]string{"label": "test"},
+				IntervalSeconds: 10,
+				For:             1 * time.Minute,
+				NoDataState:     models.KeepLastState,
+			},
+			evalResults: []eval.Results{
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.Normal,
+						EvaluatedAt: evaluationTime,
+					},
+				},
+				{
+					eval.Result{
+						Instance:    data.Labels{"instance_label": "test"},
+						State:       eval.NoData,
+						EvaluatedAt: evaluationTime.Add(10 * time.Second),
+					},
+				},
+			},
+			expectedStates: map[string]state.AlertState{
+				"map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": {
+					AlertRuleUID: "test_alert_rule_uid_2",
+					OrgID:        1,
+					CacheId:      "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]",
+					Labels: data.Labels{
+						"__alert_rule_namespace_uid__": "test_namespace_uid",
+						"__alert_rule_uid__":           "test_alert_rule_uid_2",
+						"alertname":                    "test_title",
+						"label":                        "test",
+						"instance_label":               "test",
+					},
+					State: eval.Normal,
+					Results: []state.StateEvaluation{
+						{
+							EvaluationTime:  evaluationTime,
+							EvaluationState: eval.Normal,
+						},
+						{
+							EvaluationTime:  evaluationTime.Add(10 * time.Second),
+							EvaluationState: eval.NoData,
+						},
+					},
+					StartsAt:           evaluationTime.Add(10 * time.Second),
+					EndsAt:             evaluationTime.Add(10 * time.Second).Add(1 * time.Minute),
+					LastEvaluationTime: evaluationTime.Add(10 * time.Second),
+					EvaluationDuration: evaluationDuration,
+					Annotations:        map[string]string{"annotation": "test"},
 				},
 			},
 		},
 	}
 
 	for _, tc := range testCases {
-		t.Run("all fields for a cache entry are set correctly", func(t *testing.T) {
-			st := state.NewStateTracker(log.New("test_state_tracker"))
-			_ = st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
-			for _, entry := range tc.expectedCacheEntries {
-				if !entry.Equals(st.Get(entry.CacheId)) {
-					t.Log(tc.desc)
-					printEntryDiff(entry, st.Get(entry.CacheId), t)
-				}
-				assert.True(t, entry.Equals(st.Get(entry.CacheId)))
+		st := state.NewStateTracker(log.New("test_state_tracker"))
+		t.Run(tc.desc, func(t *testing.T) {
+			for _, res := range tc.evalResults {
+				_ = st.ProcessEvalResults(tc.alertRule, res, evaluationDuration)
+			}
+			for id, s := range tc.expectedStates {
+				assert.Equal(t, s, st.Get(id))
 			}
 		})
-
-		t.Run("the expected number of entries are added to the cache", func(t *testing.T) {
-			st := state.NewStateTracker(log.New("test_state_tracker"))
-			st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
-			assert.Equal(t, len(tc.expectedCacheEntries), len(st.GetAll()))
-		})
-
-		//This test, as configured, does not quite represent the behavior of the system.
-		//It is expected that each batch of evaluation results will have only one result
-		//for a unique set of labels.
-		t.Run("the expected number of states are returned to the caller", func(t *testing.T) {
-			st := state.NewStateTracker(log.New("test_state_tracker"))
-			results := st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime)
-			assert.Equal(t, len(tc.evalResults), len(results))
-		})
-	}
-}
-
-func printEntryDiff(a, b state.AlertState, t *testing.T) {
-	if a.AlertRuleUID != b.AlertRuleUID {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.AlertRuleUID, b.AlertRuleUID))
-	}
-	if a.OrgID != b.OrgID {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.OrgID, b.OrgID))
-	}
-	if a.CacheId != b.CacheId {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.CacheId, b.CacheId))
-	}
-	if !a.Labels.Equals(b.Labels) {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.Labels, b.Labels))
-	}
-	if a.StartsAt != b.StartsAt {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.StartsAt, b.StartsAt))
-	}
-	if a.EndsAt != b.EndsAt {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.EndsAt, b.EndsAt))
-	}
-	if a.LastEvaluationTime != b.LastEvaluationTime {
-		t.Log(fmt.Sprintf("%v \t %v\n", a.LastEvaluationTime, b.LastEvaluationTime))
-	}
-	if len(a.Results) != len(b.Results) {
-		t.Log(fmt.Sprintf("a: %d b: %d", len(a.Results), len(b.Results)))
-		t.Log("a")
-		for i := 0; i < len(a.Results); i++ {
-			t.Log(fmt.Sprintf("%v\n", a.Results[i]))
-		}
-		t.Log("b")
-		for i := 0; i < len(b.Results); i++ {
-			t.Log(fmt.Sprintf("%v\n", b.Results[i]))
-		}
 	}
 }

pkg/tests/api/alerting/api_prometheus_test.go

@@ -0,0 +1,238 @@
+package alerting
+
+import (
+	"bytes"
+	"encoding/json"
+	"fmt"
+	"io/ioutil"
+	"net/http"
+	"testing"
+	"time"
+
+	"github.com/grafana/grafana/pkg/models"
+	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
+	"github.com/grafana/grafana/pkg/tests/testinfra"
+	"github.com/prometheus/common/model"
+	"github.com/stretchr/testify/assert"
+	"github.com/stretchr/testify/require"
+)
+
+func TestPrometheusRules(t *testing.T) {
+	dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{
+		EnableFeatureToggles: []string{"ngalert"},
+		AnonymousUserRole:    models.ROLE_EDITOR,
+	})
+	store := testinfra.SetUpDatabase(t, dir)
+	grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store)
+
+	// Create the namespace we'll save our alerts to.
+	require.NoError(t, createFolder(t, store, 0, "default"))
+
+	interval, err := model.ParseDuration("10s")
+	require.NoError(t, err)
+
+	// When we have no alerting rules, it returns an empty list.
+	{
+		promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
+		// nolint:gosec
+		resp, err := http.Get(promRulesURL)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		})
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		assert.Equal(t, 200, resp.StatusCode)
+		require.JSONEq(t, `{"status": "success", "data": {"groups": []}}`, string(b))
+	}
+
+	// Now, let's create some rules
+	{
+		rules := apimodels.PostableRuleGroupConfig{
+			Name: "arulegroup",
+			Rules: []apimodels.PostableExtendedRuleNode{
+				{
+					ApiRuleNode: &apimodels.ApiRuleNode{
+						For:         interval,
+						Labels:      map[string]string{"label1": "val1"},
+						Annotations: map[string]string{"annotation1": "val1"},
+					},
+					// this rule does not explicitly set no data and error states
+					// therefore it should get the default values
+					GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
+						Title:     "AlwaysFiring",
+						Condition: "A",
+						Data: []ngmodels.AlertQuery{
+							{
+								RefID: "A",
+								RelativeTimeRange: ngmodels.RelativeTimeRange{
+									From: ngmodels.Duration(time.Duration(5) * time.Hour),
+									To:   ngmodels.Duration(time.Duration(3) * time.Hour),
+								},
+								Model: json.RawMessage(`{
+									"datasourceUid": "-100",
+									"type": "math",
+									"expression": "2 + 3 > 1"
+									}`),
+							},
+						},
+					},
+				},
+				{
+					GrafanaManagedAlert: &apimodels.PostableGrafanaRule{
+						Title:     "AlwaysFiringButSilenced",
+						Condition: "A",
+						Data: []ngmodels.AlertQuery{
+							{
+								RefID: "A",
+								RelativeTimeRange: ngmodels.RelativeTimeRange{
+									From: ngmodels.Duration(time.Duration(5) * time.Hour),
+									To:   ngmodels.Duration(time.Duration(3) * time.Hour),
+								},
+								Model: json.RawMessage(`{
+									"datasourceUid": "-100",
+									"type": "math",
+									"expression": "2 + 3 > 1"
+									}`),
+							},
+						},
+						NoDataState:  apimodels.NoDataState(ngmodels.Alerting),
+						ExecErrState: apimodels.ExecutionErrorState(ngmodels.KeepLastStateErrState),
+					},
+				},
+			},
+		}
+		buf := bytes.Buffer{}
+		enc := json.NewEncoder(&buf)
+		err := enc.Encode(&rules)
+		require.NoError(t, err)
+
+		u := fmt.Sprintf("http://%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr)
+		// nolint:gosec
+		resp, err := http.Post(u, "application/json", &buf)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		})
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+
+		assert.Equal(t, resp.StatusCode, 202)
+		require.JSONEq(t, `{"message":"rule group updated successfully"}`, string(b))
+	}
+
+	// Now, let's see how this looks like.
+	{
+		promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
+		// nolint:gosec
+		resp, err := http.Get(promRulesURL)
+		require.NoError(t, err)
+		t.Cleanup(func() {
+			err := resp.Body.Close()
+			require.NoError(t, err)
+		})
+		b, err := ioutil.ReadAll(resp.Body)
+		require.NoError(t, err)
+		require.Equal(t, 200, resp.StatusCode)
+		require.JSONEq(t, `
+{
+	"status": "success",
+	"data": {
+		"groups": [{
+			"name": "arulegroup",
+			"file": "default",
+			"rules": [{
+				"state": "inactive",
+				"name": "AlwaysFiring",
+				"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
+				"duration": 10,
+				"annotations": {
+					"annotation1": "val1"
+				},
+				"labels": {
+					"label1": "val1"
+				},
+				"health": "ok",
+				"lastError": "",
+				"type": "alerting",
+				"lastEvaluation": "0001-01-01T00:00:00Z",
+				"evaluationTime": 0
+			}, {
+				"state": "inactive",
+				"name": "AlwaysFiringButSilenced",
+				"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
+				"labels": null,
+				"health": "ok",
+				"lastError": "",
+				"type": "alerting",
+				"lastEvaluation": "0001-01-01T00:00:00Z",
+				"evaluationTime": 0
+			}],
+			"interval": 60,
+			"lastEvaluation": "0001-01-01T00:00:00Z",
+			"evaluationTime": 0
+		}]
+	}
+}`, string(b))
+	}
+
+	{
+		promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr)
+		// nolint:gosec
+		require.Eventually(t, func() bool {
+			resp, err := http.Get(promRulesURL)
+			require.NoError(t, err)
+			t.Cleanup(func() {
+				err := resp.Body.Close()
+				require.NoError(t, err)
+			})
+			b, err := ioutil.ReadAll(resp.Body)
+			require.NoError(t, err)
+			require.Equal(t, 200, resp.StatusCode)
+			require.JSONEq(t, `
+{
+	"status": "success",
+	"data": {
+		"groups": [{
+			"name": "arulegroup",
+			"file": "default",
+			"rules": [{
+				"state": "inactive",
+				"name": "AlwaysFiring",
+				"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
+				"duration": 10,
+				"annotations": {
+					"annotation1": "val1"
+				},
+				"labels": {
+					"label1": "val1"
+				},
+				"health": "ok",
+				"lastError": "",
+				"type": "alerting",
+				"lastEvaluation": "0001-01-01T00:00:00Z",
+				"evaluationTime": 0
+			}, {
+				"state": "inactive",
+				"name": "AlwaysFiringButSilenced",
+				"query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]",
+				"labels": null,
+				"health": "ok",
+				"lastError": "",
+				"type": "alerting",
+				"lastEvaluation": "0001-01-01T00:00:00Z",
+				"evaluationTime": 0
+			}],
+			"interval": 60,
+			"lastEvaluation": "0001-01-01T00:00:00Z",
+			"evaluationTime": 0
+		}]
+	}
+}`, string(b))
+			return true
+		}, 18*time.Second, 2*time.Second)
+	}
+}