diff --git a/pkg/services/ngalert/api/api_prometheus.go b/pkg/services/ngalert/api/api_prometheus.go index 7568e11dfe4..dda5403b6e1 100644 --- a/pkg/services/ngalert/api/api_prometheus.go +++ b/pkg/services/ngalert/api/api_prometheus.go @@ -51,7 +51,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res DiscoveryBase: apimodels.DiscoveryBase{ Status: "success", }, - Data: apimodels.RuleDiscovery{}, + Data: apimodels.RuleDiscovery{ + RuleGroups: []*apimodels.RuleGroup{}, + }, } ruleGroupQuery := ngmodels.ListOrgRuleGroupsQuery{ @@ -99,7 +101,7 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res newRule := apimodels.Rule{ Name: rule.Title, Labels: rule.Labels, - Health: "ok", // TODO: update this in the future when error and noData states are being evaluated and set + Health: "ok", Type: apiv1.RuleTypeAlerting, LastEvaluation: time.Time{}, } @@ -131,9 +133,9 @@ func (srv PrometheusSrv) RouteGetRuleStatuses(c *models.ReqContext) response.Res case eval.Alerting: alertingRule.State = "firing" case eval.Error: - // handle Error case based on configuration in alertRule + newRule.Health = "error" case eval.NoData: - // handle NoData case based on configuration in alertRule + newRule.Health = "nodata" } alertingRule.Alerts = append(alertingRule.Alerts, alert) } diff --git a/pkg/services/ngalert/eval/eval.go b/pkg/services/ngalert/eval/eval.go index af134708edd..375ae90ec38 100644 --- a/pkg/services/ngalert/eval/eval.go +++ b/pkg/services/ngalert/eval/eval.go @@ -78,7 +78,7 @@ const ( // Pending is the eval state for an alert instance condition // that evaluated to true (Alerting) but has not yet met - // the For duration defined in AlertRule + // the For duration defined in AlertRule. Pending // NoData is the eval state for an alert rule condition diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index 4e7183a98f2..cf6cca1e604 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -307,6 +307,7 @@ func (sch *schedule) Ticker(grafanaCtx context.Context, stateTracker *state.Stat case <-grafanaCtx.Done(): err := dispatcherGroup.Wait() sch.saveAlertStates(stateTracker.GetAll()) + stateTracker.Close() return err } } diff --git a/pkg/services/ngalert/state/state_tracker.go b/pkg/services/ngalert/state/state_tracker.go index a0e63ef090a..8d37e1bf1a5 100644 --- a/pkg/services/ngalert/state/state_tracker.go +++ b/pkg/services/ngalert/state/state_tracker.go @@ -56,6 +56,10 @@ func NewStateTracker(logger log.Logger) *StateTracker { return tracker } +func (st *StateTracker) Close() { + st.quit <- struct{}{} +} + func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState { st.cache.mtxStates.Lock() defer st.cache.mtxStates.Unlock() @@ -76,13 +80,8 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R annotations = alertRule.Annotations } - newResults := []StateEvaluation{ - { - EvaluationTime: result.EvaluatedAt, - EvaluationState: result.State, - }, - } - + // If the first result we get is alerting, set StartsAt to EvaluatedAt because we + // do not have data for determining StartsAt otherwise st.Log.Debug("adding new alert state cache entry", "cacheId", id, "state", result.State.String(), "evaluatedAt", result.EvaluatedAt.String()) newState := AlertState{ AlertRuleUID: alertRule.UID, @@ -90,7 +89,6 @@ func (st *StateTracker) getOrCreate(alertRule *ngModels.AlertRule, result eval.R CacheId: id, Labels: lbs, State: result.State, - Results: newResults, Annotations: annotations, EvaluationDuration: evaluationDuration, } @@ -136,57 +134,111 @@ func (st *StateTracker) ProcessEvalResults(alertRule *ngModels.AlertRule, result //Set the current state based on evaluation results func (st *StateTracker) setNextState(alertRule *ngModels.AlertRule, result eval.Result, evaluationDuration time.Duration) AlertState { currentState := st.getOrCreate(alertRule, result, evaluationDuration) + + currentState.LastEvaluationTime = result.EvaluatedAt + currentState.EvaluationDuration = evaluationDuration + currentState.Results = append(currentState.Results, StateEvaluation{ + EvaluationTime: result.EvaluatedAt, + EvaluationState: result.State, + }) + st.Log.Debug("setting alert state", "uid", alertRule.UID) - switch { - case currentState.State == result.State: - st.Log.Debug("no state transition", "cacheId", currentState.CacheId, "state", currentState.State.String()) - currentState.LastEvaluationTime = result.EvaluatedAt - currentState.EvaluationDuration = evaluationDuration - currentState.Results = append(currentState.Results, StateEvaluation{ - EvaluationTime: result.EvaluatedAt, - EvaluationState: result.State, - }) - if currentState.State == eval.Alerting { - //TODO: Move me and unify me with the top level constant - // 10 seconds is the base evaluation interval. We use 2 times that interval to make sure we send an alert - // that would expire after at least 2 iterations and avoid flapping. - resendDelay := 10 * 2 * time.Second - if alertRule.For > resendDelay { - resendDelay = alertRule.For * 2 - } - currentState.EndsAt = result.EvaluatedAt.Add(resendDelay) + switch result.State { + case eval.Normal: + currentState = resultNormal(currentState, result) + case eval.Alerting: + currentState = currentState.resultAlerting(alertRule, result) + case eval.Error: + currentState = currentState.resultError(alertRule, result) + case eval.NoData: + currentState = currentState.resultNoData(alertRule, result) + case eval.Pending: // we do not emit results with this state + } + + st.set(currentState) + return currentState +} + +func resultNormal(alertState AlertState, result eval.Result) AlertState { + newState := alertState + if alertState.State != eval.Normal { + newState.EndsAt = result.EvaluatedAt + } + newState.State = eval.Normal + return newState +} + +func (a AlertState) resultAlerting(alertRule *ngModels.AlertRule, result eval.Result) AlertState { + switch a.State { + case eval.Alerting: + if !(alertRule.For > 0) { + // If there is not For set, we will set EndsAt to be twice the evaluation interval + // to avoid flapping with every evaluation + a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) + return a + } + a.EndsAt = result.EvaluatedAt.Add(alertRule.For) + case eval.Pending: + if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For { + a.State = eval.Alerting + a.StartsAt = result.EvaluatedAt + a.EndsAt = result.EvaluatedAt.Add(alertRule.For) } - st.set(currentState) - return currentState - case currentState.State == eval.Normal && result.State == eval.Alerting: - st.Log.Debug("state transition from normal to alerting", "cacheId", currentState.CacheId) - currentState.State = eval.Alerting - currentState.LastEvaluationTime = result.EvaluatedAt - currentState.StartsAt = result.EvaluatedAt - currentState.EndsAt = result.EvaluatedAt.Add(alertRule.For * time.Second) - currentState.EvaluationDuration = evaluationDuration - currentState.Results = append(currentState.Results, StateEvaluation{ - EvaluationTime: result.EvaluatedAt, - EvaluationState: result.State, - }) - currentState.Annotations["alerting"] = result.EvaluatedAt.String() - st.set(currentState) - return currentState - case currentState.State == eval.Alerting && result.State == eval.Normal: - st.Log.Debug("state transition from alerting to normal", "cacheId", currentState.CacheId) - currentState.State = eval.Normal - currentState.LastEvaluationTime = result.EvaluatedAt - currentState.EndsAt = result.EvaluatedAt - currentState.EvaluationDuration = evaluationDuration - currentState.Results = append(currentState.Results, StateEvaluation{ - EvaluationTime: result.EvaluatedAt, - EvaluationState: result.State, - }) - st.set(currentState) - return currentState default: - return currentState + a.StartsAt = result.EvaluatedAt + if !(alertRule.For > 0) { + a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) + a.State = eval.Alerting + } else { + a.EndsAt = result.EvaluatedAt.Add(alertRule.For) + if result.EvaluatedAt.Sub(a.StartsAt) > alertRule.For { + a.State = eval.Alerting + } else { + a.State = eval.Pending + } + } + } + return a +} + +func (a AlertState) resultError(alertRule *ngModels.AlertRule, result eval.Result) AlertState { + if a.StartsAt.IsZero() { + a.StartsAt = result.EvaluatedAt + } + if !(alertRule.For > 0) { + a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) + } else { + a.EndsAt = result.EvaluatedAt.Add(alertRule.For) + } + + switch alertRule.ExecErrState { + case ngModels.AlertingErrState: + a.State = eval.Alerting + case ngModels.KeepLastStateErrState: + } + return a +} + +func (a AlertState) resultNoData(alertRule *ngModels.AlertRule, result eval.Result) AlertState { + if a.StartsAt.IsZero() { + a.StartsAt = result.EvaluatedAt + } + if !(alertRule.For > 0) { + a.EndsAt = result.EvaluatedAt.Add(time.Duration(alertRule.IntervalSeconds*2) * time.Second) + } else { + a.EndsAt = result.EvaluatedAt.Add(alertRule.For) + } + + switch alertRule.NoDataState { + case ngModels.Alerting: + a.State = eval.Alerting + case ngModels.NoData: + a.State = eval.NoData + case ngModels.KeepLastState: + case ngModels.OK: + a.State = eval.Normal } + return a } func (st *StateTracker) GetAll() []AlertState { diff --git a/pkg/services/ngalert/tests/state_tracker_test.go b/pkg/services/ngalert/tests/state_tracker_test.go index 656e689323a..ac34b116fdd 100644 --- a/pkg/services/ngalert/tests/state_tracker_test.go +++ b/pkg/services/ngalert/tests/state_tracker_test.go @@ -1,7 +1,6 @@ package tests import ( - "fmt" "testing" "time" @@ -16,276 +15,748 @@ import ( ) func TestProcessEvalResults(t *testing.T) { - t.Skip() evaluationTime, err := time.Parse("2006-01-02", "2021-03-25") if err != nil { t.Fatalf("error parsing date format: %s", err.Error()) } - cacheId := "map[__alert_rule_namespace_uid__:test_namespace __alert_rule_uid__:test_uid alertname:test_title label1:value1 label2:value2 rule_label:rule_value]" + evaluationDuration := 10 * time.Millisecond - ruleLabels := map[string]string{ - "rule_label": "rule_value", - } - alertRule := models.AlertRule{ - ID: 1, - OrgID: 123, - Title: "test_title", - Condition: "A", - UID: "test_uid", - NamespaceUID: "test_namespace", - For: 10 * time.Second, - Labels: ruleLabels, - } - processingTime := 10 * time.Millisecond - expectedLabels := data.Labels{ - "label1": "value1", - "label2": "value2", - "rule_label": "rule_value", - "__alert_rule_uid__": "test_uid", - "__alert_rule_namespace_uid__": "test_namespace", - "alertname": "test_title", - } testCases := []struct { - desc string - uid string - evalResults eval.Results - expectedState eval.State - expectedReturnedStateCount int - expectedResultCount int - expectedCacheEntries []state.AlertState + desc string + alertRule *models.AlertRule + evalResults []eval.Results + expectedStates map[string]state.AlertState }{ { - desc: "given a single evaluation result", - uid: "test_uid", - evalResults: eval.Results{ - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Normal, - EvaluatedAt: evaluationTime, - }, - }, - expectedState: eval.Normal, - expectedReturnedStateCount: 0, - expectedResultCount: 1, - expectedCacheEntries: []state.AlertState{ + desc: "a cache entry is correctly created", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + }, + evalResults: []eval.Results{ { - AlertRuleUID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, Results: []state.StateEvaluation{ - {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, }, - StartsAt: time.Time{}, - EndsAt: time.Time{}, LastEvaluationTime: evaluationTime, + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, }, }, }, { - desc: "given a state change from normal to alerting for a single entity", - uid: "test_uid", - evalResults: eval.Results{ - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Normal, - EvaluatedAt: evaluationTime, - }, - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Alerting, - EvaluatedAt: evaluationTime.Add(1 * time.Minute), - }, - }, - expectedState: eval.Alerting, - expectedReturnedStateCount: 1, - expectedResultCount: 2, - expectedCacheEntries: []state.AlertState{ + desc: "two results create two correct cache entries", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + }, + evalResults: []eval.Results{ { - AlertRuleUID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Alerting, + eval.Result{ + Instance: data.Labels{"instance_label_1": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + eval.Result{ + Instance: data.Labels{"instance_label_2": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime, + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]": { + AlertRuleUID: "test_alert_rule_uid", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_1:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid", + "alertname": "test_title", + "label": "test", + "instance_label_1": "test", + }, + State: eval.Normal, Results: []state.StateEvaluation{ - {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, - {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting}, + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, }, - StartsAt: evaluationTime.Add(1 * time.Minute), - EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute), - LastEvaluationTime: evaluationTime.Add(1 * time.Minute), + LastEvaluationTime: evaluationTime, + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]": { + AlertRuleUID: "test_alert_rule_uid", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid alertname:test_title instance_label_2:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid", + "alertname": "test_title", + "label": "test", + "instance_label_2": "test", + }, + State: eval.Alerting, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Alerting, + }, + }, + StartsAt: evaluationTime, + EndsAt: evaluationTime.Add(20 * time.Second), + LastEvaluationTime: evaluationTime, + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, }, }, }, { - desc: "given a state change from alerting to normal for a single entity", - uid: "test_uid", - evalResults: eval.Results{ - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Alerting, - EvaluatedAt: evaluationTime, - }, - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Normal, - EvaluatedAt: evaluationTime.Add(1 * time.Minute), - }, - }, - expectedState: eval.Normal, - expectedReturnedStateCount: 1, - expectedResultCount: 2, - expectedCacheEntries: []state.AlertState{ + desc: "state is maintained", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_1", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, { - AlertRuleUID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime.Add(1 * time.Minute), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_1", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_1 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_1", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, Results: []state.StateEvaluation{ - {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, - {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal}, + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(1 * time.Minute), + EvaluationState: eval.Normal, + }, }, - StartsAt: evaluationTime, - EndsAt: evaluationTime.Add(1 * time.Minute), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, }, }, }, { - desc: "given a constant alerting state for a single entity", - uid: "test_uid", - evalResults: eval.Results{ - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Alerting, - EvaluatedAt: evaluationTime, - }, - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Alerting, - EvaluatedAt: evaluationTime.Add(1 * time.Minute), - }, - }, - expectedState: eval.Alerting, - expectedReturnedStateCount: 0, - expectedResultCount: 2, - expectedCacheEntries: []state.AlertState{ + desc: "normal -> alerting transition when For is unset", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + }, + evalResults: []eval.Results{ { - AlertRuleUID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Alerting, + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(1 * time.Minute), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Alerting, Results: []state.StateEvaluation{ - {EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, - {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Alerting}, + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(1 * time.Minute), + EvaluationState: eval.Alerting, + }, }, - StartsAt: evaluationTime, - EndsAt: evaluationTime.Add(alertRule.For * time.Second).Add(1 * time.Minute), + StartsAt: evaluationTime.Add(1 * time.Minute), + EndsAt: evaluationTime.Add(1 * time.Minute).Add(time.Duration(20) * time.Second), LastEvaluationTime: evaluationTime.Add(1 * time.Minute), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, }, }, }, { - desc: "given a constant normal state for a single entity", - uid: "test_uid", - evalResults: eval.Results{ - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Normal, - EvaluatedAt: evaluationTime, - }, - eval.Result{ - Instance: data.Labels{"label1": "value1", "label2": "value2"}, - State: eval.Normal, - EvaluatedAt: evaluationTime.Add(1 * time.Minute), - }, - }, - expectedState: eval.Normal, - expectedReturnedStateCount: 0, - expectedResultCount: 2, - expectedCacheEntries: []state.AlertState{ + desc: "normal -> alerting when For is set", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 1 * time.Minute, + }, + evalResults: []eval.Results{ { - AlertRuleUID: "test_uid", - OrgID: 123, - CacheId: cacheId, - Labels: expectedLabels, - State: eval.Normal, + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(80 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Alerting, Results: []state.StateEvaluation{ - {EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, - {EvaluationTime: evaluationTime.Add(1 * time.Minute), EvaluationState: eval.Normal}, + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.Alerting, + }, + { + EvaluationTime: evaluationTime.Add(80 * time.Second), + EvaluationState: eval.Alerting, + }, + }, + StartsAt: evaluationTime.Add(80 * time.Second), + EndsAt: evaluationTime.Add(80 * time.Second).Add(1 * time.Minute), + LastEvaluationTime: evaluationTime.Add(80 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> pending when For is set but not exceeded", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 1 * time.Minute, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Alerting, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Pending, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.Alerting, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> alerting when result is NoData and NoDataState is alerting", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.Alerting, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Alerting, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> nodata when result is NoData and NoDataState is nodata", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.NoData, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.NoData, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> normal when result is NoData and NoDataState is ok", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + NoDataState: models.OK, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), }, - StartsAt: time.Time{}, - EndsAt: time.Time{}, - LastEvaluationTime: evaluationTime.Add(1 * time.Minute), + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(20 * time.Second), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "EndsAt set correctly. normal -> alerting when result is NoData and NoDataState is alerting and For is set and For is breached", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 1 * time.Minute, + NoDataState: models.Alerting, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Alerting, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> normal when result is NoData and NoDataState is KeepLastState", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 1 * time.Minute, + NoDataState: models.KeepLastState, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, + }, + }, + }, + { + desc: "normal -> normal when result is NoData and NoDataState is KeepLastState", + alertRule: &models.AlertRule{ + OrgID: 1, + Title: "test_title", + UID: "test_alert_rule_uid_2", + NamespaceUID: "test_namespace_uid", + Annotations: map[string]string{"annotation": "test"}, + Labels: map[string]string{"label": "test"}, + IntervalSeconds: 10, + For: 1 * time.Minute, + NoDataState: models.KeepLastState, + }, + evalResults: []eval.Results{ + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.Normal, + EvaluatedAt: evaluationTime, + }, + }, + { + eval.Result{ + Instance: data.Labels{"instance_label": "test"}, + State: eval.NoData, + EvaluatedAt: evaluationTime.Add(10 * time.Second), + }, + }, + }, + expectedStates: map[string]state.AlertState{ + "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]": { + AlertRuleUID: "test_alert_rule_uid_2", + OrgID: 1, + CacheId: "map[__alert_rule_namespace_uid__:test_namespace_uid __alert_rule_uid__:test_alert_rule_uid_2 alertname:test_title instance_label:test label:test]", + Labels: data.Labels{ + "__alert_rule_namespace_uid__": "test_namespace_uid", + "__alert_rule_uid__": "test_alert_rule_uid_2", + "alertname": "test_title", + "label": "test", + "instance_label": "test", + }, + State: eval.Normal, + Results: []state.StateEvaluation{ + { + EvaluationTime: evaluationTime, + EvaluationState: eval.Normal, + }, + { + EvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationState: eval.NoData, + }, + }, + StartsAt: evaluationTime.Add(10 * time.Second), + EndsAt: evaluationTime.Add(10 * time.Second).Add(1 * time.Minute), + LastEvaluationTime: evaluationTime.Add(10 * time.Second), + EvaluationDuration: evaluationDuration, + Annotations: map[string]string{"annotation": "test"}, }, }, }, } for _, tc := range testCases { - t.Run("all fields for a cache entry are set correctly", func(t *testing.T) { - st := state.NewStateTracker(log.New("test_state_tracker")) - _ = st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) - for _, entry := range tc.expectedCacheEntries { - if !entry.Equals(st.Get(entry.CacheId)) { - t.Log(tc.desc) - printEntryDiff(entry, st.Get(entry.CacheId), t) - } - assert.True(t, entry.Equals(st.Get(entry.CacheId))) + st := state.NewStateTracker(log.New("test_state_tracker")) + t.Run(tc.desc, func(t *testing.T) { + for _, res := range tc.evalResults { + _ = st.ProcessEvalResults(tc.alertRule, res, evaluationDuration) + } + for id, s := range tc.expectedStates { + assert.Equal(t, s, st.Get(id)) } }) - - t.Run("the expected number of entries are added to the cache", func(t *testing.T) { - st := state.NewStateTracker(log.New("test_state_tracker")) - st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) - assert.Equal(t, len(tc.expectedCacheEntries), len(st.GetAll())) - }) - - //This test, as configured, does not quite represent the behavior of the system. - //It is expected that each batch of evaluation results will have only one result - //for a unique set of labels. - t.Run("the expected number of states are returned to the caller", func(t *testing.T) { - st := state.NewStateTracker(log.New("test_state_tracker")) - results := st.ProcessEvalResults(&alertRule, tc.evalResults, processingTime) - assert.Equal(t, len(tc.evalResults), len(results)) - }) - } -} - -func printEntryDiff(a, b state.AlertState, t *testing.T) { - if a.AlertRuleUID != b.AlertRuleUID { - t.Log(fmt.Sprintf("%v \t %v\n", a.AlertRuleUID, b.AlertRuleUID)) - } - if a.OrgID != b.OrgID { - t.Log(fmt.Sprintf("%v \t %v\n", a.OrgID, b.OrgID)) - } - if a.CacheId != b.CacheId { - t.Log(fmt.Sprintf("%v \t %v\n", a.CacheId, b.CacheId)) - } - if !a.Labels.Equals(b.Labels) { - t.Log(fmt.Sprintf("%v \t %v\n", a.Labels, b.Labels)) - } - if a.StartsAt != b.StartsAt { - t.Log(fmt.Sprintf("%v \t %v\n", a.StartsAt, b.StartsAt)) - } - if a.EndsAt != b.EndsAt { - t.Log(fmt.Sprintf("%v \t %v\n", a.EndsAt, b.EndsAt)) - } - if a.LastEvaluationTime != b.LastEvaluationTime { - t.Log(fmt.Sprintf("%v \t %v\n", a.LastEvaluationTime, b.LastEvaluationTime)) - } - if len(a.Results) != len(b.Results) { - t.Log(fmt.Sprintf("a: %d b: %d", len(a.Results), len(b.Results))) - t.Log("a") - for i := 0; i < len(a.Results); i++ { - t.Log(fmt.Sprintf("%v\n", a.Results[i])) - } - t.Log("b") - for i := 0; i < len(b.Results); i++ { - t.Log(fmt.Sprintf("%v\n", b.Results[i])) - } } } diff --git a/pkg/tests/api/alerting/api_prometheus_test.go b/pkg/tests/api/alerting/api_prometheus_test.go new file mode 100644 index 00000000000..b23e8467b2e --- /dev/null +++ b/pkg/tests/api/alerting/api_prometheus_test.go @@ -0,0 +1,238 @@ +package alerting + +import ( + "bytes" + "encoding/json" + "fmt" + "io/ioutil" + "net/http" + "testing" + "time" + + "github.com/grafana/grafana/pkg/models" + apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models" + "github.com/grafana/grafana/pkg/tests/testinfra" + "github.com/prometheus/common/model" + "github.com/stretchr/testify/assert" + "github.com/stretchr/testify/require" +) + +func TestPrometheusRules(t *testing.T) { + dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ + EnableFeatureToggles: []string{"ngalert"}, + AnonymousUserRole: models.ROLE_EDITOR, + }) + store := testinfra.SetUpDatabase(t, dir) + grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store) + + // Create the namespace we'll save our alerts to. + require.NoError(t, createFolder(t, store, 0, "default")) + + interval, err := model.ParseDuration("10s") + require.NoError(t, err) + + // When we have no alerting rules, it returns an empty list. + { + promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr) + // nolint:gosec + resp, err := http.Get(promRulesURL) + require.NoError(t, err) + t.Cleanup(func() { + err := resp.Body.Close() + require.NoError(t, err) + }) + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + assert.Equal(t, 200, resp.StatusCode) + require.JSONEq(t, `{"status": "success", "data": {"groups": []}}`, string(b)) + } + + // Now, let's create some rules + { + rules := apimodels.PostableRuleGroupConfig{ + Name: "arulegroup", + Rules: []apimodels.PostableExtendedRuleNode{ + { + ApiRuleNode: &apimodels.ApiRuleNode{ + For: interval, + Labels: map[string]string{"label1": "val1"}, + Annotations: map[string]string{"annotation1": "val1"}, + }, + // this rule does not explicitly set no data and error states + // therefore it should get the default values + GrafanaManagedAlert: &apimodels.PostableGrafanaRule{ + Title: "AlwaysFiring", + Condition: "A", + Data: []ngmodels.AlertQuery{ + { + RefID: "A", + RelativeTimeRange: ngmodels.RelativeTimeRange{ + From: ngmodels.Duration(time.Duration(5) * time.Hour), + To: ngmodels.Duration(time.Duration(3) * time.Hour), + }, + Model: json.RawMessage(`{ + "datasourceUid": "-100", + "type": "math", + "expression": "2 + 3 > 1" + }`), + }, + }, + }, + }, + { + GrafanaManagedAlert: &apimodels.PostableGrafanaRule{ + Title: "AlwaysFiringButSilenced", + Condition: "A", + Data: []ngmodels.AlertQuery{ + { + RefID: "A", + RelativeTimeRange: ngmodels.RelativeTimeRange{ + From: ngmodels.Duration(time.Duration(5) * time.Hour), + To: ngmodels.Duration(time.Duration(3) * time.Hour), + }, + Model: json.RawMessage(`{ + "datasourceUid": "-100", + "type": "math", + "expression": "2 + 3 > 1" + }`), + }, + }, + NoDataState: apimodels.NoDataState(ngmodels.Alerting), + ExecErrState: apimodels.ExecutionErrorState(ngmodels.KeepLastStateErrState), + }, + }, + }, + } + buf := bytes.Buffer{} + enc := json.NewEncoder(&buf) + err := enc.Encode(&rules) + require.NoError(t, err) + + u := fmt.Sprintf("http://%s/api/ruler/grafana/api/v1/rules/default", grafanaListedAddr) + // nolint:gosec + resp, err := http.Post(u, "application/json", &buf) + require.NoError(t, err) + t.Cleanup(func() { + err := resp.Body.Close() + require.NoError(t, err) + }) + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + + assert.Equal(t, resp.StatusCode, 202) + require.JSONEq(t, `{"message":"rule group updated successfully"}`, string(b)) + } + + // Now, let's see how this looks like. + { + promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr) + // nolint:gosec + resp, err := http.Get(promRulesURL) + require.NoError(t, err) + t.Cleanup(func() { + err := resp.Body.Close() + require.NoError(t, err) + }) + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + require.JSONEq(t, ` +{ + "status": "success", + "data": { + "groups": [{ + "name": "arulegroup", + "file": "default", + "rules": [{ + "state": "inactive", + "name": "AlwaysFiring", + "query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]", + "duration": 10, + "annotations": { + "annotation1": "val1" + }, + "labels": { + "label1": "val1" + }, + "health": "ok", + "lastError": "", + "type": "alerting", + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }, { + "state": "inactive", + "name": "AlwaysFiringButSilenced", + "query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]", + "labels": null, + "health": "ok", + "lastError": "", + "type": "alerting", + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }], + "interval": 60, + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }] + } +}`, string(b)) + } + + { + promRulesURL := fmt.Sprintf("http://%s/api/prometheus/grafana/api/v1/rules", grafanaListedAddr) + // nolint:gosec + require.Eventually(t, func() bool { + resp, err := http.Get(promRulesURL) + require.NoError(t, err) + t.Cleanup(func() { + err := resp.Body.Close() + require.NoError(t, err) + }) + b, err := ioutil.ReadAll(resp.Body) + require.NoError(t, err) + require.Equal(t, 200, resp.StatusCode) + require.JSONEq(t, ` +{ + "status": "success", + "data": { + "groups": [{ + "name": "arulegroup", + "file": "default", + "rules": [{ + "state": "inactive", + "name": "AlwaysFiring", + "query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]", + "duration": 10, + "annotations": { + "annotation1": "val1" + }, + "labels": { + "label1": "val1" + }, + "health": "ok", + "lastError": "", + "type": "alerting", + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }, { + "state": "inactive", + "name": "AlwaysFiringButSilenced", + "query": "[{\"datasourceUid\":\"-100\",\"expression\":\"2 + 3 \\u003e 1\",\"intervalMs\":1000,\"maxDataPoints\":100,\"type\":\"math\"}]", + "labels": null, + "health": "ok", + "lastError": "", + "type": "alerting", + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }], + "interval": 60, + "lastEvaluation": "0001-01-01T00:00:00Z", + "evaluationTime": 0 + }] + } +}`, string(b)) + return true + }, 18*time.Second, 2*time.Second) + } +}