apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Alerting: Do not store series values from past evaluations in state manager for no reason (#87525)

Browse Source 
Do not store previous execution results on states
pull/87579/head
Alexander Weaver 2 years ago committed by GitHub
parent c6a0175c04
commit a6a9ab4008
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 902 additions and 1512 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 20
      pkg/services/ngalert/api/api_prometheus_test.go
  2. 15
      pkg/services/ngalert/api/testing.go
  3. 6
      pkg/services/ngalert/schedule/alert_rule_test.go
  4. 5
      pkg/services/ngalert/state/manager.go
  5. 1802
      pkg/services/ngalert/state/manager_private_test.go
  6. 482
      pkg/services/ngalert/state/manager_test.go
  7. 22
      pkg/services/ngalert/state/state.go
  8. 62
      pkg/services/ngalert/state/state_test.go

20
pkg/services/ngalert/api/api_prometheus_test.go
Unescape View File

@ -45,9 +45,7 @@ func Test_FormatValues(t *testing.T) {
name: "with no value, it renders the evaluation string",
alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{
{Condition: "A", Values: map[string]*float64{}},
},
LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{}},
},
expected: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
},
@ -55,9 +53,7 @@ func Test_FormatValues(t *testing.T) {
name: "with one value, it renders the single value",
alertState: &state.State{
LastEvaluationString: "[ var='A' metric='vector(10) + time() % 50' labels={} value=1.1 ]",
Results: []state.Evaluation{
{Condition: "A", Values: map[string]*float64{"A": &val1}},
},
LatestResult: &state.Evaluation{Condition: "A", Values: map[string]*float64{"A": &val1}},
},
expected: "1.1e+00",
},
@ -65,9 +61,7 @@ func Test_FormatValues(t *testing.T) {
name: "with two values, it renders the value based on their refID and position",
alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
},
LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2}},
},
expected: "B0: 1.1e+00, B1: 1.4e+00",
},
@ -75,9 +69,7 @@ func Test_FormatValues(t *testing.T) {
name: "with a high number of values, it renders the value based on their refID and position using a natural order",
alertState: &state.State{
LastEvaluationString: "[ var='B0' metric='vector(10) + time() % 50' labels={} value=1.1 ], [ var='B1' metric='vector(10) + time() % 50' labels={} value=1.4 ]",
Results: []state.Evaluation{
{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
},
LatestResult: &state.Evaluation{Condition: "B", Values: map[string]*float64{"B0": &val1, "B1": &val2, "B2": &val1, "B10": &val2, "B11": &val1}},
},
expected: "B0: 1.1e+00, B10: 1.4e+00, B11: 1.1e+00, B1: 1.4e+00, B2: 1.1e+00",
},
@ -246,12 +238,12 @@ func withAlertingState() forEachState {
return func(s *state.State) *state.State {
s.State = eval.Alerting
value := float64(1.1)
s.Results = append(s.Results, state.Evaluation{
s.LatestResult = &state.Evaluation{
EvaluationState: eval.Alerting,
EvaluationTime: timeNow(),
Values: map[string]*float64{"B": &value},
Condition: "B",
})
}
return s
}
}

15
pkg/services/ngalert/api/testing.go
Unescape View File

@ -83,17 +83,10 @@ func (f *fakeAlertInstanceManager) GenerateAlertInstances(orgID int64, alertRule
"instance_label": "test",
},
State: eval.Normal,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
LatestResult: &state.Evaluation{
EvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
},
LastEvaluationTime: evaluationTime.Add(1 * time.Minute),
EvaluationDuration: evaluationDuration,

6
pkg/services/ngalert/schedule/alert_rule_test.go
Unescape View File

@ -311,13 +311,13 @@ func TestRuleRoutine(t *testing.T) {
require.Len(t, states, 1)
s := states[0]
require.Equal(t, rule.UID, s.AlertRuleUID)
require.Len(t, s.Results, 1)
require.NotNil(t, s.LatestResult)
var expectedStatus = evalState
if evalState == eval.Pending {
expectedStatus = eval.Alerting
}
require.Equal(t, expectedStatus.String(), s.Results[0].EvaluationState.String())
require.Equal(t, expectedTime, s.Results[0].EvaluationTime)
require.Equal(t, expectedStatus.String(), s.LatestResult.EvaluationState.String())
require.Equal(t, expectedTime, s.LatestResult.EvaluationTime)
})
t.Run("it should save alert instances to storage", func(t *testing.T) {
// TODO rewrite when we are able to mock/fake state manager

5
pkg/services/ngalert/state/manager.go
Unescape View File

@ -358,14 +358,13 @@ func (st *Manager) setNextState(ctx context.Context, alertRule *ngModels.AlertRu
currentState.LastEvaluationTime = result.EvaluatedAt
currentState.EvaluationDuration = result.EvaluationDuration
currentState.Results = append(currentState.Results, Evaluation{
currentState.LatestResult = &Evaluation{
EvaluationTime: result.EvaluatedAt,
EvaluationState: result.State,
Values: NewEvaluationValues(result.Values),
Condition: alertRule.Condition,
})
}
currentState.LastEvaluationString = result.EvaluationString
currentState.TrimResults(alertRule)
oldState := currentState.State
oldReason := currentState.StateReason

1802
pkg/services/ngalert/state/manager_private_test.go
View File

File diff suppressed because it is too large Load Diff

482
pkg/services/ngalert/state/manager_test.go
Unescape View File

@ -53,26 +53,22 @@ func TestWarmStateCache(t *testing.T) {
expectedEntries := []*state.State{
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test1": "testValue1"},
State: eval.Normal,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Normal},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"},
ResultFingerprint: data.Fingerprint(math.MaxUint64),
}, {
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test2": "testValue2"},
State: eval.Alerting,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -80,13 +76,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(math.MaxUint64 - 1),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test3": "testValue3"},
State: eval.NoData,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test3": "testValue3"},
State: eval.NoData,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.NoData},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -94,13 +88,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(0),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test4": "testValue4"},
State: eval.Error,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test4": "testValue4"},
State: eval.Error,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Error},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -108,13 +100,11 @@ func TestWarmStateCache(t *testing.T) {
ResultFingerprint: data.Fingerprint(1),
},
{
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test5": "testValue5"},
State: eval.Pending,
Results: []state.Evaluation{
{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
},
AlertRuleUID: rule.UID,
OrgID: rule.OrgID,
Labels: data.Labels{"test5": "testValue5"},
State: eval.Pending,
LatestResult: &state.Evaluation{EvaluationTime: evaluationTime, EvaluationState: eval.Pending},
StartsAt: evaluationTime.Add(-1 * time.Minute),
EndsAt: evaluationTime.Add(1 * time.Minute),
LastEvaluationTime: evaluationTime,
@ -226,7 +216,7 @@ func TestWarmStateCache(t *testing.T) {
setCacheID(entry)
cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID)
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" {
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "LatestResult")); diff != "" {
t.Errorf("Result mismatch (-want +got):\n%s", diff)
t.FailNow()
}
@ -331,8 +321,8 @@ func TestProcessEvalResults(t *testing.T) {
ExecErrState: models.ErrorErrState,
}
newEvaluation := func(evalTime time.Time, evalState eval.State) state.Evaluation {
return state.Evaluation{
newEvaluation := func(evalTime time.Time, evalState eval.State) *state.Evaluation {
return &state.Evaluation{
EvaluationTime: evalTime,
EvaluationState: evalState,
Values: make(map[string]*float64),
@ -395,12 +385,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
@ -419,23 +407,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
},
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t1, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t1,
@ -455,13 +439,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(tn(6), eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(tn(6), eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: tn(6),
@ -482,13 +463,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -515,13 +493,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(tn(4), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -551,13 +526,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3, // Normal -> Pending, Pending -> NoData, NoData -> Pending
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(tn(5), eval.Alerting),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -584,13 +556,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -614,11 +583,8 @@ func TestProcessEvalResults(t *testing.T) {
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Alerting),
},
State: eval.Pending,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -639,13 +605,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(t2, eval.Alerting),
StartsAt: t1,
EndsAt: t1.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -666,14 +629,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -703,15 +663,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
newEvaluation(tn(5), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(tn(5), eval.NoData),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -732,13 +688,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -759,23 +712,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -797,34 +746,28 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
Labels: labels["system + rule + labels2"],
ResultFingerprint: labels2.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
},
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -848,24 +791,19 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t3, eval.Normal),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
LatestResult: newEvaluation(t3, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t3,
},
{
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + no-data"],
ResultFingerprint: noDataLabels.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -892,16 +830,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -928,14 +861,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.NoData),
newEvaluation(tn(4), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.NoData.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.NoData),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -956,14 +886,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 0,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.NoData.String(),
LatestResult: newEvaluation(t2, eval.NoData),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
@ -984,15 +911,12 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -1022,16 +946,12 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
newEvaluation(tn(5), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: eval.Error.String(),
Error: errors.New("with_state_error"),
LatestResult: newEvaluation(tn(5), eval.Error),
StartsAt: tn(5),
EndsAt: tn(5).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(5),
@ -1064,13 +984,10 @@ func TestProcessEvalResults(t *testing.T) {
"datasource_uid": "datasource_uid_1",
"ref_id": "A",
}),
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: expr.MakeQueryError("A", "datasource_uid_1", errors.New("this is an error")),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2.Add(state.ResendDelay * 4),
LastEvaluationTime: t2,
@ -1099,16 +1016,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
newEvaluation(tn(4), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: t3,
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -1135,14 +1047,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
Results: []state.Evaluation{
newEvaluation(t3, eval.Error),
newEvaluation(tn(4), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Alerting,
StateReason: models.ConcatReasons(eval.Error.String(), models.StateReasonKeepLast),
LatestResult: newEvaluation(tn(4), eval.Error),
StartsAt: tn(4),
EndsAt: tn(4).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(4),
@ -1163,14 +1072,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 1,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t2,
@ -1191,14 +1097,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 2,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Normal,
StateReason: eval.Error.String(),
LatestResult: newEvaluation(t2, eval.Error),
StartsAt: t2,
EndsAt: t2,
LastEvaluationTime: t2,
@ -1231,14 +1134,11 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
Results: []state.Evaluation{
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.Error),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Error,
Error: fmt.Errorf("with_state_error"),
LatestResult: newEvaluation(tn(6), eval.Error),
StartsAt: tn(4),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
@ -1265,14 +1165,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(8), eval.Alerting),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.Pending,
LatestResult: newEvaluation(tn(8), eval.Alerting),
StartsAt: tn(8),
EndsAt: tn(8).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(8),
@ -1299,14 +1195,10 @@ func TestProcessEvalResults(t *testing.T) {
expectedAnnotations: 3,
expectedStates: []*state.State{
{
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
Results: []state.Evaluation{
newEvaluation(tn(4), eval.Alerting),
newEvaluation(tn(5), eval.Error),
newEvaluation(tn(6), eval.NoData),
},
Labels: labels["system + rule + labels1"],
ResultFingerprint: labels1.Fingerprint(),
State: eval.NoData,
LatestResult: newEvaluation(tn(6), eval.NoData),
StartsAt: tn(6),
EndsAt: tn(6).Add(state.ResendDelay * 4),
LastEvaluationTime: tn(6),
@ -1337,10 +1229,8 @@ func TestProcessEvalResults(t *testing.T) {
"label": "test",
"job": "prod/grafana",
}),
State: eval.Normal,
Results: []state.Evaluation{
newEvaluation(t1, eval.Normal),
},
State: eval.Normal,
LatestResult: newEvaluation(t1, eval.Normal),
StartsAt: t1,
EndsAt: t1,
LastEvaluationTime: t1,
@ -1371,14 +1261,10 @@ func TestProcessEvalResults(t *testing.T) {
},
expectedStates: []*state.State{
{
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting,
Results: []state.Evaluation{
newEvaluation(t1, eval.Alerting),
newEvaluation(t2, eval.Error),
newEvaluation(t3, eval.Alerting),
},
Labels: labels["system + rule"],
ResultFingerprint: data.Labels{}.Fingerprint(),
State: eval.Alerting,
LatestResult: newEvaluation(t3, eval.Alerting),
StartsAt: t3,
EndsAt: t3.Add(state.ResendDelay * 4),
LastEvaluationTime: t3,
@ -1627,13 +1513,11 @@ func TestStaleResultsHandler(t *testing.T) {
},
Values: make(map[string]float64),
State: eval.Normal,
Results: []state.Evaluation{
{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
Condition: "A",
},
LatestResult: &state.Evaluation{
EvaluationTime: evaluationTime,
EvaluationState: eval.Normal,
Values: make(map[string]*float64),
Condition: "A",
},
StartsAt: evaluationTime,
EndsAt: evaluationTime,

22
pkg/services/ngalert/state/state.go
Unescape View File

@ -38,8 +38,8 @@ type State struct {
// ResultFingerprint is a hash of labels of the result before it is processed by
ResultFingerprint data.Fingerprint
// Results contains the result of the current and previous evaluations.
Results []Evaluation
// LatestResult contains the result of the most recent evaluation, if available.
LatestResult *Evaluation
// Error is set if the current evaluation returned an error. If error is non-nil results
// can still contain the results of previous evaluations.
@ -427,20 +427,6 @@ func (a *State) Equals(b *State) bool {
data.Labels(a.Annotations).String() == data.Labels(b.Annotations).String()
}
func (a *State) TrimResults(alertRule *models.AlertRule) {
numBuckets := int64(alertRule.For.Seconds()) / alertRule.IntervalSeconds
if numBuckets == 0 {
numBuckets = 10 // keep at least 10 evaluations in the event For is set to 0
}
if len(a.Results) < int(numBuckets) {
return
}
newResults := make([]Evaluation, numBuckets)
copy(newResults, a.Results[len(a.Results)-int(numBuckets):])
a.Results = newResults
}
func nextEndsTime(interval int64, evaluatedAt time.Time) time.Time {
ends := ResendDelay
intv := time.Second * time.Duration(interval)
@ -464,11 +450,11 @@ func (a *State) GetLabels(opts ...models.LabelOption) map[string]string {
}
func (a *State) GetLastEvaluationValuesForCondition() map[string]float64 {
if len(a.Results) <= 0 {
if a.LatestResult == nil {
return nil
}
lastResult := a.Results[len(a.Results)-1]
lastResult := *a.LatestResult
r := make(map[string]float64, len(lastResult.Values))
for refID, value := range lastResult.Values {

62
pkg/services/ngalert/state/state_test.go
Unescape View File

@ -475,9 +475,9 @@ func TestNeedsSending(t *testing.T) {
}
func TestGetLastEvaluationValuesForCondition(t *testing.T) {
genState := func(results []Evaluation) *State {
genState := func(latestResult *Evaluation) *State {
return &State{
Results: results,
LatestResult: latestResult,
}
}
@ -487,57 +487,43 @@ func TestGetLastEvaluationValuesForCondition(t *testing.T) {
})
t.Run("should return value of the condition of the last result", func(t *testing.T) {
expected := rand.Float64()
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": util.Pointer(rand.Float64()),
},
Condition: "A",
},
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"B": util.Pointer(rand.Float64()),
"A": util.Pointer(expected),
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"B": util.Pointer(rand.Float64()),
"A": util.Pointer(expected),
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.Len(t, result, 1)
require.Contains(t, result, "A")
require.Equal(t, result["A"], expected)
})
t.Run("should return empty map if there is no value for condition", func(t *testing.T) {
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"C": util.Pointer(rand.Float64()),
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"C": util.Pointer(rand.Float64()),
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result)
require.Len(t, result, 0)
})
t.Run("should use NaN if value is not defined", func(t *testing.T) {
evals := []Evaluation{
{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": nil,
},
Condition: "A",
eval := &Evaluation{
EvaluationTime: time.Time{},
EvaluationState: 0,
Values: map[string]*float64{
"A": nil,
},
Condition: "A",
}
result := genState(evals).GetLastEvaluationValuesForCondition()
result := genState(eval).GetLastEvaluationValuesForCondition()
require.NotNil(t, result)
require.Len(t, result, 1)
require.Contains(t, result, "A")

Write Preview
Loading…
Cancel
Save