mirror of https://github.com/grafana/grafana
Alerting: update test TestAlertingTicker to not rely on clock (#58544)
* extract method processTick * make processTick return scheduled rules * move state manager tests to state manager * update test * move all tests into one file * remove unused fieldspull/58561/head
Yuri Tseretyan
3 years ago
committed by
GitHub
parent
ee7348afee
commit
bad4f28d0d
@ -1,299 +0,0 @@ |
||||
package schedule_test |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"net/url" |
||||
"runtime" |
||||
"strings" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/benbjohnson/clock" |
||||
"github.com/google/go-cmp/cmp" |
||||
"github.com/google/go-cmp/cmp/cmpopts" |
||||
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/mock" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/eval" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/image" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/metrics" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/models" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/schedule" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/state" |
||||
"github.com/grafana/grafana/pkg/services/ngalert/tests" |
||||
"github.com/grafana/grafana/pkg/setting" |
||||
) |
||||
|
||||
var testMetrics = metrics.NewNGAlert(prometheus.NewPedanticRegistry()) |
||||
|
||||
type evalAppliedInfo struct { |
||||
alertDefKey models.AlertRuleKey |
||||
now time.Time |
||||
} |
||||
|
||||
func TestWarmStateCache(t *testing.T) { |
||||
evaluationTime, err := time.Parse("2006-01-02", "2021-03-25") |
||||
require.NoError(t, err) |
||||
ctx := context.Background() |
||||
_, dbstore := tests.SetupTestEnv(t, 1) |
||||
|
||||
const mainOrgID int64 = 1 |
||||
rule := tests.CreateTestAlertRule(t, ctx, dbstore, 600, mainOrgID) |
||||
|
||||
expectedEntries := []*state.State{ |
||||
{ |
||||
AlertRuleUID: rule.UID, |
||||
OrgID: rule.OrgID, |
||||
CacheID: `[["test1","testValue1"]]`, |
||||
Labels: data.Labels{"test1": "testValue1"}, |
||||
State: eval.Normal, |
||||
Results: []state.Evaluation{ |
||||
{EvaluationTime: evaluationTime, EvaluationState: eval.Normal}, |
||||
}, |
||||
StartsAt: evaluationTime.Add(-1 * time.Minute), |
||||
EndsAt: evaluationTime.Add(1 * time.Minute), |
||||
LastEvaluationTime: evaluationTime, |
||||
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, |
||||
}, { |
||||
AlertRuleUID: rule.UID, |
||||
OrgID: rule.OrgID, |
||||
CacheID: `[["test2","testValue2"]]`, |
||||
Labels: data.Labels{"test2": "testValue2"}, |
||||
State: eval.Alerting, |
||||
Results: []state.Evaluation{ |
||||
{EvaluationTime: evaluationTime, EvaluationState: eval.Alerting}, |
||||
}, |
||||
StartsAt: evaluationTime.Add(-1 * time.Minute), |
||||
EndsAt: evaluationTime.Add(1 * time.Minute), |
||||
LastEvaluationTime: evaluationTime, |
||||
Annotations: map[string]string{"testAnnoKey": "testAnnoValue"}, |
||||
}, |
||||
} |
||||
|
||||
labels := models.InstanceLabels{"test1": "testValue1"} |
||||
_, hash, _ := labels.StringAndHash() |
||||
instance1 := models.AlertInstance{ |
||||
AlertInstanceKey: models.AlertInstanceKey{ |
||||
RuleOrgID: rule.OrgID, |
||||
RuleUID: rule.UID, |
||||
LabelsHash: hash, |
||||
}, |
||||
CurrentState: models.InstanceStateNormal, |
||||
LastEvalTime: evaluationTime, |
||||
CurrentStateSince: evaluationTime.Add(-1 * time.Minute), |
||||
CurrentStateEnd: evaluationTime.Add(1 * time.Minute), |
||||
Labels: labels, |
||||
} |
||||
|
||||
_ = dbstore.SaveAlertInstances(ctx, instance1) |
||||
|
||||
labels = models.InstanceLabels{"test2": "testValue2"} |
||||
_, hash, _ = labels.StringAndHash() |
||||
instance2 := models.AlertInstance{ |
||||
AlertInstanceKey: models.AlertInstanceKey{ |
||||
RuleOrgID: rule.OrgID, |
||||
RuleUID: rule.UID, |
||||
LabelsHash: hash, |
||||
}, |
||||
CurrentState: models.InstanceStateFiring, |
||||
LastEvalTime: evaluationTime, |
||||
CurrentStateSince: evaluationTime.Add(-1 * time.Minute), |
||||
CurrentStateEnd: evaluationTime.Add(1 * time.Minute), |
||||
Labels: labels, |
||||
} |
||||
_ = dbstore.SaveAlertInstances(ctx, instance2) |
||||
st := state.NewManager(testMetrics.GetStateMetrics(), nil, dbstore, &image.NoopImageService{}, clock.NewMock(), &state.FakeHistorian{}) |
||||
st.Warm(ctx, dbstore) |
||||
|
||||
t.Run("instance cache has expected entries", func(t *testing.T) { |
||||
for _, entry := range expectedEntries { |
||||
cacheEntry := st.Get(entry.OrgID, entry.AlertRuleUID, entry.CacheID) |
||||
|
||||
if diff := cmp.Diff(entry, cacheEntry, cmpopts.IgnoreFields(state.State{}, "Results")); diff != "" { |
||||
t.Errorf("Result mismatch (-want +got):\n%s", diff) |
||||
t.FailNow() |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
|
||||
func TestAlertingTicker(t *testing.T) { |
||||
ctx := context.Background() |
||||
_, dbstore := tests.SetupTestEnv(t, 1) |
||||
|
||||
alerts := make([]*models.AlertRule, 0) |
||||
|
||||
const mainOrgID int64 = 1 |
||||
// create alert rule under main org with one second interval
|
||||
alerts = append(alerts, tests.CreateTestAlertRule(t, ctx, dbstore, 1, mainOrgID)) |
||||
|
||||
evalAppliedCh := make(chan evalAppliedInfo, len(alerts)) |
||||
stopAppliedCh := make(chan models.AlertRuleKey, len(alerts)) |
||||
|
||||
mockedClock := clock.NewMock() |
||||
|
||||
cfg := setting.UnifiedAlertingSettings{ |
||||
BaseInterval: time.Second, |
||||
AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
|
||||
} |
||||
|
||||
notifier := &schedule.AlertsSenderMock{} |
||||
notifier.EXPECT().Send(mock.Anything, mock.Anything).Return() |
||||
|
||||
schedCfg := schedule.SchedulerCfg{ |
||||
Cfg: cfg, |
||||
C: mockedClock, |
||||
EvalAppliedFunc: func(alertDefKey models.AlertRuleKey, now time.Time) { |
||||
evalAppliedCh <- evalAppliedInfo{alertDefKey: alertDefKey, now: now} |
||||
}, |
||||
StopAppliedFunc: func(alertDefKey models.AlertRuleKey) { |
||||
stopAppliedCh <- alertDefKey |
||||
}, |
||||
RuleStore: dbstore, |
||||
Metrics: testMetrics.GetSchedulerMetrics(), |
||||
AlertSender: notifier, |
||||
} |
||||
st := state.NewManager(testMetrics.GetStateMetrics(), nil, dbstore, &image.NoopImageService{}, clock.NewMock(), &state.FakeHistorian{}) |
||||
appUrl := &url.URL{ |
||||
Scheme: "http", |
||||
Host: "localhost", |
||||
} |
||||
sched := schedule.NewScheduler(schedCfg, appUrl, st) |
||||
|
||||
go func() { |
||||
err := sched.Run(ctx) |
||||
require.NoError(t, err) |
||||
}() |
||||
runtime.Gosched() |
||||
|
||||
expectedAlertRulesEvaluated := []models.AlertRuleKey{alerts[0].GetKey()} |
||||
t.Run(fmt.Sprintf("on 1st tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
|
||||
// add alert rule under main org with three seconds interval
|
||||
var threeSecInterval int64 = 3 |
||||
alerts = append(alerts, tests.CreateTestAlertRule(t, ctx, dbstore, threeSecInterval, mainOrgID)) |
||||
t.Logf("alert rule: %v added with interval: %d", alerts[1].GetKey(), threeSecInterval) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[0].GetKey()} |
||||
t.Run(fmt.Sprintf("on 2nd tick alert rule: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[1].GetKey(), alerts[0].GetKey()} |
||||
t.Run(fmt.Sprintf("on 3rd tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[0].GetKey()} |
||||
t.Run(fmt.Sprintf("on 4th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
|
||||
key := alerts[0].GetKey() |
||||
err := dbstore.DeleteAlertRulesByUID(ctx, alerts[0].OrgID, alerts[0].UID) |
||||
require.NoError(t, err) |
||||
t.Logf("alert rule: %v deleted", key) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{} |
||||
t.Run(fmt.Sprintf("on 5th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
expectedAlertRulesStopped := []models.AlertRuleKey{alerts[0].GetKey()} |
||||
t.Run(fmt.Sprintf("on 5th tick alert rules: %s should be stopped", concatenate(expectedAlertRulesStopped)), func(t *testing.T) { |
||||
assertStopRun(t, stopAppliedCh, expectedAlertRulesStopped...) |
||||
}) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[1].GetKey()} |
||||
t.Run(fmt.Sprintf("on 6th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
|
||||
// create alert rule with one second interval
|
||||
alerts = append(alerts, tests.CreateTestAlertRule(t, ctx, dbstore, 1, mainOrgID)) |
||||
|
||||
expectedAlertRulesEvaluated = []models.AlertRuleKey{alerts[2].GetKey()} |
||||
t.Run(fmt.Sprintf("on 7th tick alert rules: %s should be evaluated", concatenate(expectedAlertRulesEvaluated)), func(t *testing.T) { |
||||
tick := advanceClock(t, mockedClock) |
||||
assertEvalRun(t, evalAppliedCh, tick, expectedAlertRulesEvaluated...) |
||||
}) |
||||
} |
||||
|
||||
func assertEvalRun(t *testing.T, ch <-chan evalAppliedInfo, tick time.Time, keys ...models.AlertRuleKey) { |
||||
timeout := time.After(time.Second) |
||||
|
||||
expected := make(map[models.AlertRuleKey]struct{}, len(keys)) |
||||
for _, k := range keys { |
||||
expected[k] = struct{}{} |
||||
} |
||||
|
||||
for { |
||||
select { |
||||
case info := <-ch: |
||||
_, ok := expected[info.alertDefKey] |
||||
if !ok { |
||||
t.Fatalf("alert rule: %v should not have been evaluated at: %v", info.alertDefKey, info.now) |
||||
} |
||||
t.Logf("alert rule: %v evaluated at: %v", info.alertDefKey, info.now) |
||||
assert.Equal(t, tick, info.now) |
||||
delete(expected, info.alertDefKey) |
||||
case <-timeout: |
||||
if len(expected) == 0 { |
||||
return |
||||
} |
||||
t.Fatal("cycle has expired") |
||||
} |
||||
} |
||||
} |
||||
|
||||
func assertStopRun(t *testing.T, ch <-chan models.AlertRuleKey, keys ...models.AlertRuleKey) { |
||||
timeout := time.After(time.Second) |
||||
|
||||
expected := make(map[models.AlertRuleKey]struct{}, len(keys)) |
||||
for _, k := range keys { |
||||
expected[k] = struct{}{} |
||||
} |
||||
|
||||
for { |
||||
select { |
||||
case alertDefKey := <-ch: |
||||
_, ok := expected[alertDefKey] |
||||
t.Logf("alert rule: %v stopped", alertDefKey) |
||||
assert.True(t, ok) |
||||
delete(expected, alertDefKey) |
||||
if len(expected) == 0 { |
||||
return |
||||
} |
||||
case <-timeout: |
||||
if len(expected) == 0 { |
||||
return |
||||
} |
||||
t.Fatal("cycle has expired") |
||||
} |
||||
} |
||||
} |
||||
|
||||
func advanceClock(t *testing.T, mockedClock *clock.Mock) time.Time { |
||||
mockedClock.Add(time.Second) |
||||
return mockedClock.Now() |
||||
// t.Logf("Tick: %v", mockedClock.Now())
|
||||
} |
||||
|
||||
func concatenate(keys []models.AlertRuleKey) string { |
||||
s := make([]string, len(keys)) |
||||
for _, k := range keys { |
||||
s = append(s, k.String()) |
||||
} |
||||
return fmt.Sprintf("[%s]", strings.Join(s, ",")) |
||||
} |
||||
Loading…
Reference in new issue