From 61cb26711e45c0987cd44d58010e1de62c32fe67 Mon Sep 17 00:00:00 2001 From: Santiago Date: Thu, 19 Oct 2023 11:27:37 +0200 Subject: [PATCH] Alerting: Fetch alerts from a remote Alertmanager (#75844) * Alerting: post alerts to the remote Alertmanager and fetch them * fix broken tests * Alerting: Add Mimir Backend image to devenv (blocks) * add alerting as code owner for mimir_backend block * Alerting: Use Mimir image to run integration tests for the remote Alertmanager * skip integration test when running all tests * skipping integration test when no Alertmanager URL is provided * fix bad host for mimir_backend * remove basic auth testing until we have an nginx image in our CI * add integration tests for alerts * fix tests * change SendCtx -> Send, add context.Context to Send, fix CI * add reover() for functions from the Prometheus Alertmanager HTTP client that could panic * add TODO to implement PutAlerts in a way that mimicks what Prometheus does * fix log format --- pkg/services/ngalert/api/api_alertmanager.go | 2 + pkg/services/ngalert/notifier/alertmanager.go | 2 +- pkg/services/ngalert/notifier/alerts.go | 6 +- .../ngalert/notifier/external_alertmanager.go | 115 +++++++++++++++--- .../notifier/external_alertmanager_test.go | 72 +++++++++++ .../ngalert/notifier/multiorg_alertmanager.go | 8 +- .../ngalert/schedule/alerts_sender_mock.go | 40 ++++-- pkg/services/ngalert/schedule/schedule.go | 6 +- .../ngalert/schedule/schedule_unit_test.go | 24 ++-- pkg/services/ngalert/sender/router.go | 4 +- pkg/services/ngalert/sender/router_test.go | 12 +- 11 files changed, 234 insertions(+), 57 deletions(-) diff --git a/pkg/services/ngalert/api/api_alertmanager.go b/pkg/services/ngalert/api/api_alertmanager.go index 1139d308744..0680f3a26c3 100644 --- a/pkg/services/ngalert/api/api_alertmanager.go +++ b/pkg/services/ngalert/api/api_alertmanager.go @@ -149,6 +149,7 @@ func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *contextmodel.ReqContext) res } groups, err := am.GetAlertGroups( + c.Req.Context(), c.QueryBoolWithDefault("active", true), c.QueryBoolWithDefault("silenced", true), c.QueryBoolWithDefault("inhibited", true), @@ -173,6 +174,7 @@ func (srv AlertmanagerSrv) RouteGetAMAlerts(c *contextmodel.ReqContext) response } alerts, err := am.GetAlerts( + c.Req.Context(), c.QueryBoolWithDefault("active", true), c.QueryBoolWithDefault("silenced", true), c.QueryBoolWithDefault("inhibited", true), diff --git a/pkg/services/ngalert/notifier/alertmanager.go b/pkg/services/ngalert/notifier/alertmanager.go index 53d86909ca8..8c138fd8638 100644 --- a/pkg/services/ngalert/notifier/alertmanager.go +++ b/pkg/services/ngalert/notifier/alertmanager.go @@ -381,7 +381,7 @@ func (am *alertmanager) buildReceiverIntegrations(receiver *alertingNotify.APIRe } // PutAlerts receives the alerts and then sends them through the corresponding route based on whenever the alert has a receiver embedded or not -func (am *alertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error { +func (am *alertmanager) PutAlerts(_ context.Context, postableAlerts apimodels.PostableAlerts) error { alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts)) for _, pa := range postableAlerts.PostableAlerts { alerts = append(alerts, &alertingNotify.PostableAlert{ diff --git a/pkg/services/ngalert/notifier/alerts.go b/pkg/services/ngalert/notifier/alerts.go index 77200dd797a..2052b2a99d6 100644 --- a/pkg/services/ngalert/notifier/alerts.go +++ b/pkg/services/ngalert/notifier/alerts.go @@ -1,13 +1,15 @@ package notifier import ( + "context" + alertingNotify "github.com/grafana/alerting/notify" ) -func (am *alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.GettableAlerts, error) { +func (am *alertmanager) GetAlerts(_ context.Context, active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.GettableAlerts, error) { return am.Base.GetAlerts(active, silenced, inhibited, filter, receivers) } -func (am *alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.AlertGroups, error) { +func (am *alertmanager) GetAlertGroups(_ context.Context, active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.AlertGroups, error) { return am.Base.GetAlertGroups(active, silenced, inhibited, filter, receivers) } diff --git a/pkg/services/ngalert/notifier/external_alertmanager.go b/pkg/services/ngalert/notifier/external_alertmanager.go index bc8e475b64e..cbaf1013291 100644 --- a/pkg/services/ngalert/notifier/external_alertmanager.go +++ b/pkg/services/ngalert/notifier/external_alertmanager.go @@ -8,10 +8,13 @@ import ( httptransport "github.com/go-openapi/runtime/client" "github.com/go-openapi/strfmt" + alertingNotify "github.com/grafana/alerting/notify" "github.com/grafana/grafana/pkg/infra/log" apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" "github.com/grafana/grafana/pkg/services/ngalert/models" amclient "github.com/prometheus/alertmanager/api/v2/client" + amalert "github.com/prometheus/alertmanager/api/v2/client/alert" + amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup" amsilence "github.com/prometheus/alertmanager/api/v2/client/silence" ) @@ -69,6 +72,10 @@ func newExternalAlertmanager(cfg externalAlertmanagerConfig, orgID int64) (*exte }, nil } +func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error { + return nil +} + func (am *externalAlertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig) error { return nil } @@ -78,6 +85,12 @@ func (am *externalAlertmanager) SaveAndApplyDefaultConfig(ctx context.Context) e } func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while creating silence", "err", r) + } + }() + params := amsilence.NewPostSilencesParamsWithContext(ctx).WithSilence(silence) res, err := am.amClient.Silence.PostSilences(params) if err != nil { @@ -88,6 +101,12 @@ func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apim } func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID string) error { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while deleting silence", "err", r) + } + }() + params := amsilence.NewDeleteSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID)) _, err := am.amClient.Silence.DeleteSilence(params) if err != nil { @@ -97,21 +116,28 @@ func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID str } func (am *externalAlertmanager) GetSilence(ctx context.Context, silenceID string) (apimodels.GettableSilence, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while getting silence", "err", r) + } + }() + params := amsilence.NewGetSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID)) res, err := am.amClient.Silence.GetSilence(params) if err != nil { return apimodels.GettableSilence{}, err } - if res != nil { - return *res.Payload, nil - } - - // In theory, this should never happen as is not possible for GetSilence to return an empty payload but no error. - return apimodels.GettableSilence{}, fmt.Errorf("unexpected error while trying to fetch silence: %s", silenceID) + return *res.Payload, nil } func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []string) (apimodels.GettableSilences, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while listing silences", "err", r) + } + }() + params := amsilence.NewGetSilencesParamsWithContext(ctx).WithFilter(filter) res, err := am.amClient.Silence.GetSilences(params) if err != nil { @@ -121,30 +147,83 @@ func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []strin return res.Payload, nil } -func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus { - return apimodels.GettableStatus{} +func (am *externalAlertmanager) GetAlerts(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while getting alerts", "err", r) + } + }() + + params := amalert.NewGetAlertsParamsWithContext(ctx). + WithActive(&active). + WithSilenced(&silenced). + WithInhibited(&inhibited). + WithFilter(filter). + WithReceiver(&receiver) + + res, err := am.amClient.Alert.GetAlerts(params) + if err != nil { + return apimodels.GettableAlerts{}, err + } + + return res.Payload, nil } -func (am *externalAlertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) { - return apimodels.GettableAlerts{}, nil +func (am *externalAlertmanager) GetAlertGroups(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while getting alert groups", "err", r) + } + }() + + params := amalertgroup.NewGetAlertGroupsParamsWithContext(ctx). + WithActive(&active). + WithSilenced(&silenced). + WithInhibited(&inhibited). + WithFilter(filter). + WithReceiver(&receiver) + + res, err := am.amClient.Alertgroup.GetAlertGroups(params) + if err != nil { + return apimodels.AlertGroups{}, err + } + + return res.Payload, nil } -func (am *externalAlertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) { - return apimodels.AlertGroups{}, nil +// TODO: implement PutAlerts in a way that is similar to what Prometheus does. +// This current implementation is only good for testing methods that retrieve alerts from the remote Alertmanager. +// More details in issue https://github.com/grafana/grafana/issues/76692 +func (am *externalAlertmanager) PutAlerts(ctx context.Context, postableAlerts apimodels.PostableAlerts) error { + defer func() { + if r := recover(); r != nil { + am.log.Error("Panic while putting alerts", "err", r) + } + }() + + alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts)) + for _, pa := range postableAlerts.PostableAlerts { + alerts = append(alerts, &alertingNotify.PostableAlert{ + Annotations: pa.Annotations, + EndsAt: pa.EndsAt, + StartsAt: pa.StartsAt, + Alert: pa.Alert, + }) + } + + params := amalert.NewPostAlertsParamsWithContext(ctx).WithAlerts(alerts) + _, err := am.amClient.Alert.PostAlerts(params) + return err } -func (am *externalAlertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error { - return nil +func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus { + return apimodels.GettableStatus{} } func (am *externalAlertmanager) GetReceivers(ctx context.Context) []apimodels.Receiver { return []apimodels.Receiver{} } -func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error { - return nil -} - func (am *externalAlertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error) { return &TestReceiversResult{}, nil } diff --git a/pkg/services/ngalert/notifier/external_alertmanager_test.go b/pkg/services/ngalert/notifier/external_alertmanager_test.go index b54d73f7762..c074c70d0a3 100644 --- a/pkg/services/ngalert/notifier/external_alertmanager_test.go +++ b/pkg/services/ngalert/notifier/external_alertmanager_test.go @@ -168,6 +168,61 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) { require.Equal(t, *silences[1].Status.State, "expired") } +func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) { + if testing.Short() { + t.Skip("skipping integration test") + } + + amURL, ok := os.LookupEnv("AM_URL") + if !ok { + t.Skip("No Alertmanager URL provided") + } + tenantID := os.Getenv("AM_TENANT_ID") + password := os.Getenv("AM_PASSWORD") + + cfg := externalAlertmanagerConfig{ + URL: amURL + "/alertmanager", + TenantID: tenantID, + BasicAuthPassword: password, + DefaultConfig: validConfig, + } + am, err := newExternalAlertmanager(cfg, 1) + require.NoError(t, err) + + // We should have no alerts and no groups at first. + alerts, err := am.GetAlerts(context.Background(), true, true, true, []string{}, "") + require.NoError(t, err) + require.Equal(t, 0, len(alerts)) + + alertGroups, err := am.GetAlertGroups(context.Background(), true, true, true, []string{}, "") + require.NoError(t, err) + require.Equal(t, 0, len(alertGroups)) + + // Let's create two active alerts and one expired one. + alert1 := genAlert(true, map[string]string{"test_1": "test_1"}) + alert2 := genAlert(true, map[string]string{"test_2": "test_2"}) + alert3 := genAlert(false, map[string]string{"test_3": "test_3"}) + postableAlerts := apimodels.PostableAlerts{ + PostableAlerts: []amv2.PostableAlert{alert1, alert2, alert3}, + } + err = am.PutAlerts(context.Background(), postableAlerts) + require.NoError(t, err) + + // We should have two alerts and one group now. + alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{}, "") + require.NoError(t, err) + require.Equal(t, 2, len(alerts)) + + alertGroups, err = am.GetAlertGroups(context.Background(), true, true, true, []string{}, "") + require.NoError(t, err) + require.Equal(t, 1, len(alertGroups)) + + // Filtering by `test_1=test_1` should return one alert. + alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{"test_1=test_1"}, "") + require.NoError(t, err) + require.Equal(t, 1, len(alerts)) +} + func genSilence(createdBy string) apimodels.PostableSilence { starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second)) ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second)) @@ -188,3 +243,20 @@ func genSilence(createdBy string) apimodels.PostableSilence { }, } } + +func genAlert(active bool, labels map[string]string) amv2.PostableAlert { + endsAt := time.Now() + if active { + endsAt = time.Now().Add(1 * time.Minute) + } + + return amv2.PostableAlert{ + Annotations: amv2.LabelSet(map[string]string{"test_annotation": "test_annotation_value"}), + StartsAt: strfmt.DateTime(time.Now()), + EndsAt: strfmt.DateTime(endsAt), + Alert: amv2.Alert{ + GeneratorURL: strfmt.URI("http://localhost:8080"), + Labels: amv2.LabelSet(labels), + }, + } +} diff --git a/pkg/services/ngalert/notifier/multiorg_alertmanager.go b/pkg/services/ngalert/notifier/multiorg_alertmanager.go index edf286d3093..3c3527787e3 100644 --- a/pkg/services/ngalert/notifier/multiorg_alertmanager.go +++ b/pkg/services/ngalert/notifier/multiorg_alertmanager.go @@ -32,10 +32,10 @@ var ( type Alertmanager interface { // Configuration + ApplyConfig(context.Context, *models.AlertConfiguration) error SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error SaveAndApplyDefaultConfig(ctx context.Context) error GetStatus() apimodels.GettableStatus - ApplyConfig(context.Context, *models.AlertConfiguration) error // Silences CreateSilence(context.Context, *apimodels.PostableSilence) (string, error) @@ -44,9 +44,9 @@ type Alertmanager interface { ListSilences(context.Context, []string) (apimodels.GettableSilences, error) // Alerts - GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) - GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) - PutAlerts(postableAlerts apimodels.PostableAlerts) error + GetAlerts(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) + GetAlertGroups(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) + PutAlerts(context.Context, apimodels.PostableAlerts) error // Receivers GetReceivers(ctx context.Context) []apimodels.Receiver diff --git a/pkg/services/ngalert/schedule/alerts_sender_mock.go b/pkg/services/ngalert/schedule/alerts_sender_mock.go index 9a11892c70b..75a9e2ecc5e 100644 --- a/pkg/services/ngalert/schedule/alerts_sender_mock.go +++ b/pkg/services/ngalert/schedule/alerts_sender_mock.go @@ -1,11 +1,13 @@ -// Code generated by mockery v2.10.0. DO NOT EDIT. +// Code generated by mockery v2.34.2. DO NOT EDIT. package schedule import ( - mock "github.com/stretchr/testify/mock" + context "context" definitions "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" + mock "github.com/stretchr/testify/mock" + models "github.com/grafana/grafana/pkg/services/ngalert/models" ) @@ -22,9 +24,9 @@ func (_m *AlertsSenderMock) EXPECT() *AlertsSenderMock_Expecter { return &AlertsSenderMock_Expecter{mock: &_m.Mock} } -// Send provides a mock function with given fields: key, alerts -func (_m *AlertsSenderMock) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) { - _m.Called(key, alerts) +// Send provides a mock function with given fields: ctx, key, alerts +func (_m *AlertsSenderMock) Send(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts) { + _m.Called(ctx, key, alerts) } // AlertsSenderMock_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send' @@ -33,15 +35,16 @@ type AlertsSenderMock_Send_Call struct { } // Send is a helper method to define mock.On call +// - ctx context.Context // - key models.AlertRuleKey // - alerts definitions.PostableAlerts -func (_e *AlertsSenderMock_Expecter) Send(key any, alerts any) *AlertsSenderMock_Send_Call { - return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", key, alerts)} +func (_e *AlertsSenderMock_Expecter) Send(ctx interface{}, key interface{}, alerts interface{}) *AlertsSenderMock_Send_Call { + return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", ctx, key, alerts)} } -func (_c *AlertsSenderMock_Send_Call) Run(run func(key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call { +func (_c *AlertsSenderMock_Send_Call) Run(run func(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call { _c.Call.Run(func(args mock.Arguments) { - run(args[0].(models.AlertRuleKey), args[1].(definitions.PostableAlerts)) + run(args[0].(context.Context), args[1].(models.AlertRuleKey), args[2].(definitions.PostableAlerts)) }) return _c } @@ -50,3 +53,22 @@ func (_c *AlertsSenderMock_Send_Call) Return() *AlertsSenderMock_Send_Call { _c.Call.Return() return _c } + +func (_c *AlertsSenderMock_Send_Call) RunAndReturn(run func(context.Context, models.AlertRuleKey, definitions.PostableAlerts)) *AlertsSenderMock_Send_Call { + _c.Call.Return(run) + return _c +} + +// NewAlertsSenderMock creates a new instance of AlertsSenderMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations. +// The first argument is typically a *testing.T value. +func NewAlertsSenderMock(t interface { + mock.TestingT + Cleanup(func()) +}) *AlertsSenderMock { + mock := &AlertsSenderMock{} + mock.Mock.Test(t) + + t.Cleanup(func() { mock.AssertExpectations(t) }) + + return mock +} diff --git a/pkg/services/ngalert/schedule/schedule.go b/pkg/services/ngalert/schedule/schedule.go index fd6077186d6..13ac4d04819 100644 --- a/pkg/services/ngalert/schedule/schedule.go +++ b/pkg/services/ngalert/schedule/schedule.go @@ -38,7 +38,7 @@ type ScheduleService interface { // //go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter type AlertsSender interface { - Send(key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts) + Send(ctx context.Context, key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts) } // RulesStore is a store that provides alert rules for scheduling @@ -360,7 +360,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR notify := func(states []state.StateTransition) { expiredAlerts := state.FromAlertsStateToStoppedAlert(states, sch.appURL, sch.clock) if len(expiredAlerts.PostableAlerts) > 0 { - sch.alertsSender.Send(key, expiredAlerts) + sch.alertsSender.Send(grafanaCtx, key, expiredAlerts) } } @@ -437,7 +437,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))), )) if len(alerts.PostableAlerts) > 0 { - sch.alertsSender.Send(key, alerts) + sch.alertsSender.Send(ctx, key, alerts) } sendDuration.Observe(sch.clock.Now().Sub(start).Seconds()) } diff --git a/pkg/services/ngalert/schedule/schedule_unit_test.go b/pkg/services/ngalert/schedule/schedule_unit_test.go index 0874ef99f76..5e362580d1e 100644 --- a/pkg/services/ngalert/schedule/schedule_unit_test.go +++ b/pkg/services/ngalert/schedule/schedule_unit_test.go @@ -59,7 +59,7 @@ func TestProcessTicks(t *testing.T) { mockedClock := clock.NewMock() notifier := &AlertsSenderMock{} - notifier.EXPECT().Send(mock.Anything, mock.Anything).Return() + notifier.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return() appUrl := &url.URL{ Scheme: "http", @@ -578,7 +578,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { updateChan := make(chan ruleVersionAndPauseStatus) sender := AlertsSenderMock{} - sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return() + sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return() sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender) ruleStore.PutRule(context.Background(), rule) @@ -645,8 +645,8 @@ func TestSchedule_ruleRoutine(t *testing.T) { require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID)) sender.AssertNumberOfCalls(t, "Send", 1) - args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts) - require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1])) + args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts) + require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2])) require.Len(t, args.PostableAlerts, expectedToBeSent) }) }) @@ -659,7 +659,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { evalAppliedChan := make(chan time.Time) sender := AlertsSenderMock{} - sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return() + sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return() sch, ruleStore, _, reg := createSchedule(evalAppliedChan, &sender) ruleStore.PutRule(context.Background(), rule) @@ -748,8 +748,8 @@ func TestSchedule_ruleRoutine(t *testing.T) { t.Run("it should send special alert DatasourceError", func(t *testing.T) { sender.AssertNumberOfCalls(t, "Send", 1) - args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts) - require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1])) + args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts) + require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2])) assert.Len(t, args.PostableAlerts, 1) assert.Equal(t, state.ErrorAlertName, args.PostableAlerts[0].Labels[prometheusModel.AlertNameLabel]) }) @@ -764,7 +764,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { evalAppliedChan := make(chan time.Time) sender := AlertsSenderMock{} - sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return() + sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return() sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender) ruleStore.PutRule(context.Background(), rule) @@ -783,8 +783,8 @@ func TestSchedule_ruleRoutine(t *testing.T) { waitForTimeChannel(t, evalAppliedChan) sender.AssertNumberOfCalls(t, "Send", 1) - args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts) - require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1])) + args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts) + require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2])) require.Len(t, args.PostableAlerts, 1) }) @@ -797,7 +797,7 @@ func TestSchedule_ruleRoutine(t *testing.T) { evalAppliedChan := make(chan time.Time) sender := AlertsSenderMock{} - sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return() + sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return() sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender) ruleStore.PutRule(context.Background(), rule) @@ -873,7 +873,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor if senderMock == nil { senderMock = &AlertsSenderMock{} - senderMock.EXPECT().Send(mock.Anything, mock.Anything).Return() + senderMock.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return() } cfg := setting.UnifiedAlertingSettings{ diff --git a/pkg/services/ngalert/sender/router.go b/pkg/services/ngalert/sender/router.go index 56302f7eadf..cce4af1399b 100644 --- a/pkg/services/ngalert/sender/router.go +++ b/pkg/services/ngalert/sender/router.go @@ -288,7 +288,7 @@ func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, err password, parsed.Host, parsed.Path, parsed.RawQuery), nil } -func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) { +func (d *AlertsRouter) Send(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts) { logger := d.logger.New(key.LogContext()...) if len(alerts.PostableAlerts) == 0 { logger.Info("No alerts to notify about") @@ -304,7 +304,7 @@ func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.Postable n, err := d.multiOrgNotifier.AlertmanagerFor(key.OrgID) if err == nil { localNotifierExist = true - if err := n.PutAlerts(alerts); err != nil { + if err := n.PutAlerts(ctx, alerts); err != nil { logger.Error("Failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "error", err) } } else { diff --git a/pkg/services/ngalert/sender/router_test.go b/pkg/services/ngalert/sender/router_test.go index 77bc7b168c4..5fb627ca09d 100644 --- a/pkg/services/ngalert/sender/router_test.go +++ b/pkg/services/ngalert/sender/router_test.go @@ -85,7 +85,7 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) { alerts.PostableAlerts = append(alerts.PostableAlerts, alert) } - alertsRouter.Send(ruleKey, alerts) + alertsRouter.Send(context.Background(), ruleKey, alerts) // Eventually, our Alertmanager should have received at least one alert. assertAlertsDelivered(t, fakeAM, expected) @@ -189,8 +189,8 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T) alerts2.PostableAlerts = append(alerts2.PostableAlerts, alert) } - alertsRouter.Send(ruleKey1, alerts1) - alertsRouter.Send(ruleKey2, alerts2) + alertsRouter.Send(context.Background(), ruleKey1, alerts1) + alertsRouter.Send(context.Background(), ruleKey2, alerts2) assertAlertsDelivered(t, fakeAM, expected) @@ -315,7 +315,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) { expected = append(expected, &alert) alerts.PostableAlerts = append(alerts.PostableAlerts, alert) } - alertsRouter.Send(ruleKey, alerts) + alertsRouter.Send(context.Background(), ruleKey, alerts) // Eventually, our Alertmanager should have received at least one alert. assertAlertsDelivered(t, fakeAM, expected) @@ -347,11 +347,11 @@ func TestChangingAlertmanagersChoice(t *testing.T) { assertAlertmanagersStatusForOrg(t, alertsRouter, ruleKey.OrgID, 1, 0) require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID]) - alertsRouter.Send(ruleKey, alerts) + alertsRouter.Send(context.Background(), ruleKey, alerts) am, err := moa.AlertmanagerFor(ruleKey.OrgID) require.NoError(t, err) - actualAlerts, err := am.GetAlerts(true, true, true, nil, "") + actualAlerts, err := am.GetAlerts(context.Background(), true, true, true, nil, "") require.NoError(t, err) require.Len(t, actualAlerts, len(expected)) }