Alerting: Fetch alerts from a remote Alertmanager (#75844)

* Alerting: post alerts to the remote Alertmanager and fetch them

* fix broken tests

* Alerting: Add Mimir Backend image to devenv (blocks)

* add alerting as code owner for mimir_backend block

* Alerting: Use Mimir image to run integration tests for the remote Alertmanager

* skip integration test when running all tests

* skipping integration test when no Alertmanager URL is provided

* fix bad host for mimir_backend

* remove basic auth testing until we have an nginx image in our CI

* add integration tests for alerts

* fix tests

* change SendCtx -> Send, add context.Context to Send, fix CI

* add reover() for functions from the Prometheus Alertmanager HTTP client that could panic

* add TODO to implement PutAlerts in a way that mimicks what Prometheus does

* fix log format

pkg/services/ngalert/api/api_alertmanager.go

@@ -149,6 +149,7 @@ func (srv AlertmanagerSrv) RouteGetAMAlertGroups(c *contextmodel.ReqContext) res
 	}
 
 	groups, err := am.GetAlertGroups(
+		c.Req.Context(),
 		c.QueryBoolWithDefault("active", true),
 		c.QueryBoolWithDefault("silenced", true),
 		c.QueryBoolWithDefault("inhibited", true),
@@ -173,6 +174,7 @@ func (srv AlertmanagerSrv) RouteGetAMAlerts(c *contextmodel.ReqContext) response
 	}
 
 	alerts, err := am.GetAlerts(
+		c.Req.Context(),
 		c.QueryBoolWithDefault("active", true),
 		c.QueryBoolWithDefault("silenced", true),
 		c.QueryBoolWithDefault("inhibited", true),

pkg/services/ngalert/notifier/alertmanager.go

@@ -381,7 +381,7 @@ func (am *alertmanager) buildReceiverIntegrations(receiver *alertingNotify.APIRe
 }
 
 // PutAlerts receives the alerts and then sends them through the corresponding route based on whenever the alert has a receiver embedded or not
-func (am *alertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error {
+func (am *alertmanager) PutAlerts(_ context.Context, postableAlerts apimodels.PostableAlerts) error {
 	alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts))
 	for _, pa := range postableAlerts.PostableAlerts {
 		alerts = append(alerts, &alertingNotify.PostableAlert{

pkg/services/ngalert/notifier/alerts.go

@@ -1,13 +1,15 @@
 package notifier
 
 import (
+	"context"
+
 	alertingNotify "github.com/grafana/alerting/notify"
 )
 
-func (am *alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.GettableAlerts, error) {
+func (am *alertmanager) GetAlerts(_ context.Context, active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.GettableAlerts, error) {
 	return am.Base.GetAlerts(active, silenced, inhibited, filter, receivers)
 }
 
-func (am *alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.AlertGroups, error) {
+func (am *alertmanager) GetAlertGroups(_ context.Context, active, silenced, inhibited bool, filter []string, receivers string) (alertingNotify.AlertGroups, error) {
 	return am.Base.GetAlertGroups(active, silenced, inhibited, filter, receivers)
 }

pkg/services/ngalert/notifier/external_alertmanager.go

@@ -8,10 +8,13 @@ import (
 
 	httptransport "github.com/go-openapi/runtime/client"
 	"github.com/go-openapi/strfmt"
+	alertingNotify "github.com/grafana/alerting/notify"
 	"github.com/grafana/grafana/pkg/infra/log"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 	amclient "github.com/prometheus/alertmanager/api/v2/client"
+	amalert "github.com/prometheus/alertmanager/api/v2/client/alert"
+	amalertgroup "github.com/prometheus/alertmanager/api/v2/client/alertgroup"
 	amsilence "github.com/prometheus/alertmanager/api/v2/client/silence"
 )
 
@@ -69,6 +72,10 @@ func newExternalAlertmanager(cfg externalAlertmanagerConfig, orgID int64) (*exte
 	}, nil
 }
 
+func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error {
+	return nil
+}
+
 func (am *externalAlertmanager) SaveAndApplyConfig(ctx context.Context, cfg *apimodels.PostableUserConfig) error {
 	return nil
 }
@@ -78,6 +85,12 @@ func (am *externalAlertmanager) SaveAndApplyDefaultConfig(ctx context.Context) e
 }
 
 func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apimodels.PostableSilence) (string, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while creating silence", "err", r)
+		}
+	}()
+
 	params := amsilence.NewPostSilencesParamsWithContext(ctx).WithSilence(silence)
 	res, err := am.amClient.Silence.PostSilences(params)
 	if err != nil {
@@ -88,6 +101,12 @@ func (am *externalAlertmanager) CreateSilence(ctx context.Context, silence *apim
 }
 
 func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID string) error {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while deleting silence", "err", r)
+		}
+	}()
+
 	params := amsilence.NewDeleteSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID))
 	_, err := am.amClient.Silence.DeleteSilence(params)
 	if err != nil {
@@ -97,21 +116,28 @@ func (am *externalAlertmanager) DeleteSilence(ctx context.Context, silenceID str
 }
 
 func (am *externalAlertmanager) GetSilence(ctx context.Context, silenceID string) (apimodels.GettableSilence, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while getting silence", "err", r)
+		}
+	}()
+
 	params := amsilence.NewGetSilenceParamsWithContext(ctx).WithSilenceID(strfmt.UUID(silenceID))
 	res, err := am.amClient.Silence.GetSilence(params)
 	if err != nil {
 		return apimodels.GettableSilence{}, err
 	}
 
-	if res != nil {
-		return *res.Payload, nil
-	}
-
-	// In theory, this should never happen as is not possible for GetSilence to return an empty payload but no error.
-	return apimodels.GettableSilence{}, fmt.Errorf("unexpected error while trying to fetch silence: %s", silenceID)
+	return *res.Payload, nil
 }
 
 func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []string) (apimodels.GettableSilences, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while listing silences", "err", r)
+		}
+	}()
+
 	params := amsilence.NewGetSilencesParamsWithContext(ctx).WithFilter(filter)
 	res, err := am.amClient.Silence.GetSilences(params)
 	if err != nil {
@@ -121,30 +147,83 @@ func (am *externalAlertmanager) ListSilences(ctx context.Context, filter []strin
 	return res.Payload, nil
 }
 
-func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus {
-	return apimodels.GettableStatus{}
+func (am *externalAlertmanager) GetAlerts(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while getting alerts", "err", r)
+		}
+	}()
+
+	params := amalert.NewGetAlertsParamsWithContext(ctx).
+		WithActive(&active).
+		WithSilenced(&silenced).
+		WithInhibited(&inhibited).
+		WithFilter(filter).
+		WithReceiver(&receiver)
+
+	res, err := am.amClient.Alert.GetAlerts(params)
+	if err != nil {
+		return apimodels.GettableAlerts{}, err
+	}
+
+	return res.Payload, nil
 }
 
-func (am *externalAlertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error) {
-	return apimodels.GettableAlerts{}, nil
+func (am *externalAlertmanager) GetAlertGroups(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while getting alert groups", "err", r)
+		}
+	}()
+
+	params := amalertgroup.NewGetAlertGroupsParamsWithContext(ctx).
+		WithActive(&active).
+		WithSilenced(&silenced).
+		WithInhibited(&inhibited).
+		WithFilter(filter).
+		WithReceiver(&receiver)
+
+	res, err := am.amClient.Alertgroup.GetAlertGroups(params)
+	if err != nil {
+		return apimodels.AlertGroups{}, err
+	}
+
+	return res.Payload, nil
 }
 
-func (am *externalAlertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error) {
-	return apimodels.AlertGroups{}, nil
+// TODO: implement PutAlerts in a way that is similar to what Prometheus does.
+// This current implementation is only good for testing methods that retrieve alerts from the remote Alertmanager.
+// More details in issue https://github.com/grafana/grafana/issues/76692
+func (am *externalAlertmanager) PutAlerts(ctx context.Context, postableAlerts apimodels.PostableAlerts) error {
+	defer func() {
+		if r := recover(); r != nil {
+			am.log.Error("Panic while putting alerts", "err", r)
+		}
+	}()
+
+	alerts := make(alertingNotify.PostableAlerts, 0, len(postableAlerts.PostableAlerts))
+	for _, pa := range postableAlerts.PostableAlerts {
+		alerts = append(alerts, &alertingNotify.PostableAlert{
+			Annotations: pa.Annotations,
+			EndsAt:      pa.EndsAt,
+			StartsAt:    pa.StartsAt,
+			Alert:       pa.Alert,
+		})
+	}
+
+	params := amalert.NewPostAlertsParamsWithContext(ctx).WithAlerts(alerts)
+	_, err := am.amClient.Alert.PostAlerts(params)
+	return err
 }
 
-func (am *externalAlertmanager) PutAlerts(postableAlerts apimodels.PostableAlerts) error {
-	return nil
+func (am *externalAlertmanager) GetStatus() apimodels.GettableStatus {
+	return apimodels.GettableStatus{}
 }
 
 func (am *externalAlertmanager) GetReceivers(ctx context.Context) []apimodels.Receiver {
 	return []apimodels.Receiver{}
 }
 
-func (am *externalAlertmanager) ApplyConfig(ctx context.Context, config *models.AlertConfiguration) error {
-	return nil
-}
-
 func (am *externalAlertmanager) TestReceivers(ctx context.Context, c apimodels.TestReceiversConfigBodyParams) (*TestReceiversResult, error) {
 	return &TestReceiversResult{}, nil
 }

pkg/services/ngalert/notifier/external_alertmanager_test.go

@@ -168,6 +168,61 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
 	require.Equal(t, *silences[1].Status.State, "expired")
 }
 
+func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	amURL, ok := os.LookupEnv("AM_URL")
+	if !ok {
+		t.Skip("No Alertmanager URL provided")
+	}
+	tenantID := os.Getenv("AM_TENANT_ID")
+	password := os.Getenv("AM_PASSWORD")
+
+	cfg := externalAlertmanagerConfig{
+		URL:               amURL + "/alertmanager",
+		TenantID:          tenantID,
+		BasicAuthPassword: password,
+		DefaultConfig:     validConfig,
+	}
+	am, err := newExternalAlertmanager(cfg, 1)
+	require.NoError(t, err)
+
+	// We should have no alerts and no groups at first.
+	alerts, err := am.GetAlerts(context.Background(), true, true, true, []string{}, "")
+	require.NoError(t, err)
+	require.Equal(t, 0, len(alerts))
+
+	alertGroups, err := am.GetAlertGroups(context.Background(), true, true, true, []string{}, "")
+	require.NoError(t, err)
+	require.Equal(t, 0, len(alertGroups))
+
+	// Let's create two active alerts and one expired one.
+	alert1 := genAlert(true, map[string]string{"test_1": "test_1"})
+	alert2 := genAlert(true, map[string]string{"test_2": "test_2"})
+	alert3 := genAlert(false, map[string]string{"test_3": "test_3"})
+	postableAlerts := apimodels.PostableAlerts{
+		PostableAlerts: []amv2.PostableAlert{alert1, alert2, alert3},
+	}
+	err = am.PutAlerts(context.Background(), postableAlerts)
+	require.NoError(t, err)
+
+	// We should have two alerts and one group now.
+	alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{}, "")
+	require.NoError(t, err)
+	require.Equal(t, 2, len(alerts))
+
+	alertGroups, err = am.GetAlertGroups(context.Background(), true, true, true, []string{}, "")
+	require.NoError(t, err)
+	require.Equal(t, 1, len(alertGroups))
+
+	// Filtering by `test_1=test_1` should return one alert.
+	alerts, err = am.GetAlerts(context.Background(), true, true, true, []string{"test_1=test_1"}, "")
+	require.NoError(t, err)
+	require.Equal(t, 1, len(alerts))
+}
+
 func genSilence(createdBy string) apimodels.PostableSilence {
 	starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
 	ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
@@ -188,3 +243,20 @@ func genSilence(createdBy string) apimodels.PostableSilence {
 		},
 	}
 }
+
+func genAlert(active bool, labels map[string]string) amv2.PostableAlert {
+	endsAt := time.Now()
+	if active {
+		endsAt = time.Now().Add(1 * time.Minute)
+	}
+
+	return amv2.PostableAlert{
+		Annotations: amv2.LabelSet(map[string]string{"test_annotation": "test_annotation_value"}),
+		StartsAt:    strfmt.DateTime(time.Now()),
+		EndsAt:      strfmt.DateTime(endsAt),
+		Alert: amv2.Alert{
+			GeneratorURL: strfmt.URI("http://localhost:8080"),
+			Labels:       amv2.LabelSet(labels),
+		},
+	}
+}

pkg/services/ngalert/notifier/multiorg_alertmanager.go

@@ -32,10 +32,10 @@ var (
 
 type Alertmanager interface {
 	// Configuration
+	ApplyConfig(context.Context, *models.AlertConfiguration) error
 	SaveAndApplyConfig(ctx context.Context, config *apimodels.PostableUserConfig) error
 	SaveAndApplyDefaultConfig(ctx context.Context) error
 	GetStatus() apimodels.GettableStatus
-	ApplyConfig(context.Context, *models.AlertConfiguration) error
 
 	// Silences
 	CreateSilence(context.Context, *apimodels.PostableSilence) (string, error)
@@ -44,9 +44,9 @@ type Alertmanager interface {
 	ListSilences(context.Context, []string) (apimodels.GettableSilences, error)
 
 	// Alerts
-	GetAlerts(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
-	GetAlertGroups(active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
-	PutAlerts(postableAlerts apimodels.PostableAlerts) error
+	GetAlerts(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.GettableAlerts, error)
+	GetAlertGroups(ctx context.Context, active, silenced, inhibited bool, filter []string, receiver string) (apimodels.AlertGroups, error)
+	PutAlerts(context.Context, apimodels.PostableAlerts) error
 
 	// Receivers
 	GetReceivers(ctx context.Context) []apimodels.Receiver

pkg/services/ngalert/schedule/alerts_sender_mock.go

@@ -1,11 +1,13 @@
-// Code generated by mockery v2.10.0. DO NOT EDIT.
+// Code generated by mockery v2.34.2. DO NOT EDIT.
 
 package schedule
 
 import (
-	mock "github.com/stretchr/testify/mock"
+	context "context"
 
 	definitions "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	mock "github.com/stretchr/testify/mock"
+
 	models "github.com/grafana/grafana/pkg/services/ngalert/models"
 )
 
@@ -22,9 +24,9 @@ func (_m *AlertsSenderMock) EXPECT() *AlertsSenderMock_Expecter {
 	return &AlertsSenderMock_Expecter{mock: &_m.Mock}
 }
 
-// Send provides a mock function with given fields: key, alerts
-func (_m *AlertsSenderMock) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
-	_m.Called(key, alerts)
+// Send provides a mock function with given fields: ctx, key, alerts
+func (_m *AlertsSenderMock) Send(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts) {
+	_m.Called(ctx, key, alerts)
 }
 
 // AlertsSenderMock_Send_Call is a *mock.Call that shadows Run/Return methods with type explicit version for method 'Send'
@@ -33,15 +35,16 @@ type AlertsSenderMock_Send_Call struct {
 }
 
 // Send is a helper method to define mock.On call
+//   - ctx context.Context
 //   - key models.AlertRuleKey
 //   - alerts definitions.PostableAlerts
-func (_e *AlertsSenderMock_Expecter) Send(key any, alerts any) *AlertsSenderMock_Send_Call {
-	return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", key, alerts)}
+func (_e *AlertsSenderMock_Expecter) Send(ctx interface{}, key interface{}, alerts interface{}) *AlertsSenderMock_Send_Call {
+	return &AlertsSenderMock_Send_Call{Call: _e.mock.On("Send", ctx, key, alerts)}
 }
 
-func (_c *AlertsSenderMock_Send_Call) Run(run func(key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
+func (_c *AlertsSenderMock_Send_Call) Run(run func(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
 	_c.Call.Run(func(args mock.Arguments) {
-		run(args[0].(models.AlertRuleKey), args[1].(definitions.PostableAlerts))
+		run(args[0].(context.Context), args[1].(models.AlertRuleKey), args[2].(definitions.PostableAlerts))
 	})
 	return _c
 }
@@ -50,3 +53,22 @@ func (_c *AlertsSenderMock_Send_Call) Return() *AlertsSenderMock_Send_Call {
 	_c.Call.Return()
 	return _c
 }
+
+func (_c *AlertsSenderMock_Send_Call) RunAndReturn(run func(context.Context, models.AlertRuleKey, definitions.PostableAlerts)) *AlertsSenderMock_Send_Call {
+	_c.Call.Return(run)
+	return _c
+}
+
+// NewAlertsSenderMock creates a new instance of AlertsSenderMock. It also registers a testing interface on the mock and a cleanup function to assert the mocks expectations.
+// The first argument is typically a *testing.T value.
+func NewAlertsSenderMock(t interface {
+	mock.TestingT
+	Cleanup(func())
+}) *AlertsSenderMock {
+	mock := &AlertsSenderMock{}
+	mock.Mock.Test(t)
+
+	t.Cleanup(func() { mock.AssertExpectations(t) })
+
+	return mock
+}

pkg/services/ngalert/schedule/schedule.go

@@ -38,7 +38,7 @@ type ScheduleService interface {
 //
 //go:generate mockery --name AlertsSender --structname AlertsSenderMock --inpackage --filename alerts_sender_mock.go --with-expecter
 type AlertsSender interface {
-	Send(key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts)
+	Send(ctx context.Context, key ngmodels.AlertRuleKey, alerts definitions.PostableAlerts)
 }
 
 // RulesStore is a store that provides alert rules for scheduling
@@ -360,7 +360,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 	notify := func(states []state.StateTransition) {
 		expiredAlerts := state.FromAlertsStateToStoppedAlert(states, sch.appURL, sch.clock)
 		if len(expiredAlerts.PostableAlerts) > 0 {
-			sch.alertsSender.Send(key, expiredAlerts)
+			sch.alertsSender.Send(grafanaCtx, key, expiredAlerts)
 		}
 	}
 
@@ -437,7 +437,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 			attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))),
 		))
 		if len(alerts.PostableAlerts) > 0 {
-			sch.alertsSender.Send(key, alerts)
+			sch.alertsSender.Send(ctx, key, alerts)
 		}
 		sendDuration.Observe(sch.clock.Now().Sub(start).Seconds())
 	}

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -59,7 +59,7 @@ func TestProcessTicks(t *testing.T) {
 	mockedClock := clock.NewMock()
 
 	notifier := &AlertsSenderMock{}
-	notifier.EXPECT().Send(mock.Anything, mock.Anything).Return()
+	notifier.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return()
 
 	appUrl := &url.URL{
 		Scheme: "http",
@@ -578,7 +578,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 		updateChan := make(chan ruleVersionAndPauseStatus)
 
 		sender := AlertsSenderMock{}
-		sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
+		sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
 
 		sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
 		ruleStore.PutRule(context.Background(), rule)
@@ -645,8 +645,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 
 			require.Empty(t, sch.stateManager.GetStatesForRuleUID(rule.OrgID, rule.UID))
 			sender.AssertNumberOfCalls(t, "Send", 1)
-			args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
-			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
+			args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
+			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
 			require.Len(t, args.PostableAlerts, expectedToBeSent)
 		})
 	})
@@ -659,7 +659,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 		evalAppliedChan := make(chan time.Time)
 
 		sender := AlertsSenderMock{}
-		sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
+		sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
 
 		sch, ruleStore, _, reg := createSchedule(evalAppliedChan, &sender)
 		ruleStore.PutRule(context.Background(), rule)
@@ -748,8 +748,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 
 		t.Run("it should send special alert DatasourceError", func(t *testing.T) {
 			sender.AssertNumberOfCalls(t, "Send", 1)
-			args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
-			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
+			args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
+			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
 			assert.Len(t, args.PostableAlerts, 1)
 			assert.Equal(t, state.ErrorAlertName, args.PostableAlerts[0].Labels[prometheusModel.AlertNameLabel])
 		})
@@ -764,7 +764,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 			evalAppliedChan := make(chan time.Time)
 
 			sender := AlertsSenderMock{}
-			sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
+			sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
 
 			sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
 			ruleStore.PutRule(context.Background(), rule)
@@ -783,8 +783,8 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 			waitForTimeChannel(t, evalAppliedChan)
 
 			sender.AssertNumberOfCalls(t, "Send", 1)
-			args, ok := sender.Calls[0].Arguments[1].(definitions.PostableAlerts)
-			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[1]))
+			args, ok := sender.Calls[0].Arguments[2].(definitions.PostableAlerts)
+			require.Truef(t, ok, fmt.Sprintf("expected argument of function was supposed to be 'definitions.PostableAlerts' but got %T", sender.Calls[0].Arguments[2]))
 
 			require.Len(t, args.PostableAlerts, 1)
 		})
@@ -797,7 +797,7 @@ func TestSchedule_ruleRoutine(t *testing.T) {
 		evalAppliedChan := make(chan time.Time)
 
 		sender := AlertsSenderMock{}
-		sender.EXPECT().Send(rule.GetKey(), mock.Anything).Return()
+		sender.EXPECT().Send(mock.Anything, rule.GetKey(), mock.Anything).Return()
 
 		sch, ruleStore, _, _ := createSchedule(evalAppliedChan, &sender)
 		ruleStore.PutRule(context.Background(), rule)
@@ -873,7 +873,7 @@ func setupScheduler(t *testing.T, rs *fakeRulesStore, is *state.FakeInstanceStor
 
 	if senderMock == nil {
 		senderMock = &AlertsSenderMock{}
-		senderMock.EXPECT().Send(mock.Anything, mock.Anything).Return()
+		senderMock.EXPECT().Send(mock.Anything, mock.Anything, mock.Anything).Return()
 	}
 
 	cfg := setting.UnifiedAlertingSettings{

pkg/services/ngalert/sender/router.go

@@ -288,7 +288,7 @@ func (d *AlertsRouter) buildExternalURL(ds *datasources.DataSource) (string, err
 		password, parsed.Host, parsed.Path, parsed.RawQuery), nil
 }
 
-func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.PostableAlerts) {
+func (d *AlertsRouter) Send(ctx context.Context, key models.AlertRuleKey, alerts definitions.PostableAlerts) {
 	logger := d.logger.New(key.LogContext()...)
 	if len(alerts.PostableAlerts) == 0 {
 		logger.Info("No alerts to notify about")
@@ -304,7 +304,7 @@ func (d *AlertsRouter) Send(key models.AlertRuleKey, alerts definitions.Postable
 		n, err := d.multiOrgNotifier.AlertmanagerFor(key.OrgID)
 		if err == nil {
 			localNotifierExist = true
-			if err := n.PutAlerts(alerts); err != nil {
+			if err := n.PutAlerts(ctx, alerts); err != nil {
 				logger.Error("Failed to put alerts in the local notifier", "count", len(alerts.PostableAlerts), "error", err)
 			}
 		} else {

pkg/services/ngalert/sender/router_test.go

@@ -85,7 +85,7 @@ func TestIntegrationSendingToExternalAlertmanager(t *testing.T) {
 		alerts.PostableAlerts = append(alerts.PostableAlerts, alert)
 	}
 
-	alertsRouter.Send(ruleKey, alerts)
+	alertsRouter.Send(context.Background(), ruleKey, alerts)
 
 	// Eventually, our Alertmanager should have received at least one alert.
 	assertAlertsDelivered(t, fakeAM, expected)
@@ -189,8 +189,8 @@ func TestIntegrationSendingToExternalAlertmanager_WithMultipleOrgs(t *testing.T)
 		alerts2.PostableAlerts = append(alerts2.PostableAlerts, alert)
 	}
 
-	alertsRouter.Send(ruleKey1, alerts1)
-	alertsRouter.Send(ruleKey2, alerts2)
+	alertsRouter.Send(context.Background(), ruleKey1, alerts1)
+	alertsRouter.Send(context.Background(), ruleKey2, alerts2)
 
 	assertAlertsDelivered(t, fakeAM, expected)
 
@@ -315,7 +315,7 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 		expected = append(expected, &alert)
 		alerts.PostableAlerts = append(alerts.PostableAlerts, alert)
 	}
-	alertsRouter.Send(ruleKey, alerts)
+	alertsRouter.Send(context.Background(), ruleKey, alerts)
 
 	// Eventually, our Alertmanager should have received at least one alert.
 	assertAlertsDelivered(t, fakeAM, expected)
@@ -347,11 +347,11 @@ func TestChangingAlertmanagersChoice(t *testing.T) {
 	assertAlertmanagersStatusForOrg(t, alertsRouter, ruleKey.OrgID, 1, 0)
 	require.Equal(t, models.InternalAlertmanager, alertsRouter.sendAlertsTo[ruleKey.OrgID])
 
-	alertsRouter.Send(ruleKey, alerts)
+	alertsRouter.Send(context.Background(), ruleKey, alerts)
 
 	am, err := moa.AlertmanagerFor(ruleKey.OrgID)
 	require.NoError(t, err)
-	actualAlerts, err := am.GetAlerts(true, true, true, nil, "")
+	actualAlerts, err := am.GetAlerts(context.Background(), true, true, true, nil, "")
 	require.NoError(t, err)
 	require.Len(t, actualAlerts, len(expected))
 }