apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Alerting: Test MOA in remote secondary mode (#79828)

Browse Source
pull/80034/head
Santiago 2 years ago committed by GitHub
parent 2b02ada7ad
commit 1f6575e65e
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 281 additions and 4 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      pkg/services/ngalert/api/api_alertmanager_test.go
  2. 277
      pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
  3. 4
      pkg/services/ngalert/notifier/testing.go
  4. 2
      pkg/services/ngalert/sender/router_test.go

2
pkg/services/ngalert/api/api_alertmanager_test.go
Unescape View File

@ -679,7 +679,7 @@ func createMultiOrgAlertmanager(t *testing.T) *notifier.MultiOrgAlertmanager {
}, // do not poll in tests.
}
mam, err := notifier.NewMultiOrgAlertmanager(cfg, configStore, &orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
mam, err := notifier.NewMultiOrgAlertmanager(cfg, configStore, orgStore, kvStore, provStore, decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
err = mam.LoadAndSyncAlertmanagersForOrgs(context.Background())
require.NoError(t, err)

277
pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
Unescape View File

@ -0,0 +1,277 @@
package notifier_test
import (
"context"
"encoding/json"
"net/http"
"net/http/httptest"
"testing"
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/notifier"
"github.com/grafana/grafana/pkg/services/ngalert/provisioning"
"github.com/grafana/grafana/pkg/services/ngalert/remote"
remoteClient "github.com/grafana/grafana/pkg/services/ngalert/remote/client"
ngfakes "github.com/grafana/grafana/pkg/services/ngalert/tests/fakes"
"github.com/grafana/grafana/pkg/services/secrets/fakes"
secretsManager "github.com/grafana/grafana/pkg/services/secrets/manager"
"github.com/grafana/grafana/pkg/setting"
"github.com/prometheus/client_golang/prometheus"
"github.com/stretchr/testify/require"
)
func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
ctx := context.Background()
nopLogger := log.NewNopLogger()
tenantID := "testTenantID"
password := "testPassword"
reg := prometheus.NewPedanticRegistry()
m := metrics.NewNGAlert(reg)
// We're gonna use an test server to send configuration and state to.
fakeAM := newFakeRemoteAlertmanager(t, tenantID, password)
testsrv := httptest.NewServer(fakeAM)
// We'll start with the default config and no values for silences and notifications.
kvStore := ngfakes.NewFakeKVStore(t)
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, ""))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, ""))
configStore := notifier.NewFakeConfigStore(t, map[int64]*models.AlertConfiguration{
1: {
OrgID: 1,
AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
CreatedAt: time.Now().Unix(),
Default: true,
},
})
// Create the factory function for the MOA using the forked Alertmanager in remote secondary mode.
override := notifier.WithAlertmanagerOverride(func(factoryFn notifier.OrgAlertmanagerFactory) notifier.OrgAlertmanagerFactory {
return func(ctx context.Context, orgID int64) (notifier.Alertmanager, error) {
// Create internal Alertmanager.
internalAM, err := factoryFn(ctx, orgID)
require.NoError(t, err)
// Create remote Alertmanager.
externalAMCfg := remote.AlertmanagerConfig{
OrgID: 1,
URL: testsrv.URL,
TenantID: tenantID,
BasicAuthPassword: password,
}
// We won't be handling files on disk, we can pass an empty string as workingDirPath.
stateStore := notifier.NewFileStore(orgID, kvStore, "")
remoteAM, err := remote.NewAlertmanager(externalAMCfg, stateStore)
require.NoError(t, err)
// Use both Alertmanager implementations in the forked Alertmanager.
cfg := remote.RemoteSecondaryConfig{
Logger: nopLogger,
OrgID: orgID,
Store: configStore,
// Note that we're setting a sync interval of 10 seconds.
SyncInterval: 10 * time.Second,
}
return remote.NewRemoteSecondaryForkedAlertmanager(cfg, internalAM, remoteAM)
}
})
cfg := &setting.Cfg{
DataPath: t.TempDir(),
UnifiedAlerting: setting.UnifiedAlertingSettings{
AlertmanagerConfigPollInterval: 3 * time.Minute,
DefaultConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
}, // do not poll in tests.
}
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
moa, err := notifier.NewMultiOrgAlertmanager(
cfg,
configStore,
notifier.NewFakeOrgStore(t, []int64{1}),
kvStore,
provisioning.NewFakeProvisioningStore(),
secretsService.GetDecryptedValue,
m.GetMultiOrgAlertmanagerMetrics(),
nil,
nopLogger,
secretsService,
override,
)
require.NoError(t, err)
// It should send config and state on startup.
var lastConfig *remoteClient.UserGrafanaConfig
var lastState *remoteClient.UserGrafanaState
{
// We should start with no config and no state in the external Alertmanager.
require.Empty(t, fakeAM.config)
require.Empty(t, fakeAM.state)
// On the first sync (startup), both config and state should be updated.
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
require.NotEmpty(t, fakeAM.config)
require.NotEmpty(t, fakeAM.state)
lastConfig, lastState = fakeAM.config, fakeAM.state
}
// It should send config and state on an interval.
{
// Let's change the configuration and state.
require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: validConfig,
OrgID: 1,
LastApplied: time.Now().Unix(),
}))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdAo=")) // base64-encoded string "test"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdAo=")) // base64-encoded string "test"
// The sync interval (10s) has not elapsed yet, syncing should have no effect.
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
require.Equal(t, fakeAM.config, lastConfig)
require.Equal(t, fakeAM.state, lastState)
// Syncing after the sync interval elapses should update both config and state.
require.Eventually(t, func() bool {
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(ctx))
return fakeAM.config != lastConfig && fakeAM.state != lastState
}, 15*time.Second, 300*time.Millisecond)
lastConfig, lastState = fakeAM.config, fakeAM.state
}
// It should send config and state on shutdown.
{
// Let's change the configuration and state again.
require.NoError(t, configStore.SaveAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
AlertmanagerConfiguration: setting.GetAlertmanagerDefaultConfiguration(),
Default: true,
OrgID: 1,
LastApplied: time.Now().Unix(),
}))
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.SilencesFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2"
require.NoError(t, kvStore.Set(ctx, 1, "alertmanager", notifier.NotificationLogFilename, "dGVzdC0yCg==")) // base64-encoded string "test-2"
// Both state and config should be updated when shutting the Alertmanager down.
moa.StopAndWait()
require.NotEqual(t, fakeAM.config, lastConfig)
require.NotEqual(t, fakeAM.state, lastState)
}
}
func newFakeRemoteAlertmanager(t *testing.T, user, pass string) *fakeRemoteAlertmanager {
return &fakeRemoteAlertmanager{
t: t,
username: user,
password: pass,
}
}
type fakeRemoteAlertmanager struct {
t *testing.T
config *remoteClient.UserGrafanaConfig
state *remoteClient.UserGrafanaState
username string
password string
}
// ServeHTTP handles all routes we need for getting and setting state and config.
func (f *fakeRemoteAlertmanager) ServeHTTP(w http.ResponseWriter, r *http.Request) {
w.Header().Add("Content-Type", "application/json")
// Check that basic auth is in place.
user, pass, ok := r.BasicAuth()
require.True(f.t, ok)
require.Equal(f.t, f.username, user)
require.Equal(f.t, f.password, pass)
switch r.Method {
// GET routes
case http.MethodGet:
switch r.RequestURI {
case "/alertmanager/-/ready":
// Make the readiness check succeed.
w.WriteHeader(http.StatusOK)
case "/api/v1/grafana/config":
f.getConfig(w)
case "/api/v1/grafana/state":
f.getState(w)
default:
w.WriteHeader(http.StatusNotFound)
}
// POST routes
case http.MethodPost:
switch r.RequestURI {
case "/api/v1/grafana/config":
f.postConfig(w, r)
case "/api/v1/grafana/state":
f.postState(w, r)
default:
w.WriteHeader(http.StatusNotFound)
}
}
}
type response struct {
Data any `json:"data"`
Status string `json:"status"`
}
func (f *fakeRemoteAlertmanager) postConfig(w http.ResponseWriter, r *http.Request) {
var cfg remoteClient.UserGrafanaConfig
require.NoError(f.t, json.NewDecoder(r.Body).Decode(&cfg))
f.config = &cfg
w.WriteHeader(http.StatusCreated)
require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"}))
}
func (f *fakeRemoteAlertmanager) getConfig(w http.ResponseWriter) {
res := response{
Data: f.config,
Status: "success",
}
require.NoError(f.t, json.NewEncoder(w).Encode(res))
}
func (f *fakeRemoteAlertmanager) postState(w http.ResponseWriter, r *http.Request) {
var state remoteClient.UserGrafanaState
require.NoError(f.t, json.NewDecoder(r.Body).Decode(&state))
f.state = &state
w.WriteHeader(http.StatusCreated)
require.NoError(f.t, json.NewEncoder(w).Encode(response{Status: "success"}))
}
func (f *fakeRemoteAlertmanager) getState(w http.ResponseWriter) {
res := response{
Data: f.state,
Status: "success",
}
require.NoError(f.t, json.NewEncoder(w).Encode(res))
}
var validConfig = `{
"template_files": {
"a": "template"
},
"alertmanager_config": {
"route": {
"receiver": "grafana-default-email"
},
"receivers": [{
"name": "grafana-default-email",
"grafana_managed_receiver_configs": [{
"uid": "",
"name": "email receiver",
"type": "email",
"isDefault": true,
"settings": {
"addresses": "<example@email.com>"
}
}]
}]
}
}`

4
pkg/services/ngalert/notifier/testing.go
Unescape View File

@ -180,10 +180,10 @@ type FakeOrgStore struct {
orgs []int64
}
func NewFakeOrgStore(t *testing.T, orgs []int64) FakeOrgStore {
func NewFakeOrgStore(t *testing.T, orgs []int64) *FakeOrgStore {
t.Helper()
return FakeOrgStore{
return &FakeOrgStore{
orgs: orgs,
}
}

2
pkg/services/ngalert/sender/router_test.go
Unescape View File

@ -411,7 +411,7 @@ func createMultiOrgAlertmanager(t *testing.T, orgs []int64) *notifier.MultiOrgAl
m := metrics.NewNGAlert(registry)
secretsService := secretsManager.SetupTestService(t, fake_secrets.NewFakeSecretsStore())
decryptFn := secretsService.GetDecryptedValue
moa, err := notifier.NewMultiOrgAlertmanager(cfg, cfgStore, &orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
moa, err := notifier.NewMultiOrgAlertmanager(cfg, cfgStore, orgStore, kvStore, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
require.NoError(t, err)
require.NoError(t, moa.LoadAndSyncAlertmanagersForOrgs(context.Background()))
require.Eventually(t, func() bool {

Write Preview
Loading…
Cancel
Save