Alerting: Use Mimir image to run integration tests for the remote Alertmanager (#76608)

* Alerting: Use Mimir image to run integration tests for the remote Alertmanager

* skip integration test when running all tests

* skipping integration test when no Alertmanager URL is provided

* fix bad host for mimir_backend

* remove basic auth testing until we have an nginx image in our CI
pull/75844/head
Santiago 2 years ago committed by GitHub
parent 67e2430197
commit 7d9b2c73c7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 34
      .drone.yml
  2. 7
      Makefile
  3. 102
      pkg/services/ngalert/notifier/external_alertmanager_test.go
  4. 154
      pkg/services/ngalert/notifier/fake/external_alertmanager_fake.go
  5. 2
      scripts/drone/services/services.star
  6. 10
      scripts/drone/steps/lib.star

@ -800,7 +800,7 @@ services:
- /bin/mimir -target=backend
environment: {}
image: grafana/mimir:latest
name: mimir
name: mimir_backend
- environment: {}
image: redis:6.2.11-alpine
name: redis
@ -966,16 +966,20 @@ steps:
image: golang:1.20.10-alpine
name: memcached-integration-tests
- commands:
- dockerize -wait tcp://mimir:8080 -timeout 120s
- dockerize -wait tcp://mimir_backend:8080 -timeout 120s
image: jwilder/dockerize:0.6.1
name: wait-for-remote-alertmanager
- commands:
- apk add --update build-base
- go clean -testcache
- go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
- go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
depends_on:
- wire-install
- wait-for-remote-alertmanager
environment:
AM_PASSWORD: test
AM_TENANT_ID: test
AM_URL: http://mimir_backend:8080
image: golang:1.20.10-alpine
name: remote-alertmanager-integration-tests
trigger:
@ -1178,7 +1182,7 @@ services:
- /bin/mimir -target=backend
environment: {}
image: grafana/mimir:latest
name: mimir
name: mimir_backend
- environment: {}
image: redis:6.2.11-alpine
name: redis
@ -2103,7 +2107,7 @@ services:
- /bin/mimir -target=backend
environment: {}
image: grafana/mimir:latest
name: mimir
name: mimir_backend
- environment: {}
image: redis:6.2.11-alpine
name: redis
@ -2248,16 +2252,20 @@ steps:
image: golang:1.20.10-alpine
name: memcached-integration-tests
- commands:
- dockerize -wait tcp://mimir:8080 -timeout 120s
- dockerize -wait tcp://mimir_backend:8080 -timeout 120s
image: jwilder/dockerize:0.6.1
name: wait-for-remote-alertmanager
- commands:
- apk add --update build-base
- go clean -testcache
- go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
- go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
depends_on:
- wire-install
- wait-for-remote-alertmanager
environment:
AM_PASSWORD: test
AM_TENANT_ID: test
AM_URL: http://mimir_backend:8080
image: golang:1.20.10-alpine
name: remote-alertmanager-integration-tests
trigger:
@ -3762,7 +3770,7 @@ services:
- /bin/mimir -target=backend
environment: {}
image: grafana/mimir:latest
name: mimir
name: mimir_backend
- environment: {}
image: redis:6.2.11-alpine
name: redis
@ -3900,16 +3908,20 @@ steps:
image: golang:1.20.10-alpine
name: memcached-integration-tests
- commands:
- dockerize -wait tcp://mimir:8080 -timeout 120s
- dockerize -wait tcp://mimir_backend:8080 -timeout 120s
image: jwilder/dockerize:0.6.1
name: wait-for-remote-alertmanager
- commands:
- apk add --update build-base
- go clean -testcache
- go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...
- go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...
depends_on:
- wire-install
- wait-for-remote-alertmanager
environment:
AM_PASSWORD: test
AM_TENANT_ID: test
AM_URL: http://mimir_backend:8080
image: golang:1.20.10-alpine
name: remote-alertmanager-integration-tests
trigger:
@ -4598,6 +4610,6 @@ kind: secret
name: gcr_credentials
---
kind: signature
hmac: ff105572d451a06880931bc5d3abdb86e50161eb84091e09118d9bf6a229e39b
hmac: d6bd1e6c990959426e575500bd89b4e28cdbc991f245e0723dc912ccc4460470
...

@ -164,6 +164,13 @@ test-go-integration: ## Run integration tests for backend with flags.
@echo "test backend integration tests"
$(GO) test -count=1 -run "^TestIntegration" -covermode=atomic -timeout=5m $(GO_INTEGRATION_TESTS)
.PHONY: test-go-integration-alertmanager
test-go-integration-alertmanager: ## Run integration tests for the remote alertmanager (config taken from the mimir_backend block).
@echo "test remote alertmanager integration tests"
$(GO) clean -testcache
AM_URL=http://localhost:8080 AM_TENANT_ID=test AM_PASSWORD=test \
$(GO) test -count=1 -run "^TestIntegrationRemoteAlertmanager" -covermode=atomic -timeout=5m ./pkg/services/ngalert/notifier/...
.PHONY: test-go-integration-postgres
test-go-integration-postgres: devenv-postgres ## Run integration tests for postgres backend with flags.
@echo "test backend integration postgres tests"

@ -2,11 +2,14 @@ package notifier
import (
"context"
"math/rand"
"os"
"testing"
"time"
"github.com/go-openapi/strfmt"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
amfake "github.com/grafana/grafana/pkg/services/ngalert/notifier/fake"
"github.com/grafana/grafana/pkg/util"
amv2 "github.com/prometheus/alertmanager/api/v2/models"
"github.com/stretchr/testify/require"
)
@ -85,88 +88,103 @@ func TestNewExternalAlertmanager(t *testing.T) {
}
}
func TestSilences(t *testing.T) {
const (
tenantID = "1"
password = "password"
)
fakeAm := amfake.NewFakeExternalAlertmanager(t, tenantID, password)
func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
amURL, ok := os.LookupEnv("AM_URL")
if !ok {
t.Skip("No Alertmanager URL provided")
}
tenantID := os.Getenv("AM_TENANT_ID")
password := os.Getenv("AM_PASSWORD")
// Using a wrong password should cause an error.
cfg := externalAlertmanagerConfig{
URL: fakeAm.Server.URL + "/alertmanager",
URL: amURL + "/alertmanager",
TenantID: tenantID,
BasicAuthPassword: "wrongpassword",
BasicAuthPassword: password,
DefaultConfig: validConfig,
}
am, err := newExternalAlertmanager(cfg, 1)
require.NoError(t, err)
_, err = am.ListSilences(context.Background(), []string{})
require.NotNil(t, err)
// Using the correct password should make the request succeed.
cfg.BasicAuthPassword = password
am, err = newExternalAlertmanager(cfg, 1)
require.NoError(t, err)
// We should have no silences at first.
silences, err := am.ListSilences(context.Background(), []string{})
require.NoError(t, err)
require.Equal(t, 0, len(silences))
// Creating a silence should succeed.
testSilence := createSilence("test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
silenceID, err := am.CreateSilence(context.Background(), &testSilence)
testSilence := genSilence("test")
id, err := am.CreateSilence(context.Background(), &testSilence)
require.NoError(t, err)
require.NotEmpty(t, silenceID)
require.NotEmpty(t, id)
testSilence.ID = id
// We should be able to retrieve a specific silence.
silence, err := am.GetSilence(context.Background(), silenceID)
silence, err := am.GetSilence(context.Background(), testSilence.ID)
require.NoError(t, err)
require.Equal(t, *testSilence.Comment, *silence.Comment)
require.Equal(t, *testSilence.CreatedBy, *silence.CreatedBy)
require.Equal(t, *testSilence.StartsAt, *silence.StartsAt)
require.Equal(t, *testSilence.EndsAt, *silence.EndsAt)
require.Equal(t, testSilence.Matchers, silence.Matchers)
require.Equal(t, testSilence.ID, *silence.ID)
// Trying to retrieve a non-existing silence should fail.
_, err = am.GetSilence(context.Background(), "invalid")
_, err = am.GetSilence(context.Background(), util.GenerateShortUID())
require.Error(t, err)
// After creating another silence, the total amount should be 2.
testSilence2 := createSilence("another test comment", "1", amv2.Matchers{}, strfmt.NewDateTime(), strfmt.NewDateTime())
silenceID2, err := am.CreateSilence(context.Background(), &testSilence2)
testSilence2 := genSilence("test")
id, err = am.CreateSilence(context.Background(), &testSilence2)
require.NoError(t, err)
require.NotEmpty(t, silenceID2)
require.NotEmpty(t, id)
testSilence2.ID = id
silences, err = am.ListSilences(context.Background(), []string{})
require.NoError(t, err)
require.Equal(t, 2, len(silences))
require.True(t, *silences[0].ID == silenceID || *silences[0].ID == silenceID2)
require.True(t, *silences[1].ID == silenceID || *silences[1].ID == silenceID2)
require.True(t, *silences[0].ID == testSilence.ID || *silences[0].ID == testSilence2.ID)
require.True(t, *silences[1].ID == testSilence.ID || *silences[1].ID == testSilence2.ID)
// After deleting one of those silences, the total amount should be 2.
err = am.DeleteSilence(context.Background(), silenceID)
// After deleting one of those silences, the total amount should be 2 but one of those should be expired.
err = am.DeleteSilence(context.Background(), testSilence.ID)
require.NoError(t, err)
silences, err = am.ListSilences(context.Background(), []string{})
require.NoError(t, err)
require.Equal(t, 1, len(silences))
// Trying to delete the same error should fail.
err = am.DeleteSilence(context.Background(), silenceID)
require.NotNil(t, err)
for _, s := range silences {
if *s.ID == testSilence.ID {
require.Equal(t, *s.Status.State, "expired")
} else {
require.Equal(t, *s.Status.State, "pending")
}
}
func createSilence(comment, createdBy string, matchers amv2.Matchers, startsAt, endsAt strfmt.DateTime) apimodels.PostableSilence {
// When deleting the other silence, both should be expired.
err = am.DeleteSilence(context.Background(), testSilence2.ID)
require.NoError(t, err)
silences, err = am.ListSilences(context.Background(), []string{})
require.NoError(t, err)
require.Equal(t, *silences[0].Status.State, "expired")
require.Equal(t, *silences[1].Status.State, "expired")
}
func genSilence(createdBy string) apimodels.PostableSilence {
starts := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+1) * time.Second))
ends := strfmt.DateTime(time.Now().Add(time.Duration(rand.Int63n(9)+10) * time.Second))
comment := "test comment"
isEqual := true
name := "test"
value := "test"
isRegex := false
matchers := amv2.Matchers{&amv2.Matcher{IsEqual: &isEqual, Name: &name, Value: &value, IsRegex: &isRegex}}
return apimodels.PostableSilence{
Silence: amv2.Silence{
Comment: &comment,
CreatedBy: &createdBy,
Matchers: matchers,
StartsAt: &startsAt,
EndsAt: &endsAt,
StartsAt: &starts,
EndsAt: &ends,
},
}
}

@ -1,154 +0,0 @@
package fake
import (
"encoding/json"
"net/http"
"net/http/httptest"
"sync"
"testing"
"github.com/go-openapi/strfmt"
alertingNotify "github.com/grafana/alerting/notify"
"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/web"
"github.com/stretchr/testify/require"
)
type FakeExternalAlertmanager struct {
t *testing.T
mtx sync.RWMutex
tenantID string
password string
Server *httptest.Server
silences alertingNotify.GettableSilences
}
func NewFakeExternalAlertmanager(t *testing.T, tenantID, password string) *FakeExternalAlertmanager {
t.Helper()
am := &FakeExternalAlertmanager{
t: t,
tenantID: tenantID,
password: password,
mtx: sync.RWMutex{},
}
mux := web.New()
mux.SetURLPrefix("/alertmanager/api/")
mux.UseMiddleware(am.basicAuthMiddleware)
mux.UseMiddleware(am.contentTypeJSONMiddleware)
// Routes
mux.Get("/v2/silences", http.HandlerFunc(am.getSilences))
mux.Get("/v2/silence/:silenceID", http.HandlerFunc(am.getSilence))
mux.Post("/v2/silences", http.HandlerFunc(am.postSilence))
mux.Delete("/v2/silence/:silenceID", http.HandlerFunc(am.deleteSilence))
am.Server = httptest.NewServer(mux)
return am
}
func (am *FakeExternalAlertmanager) getSilences(w http.ResponseWriter, r *http.Request) {
am.mtx.RLock()
if err := json.NewEncoder(w).Encode(am.silences); err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
am.mtx.RUnlock()
}
func (am *FakeExternalAlertmanager) getSilence(w http.ResponseWriter, r *http.Request) {
silenceID, ok := web.Params(r)[":silenceID"]
if !ok {
return
}
am.mtx.RLock()
var matching *alertingNotify.GettableSilence
for _, silence := range am.silences {
if *silence.ID == silenceID {
matching = silence
break
}
}
am.mtx.RUnlock()
if matching == nil {
w.WriteHeader(http.StatusNotFound)
return
}
if err := json.NewEncoder(w).Encode(matching); err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
}
func (am *FakeExternalAlertmanager) postSilence(w http.ResponseWriter, r *http.Request) {
var silence definitions.PostableSilence
require.NoError(am.t, json.NewDecoder(r.Body).Decode(&silence))
updatedAt := strfmt.NewDateTime()
id := util.GenerateShortUID()
am.mtx.Lock()
am.silences = append(am.silences, &alertingNotify.GettableSilence{
ID: &id,
UpdatedAt: &updatedAt,
Silence: silence.Silence,
})
am.mtx.Unlock()
res := map[string]string{"silenceID": id}
if err := json.NewEncoder(w).Encode(res); err != nil {
w.WriteHeader(http.StatusInternalServerError)
}
}
func (am *FakeExternalAlertmanager) deleteSilence(w http.ResponseWriter, r *http.Request) {
silenceID, ok := web.Params(r)[":silenceID"]
if !ok {
return
}
am.mtx.Lock()
defer am.mtx.Unlock()
var newSilences []*alertingNotify.GettableSilence
for _, silence := range am.silences {
if *silence.ID != silenceID {
newSilences = append(newSilences, silence)
}
}
if len(newSilences) == len(am.silences) {
w.WriteHeader(http.StatusNotFound)
return
}
am.silences = newSilences
w.WriteHeader(http.StatusOK)
}
func (am *FakeExternalAlertmanager) basicAuthMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
username, password, ok := r.BasicAuth()
if !ok {
w.WriteHeader(http.StatusUnauthorized)
return
}
if username != am.tenantID || password != am.password || r.Header.Get("X-Scope-OrgID") != am.tenantID {
w.WriteHeader(http.StatusForbidden)
return
}
next.ServeHTTP(w, r)
})
}
func (am *FakeExternalAlertmanager) contentTypeJSONMiddleware(next http.Handler) http.Handler {
return http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
w.Header().Set("Content-Type", "application/json")
next.ServeHTTP(w, r)
})
}
func (am *FakeExternalAlertmanager) Close() {
am.Server.Close()
}

@ -54,7 +54,7 @@ def integration_test_services():
"commands": ["docker-entrypoint.sh mysqld --default-authentication-plugin=mysql_native_password"],
},
{
"name": "mimir",
"name": "mimir_backend",
"image": images["mimir"],
"environment": {},
"commands": ["/bin/mimir -target=backend"],

@ -964,10 +964,16 @@ def redis_integration_tests_steps():
def remote_alertmanager_integration_tests_steps():
cmds = [
"go clean -testcache",
"go test -run IntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/...",
"go test -run TestIntegrationRemoteAlertmanager -covermode=atomic -timeout=2m ./pkg/services/ngalert/notifier/...",
]
return integration_tests_steps("remote-alertmanager", cmds, "mimir", "8080", None)
environment = {
"AM_TENANT_ID": "test",
"AM_PASSWORD": "test",
"AM_URL": "http://mimir_backend:8080",
}
return integration_tests_steps("remote-alertmanager", cmds, "mimir_backend", "8080", environment = environment)
def memcached_integration_tests_steps():
cmds = [

Loading…
Cancel
Save