apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

notifier: fix flaky TestStop_DrainingEnabled and TestStop_DrainingDisabled race conditions (#17938)

Browse Source 
Fix flaky TestStop_DrainingEnabled and TestStop_DrainingDisabled tests.
The tests used real HTTP servers and real time, making them susceptible to
race conditions and timing-dependent failures.
The solution is to convert both tests to use synctest for deterministic fake time.

---------

Signed-off-by: Arve Knudsen <arve.knudsen@gmail.com>
pull/17962/head
Arve Knudsen 3 months ago committed by GitHub
parent 00a7faa2e3
commit 020a0b30a0
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
1 changed files with 112 additions and 130 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 242
      notifier/manager_test.go

242
notifier/manager_test.go
Unescape View File

@ -831,171 +831,153 @@ func TestHangingNotifier(t *testing.T) {
}
func TestStop_DrainingDisabled(t *testing.T) {
releaseReceiver := make(chan struct{})
receiverReceivedRequest := make(chan struct{}, 2)
alertsReceived := atomic.NewInt64(0)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
// Let the test know we've received a request.
receiverReceivedRequest <- struct{}{}
var alerts []*Alert
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
err = json.Unmarshal(b, &alerts)
require.NoError(t, err)
synctest.Test(t, func(t *testing.T) {
const alertmanagerURL = "http://alertmanager:9093/api/v2/alerts"
alertsReceived.Add(int64(len(alerts)))
handlerStarted := make(chan struct{})
alertsReceived := atomic.NewInt64(0)
// Wait for the test to release us.
<-releaseReceiver
// Fake Do function that simulates a hanging alertmanager that times out.
fakeDo := func(ctx context.Context, _ *http.Client, req *http.Request) (*http.Response, error) {
var alerts []*Alert
b, err := io.ReadAll(req.Body)
if err != nil {
return nil, fmt.Errorf("read request body: %w", err)
}
if err := json.Unmarshal(b, &alerts); err != nil {
return nil, fmt.Errorf("unmarshal request body: %w", err)
}
alertsReceived.Add(int64(len(alerts)))
w.WriteHeader(http.StatusOK)
}))
defer func() {
server.Close()
}()
// Signal arrival, then block until context times out.
handlerStarted <- struct{}{}
<-ctx.Done()
reg := prometheus.NewRegistry()
m := NewManager(
&Options{
QueueCapacity: 10,
DrainOnShutdown: false,
Registerer: reg,
},
model.UTF8Validation,
nil,
)
return nil, ctx.Err()
}
m.alertmanagers = make(map[string]*alertmanagerSet)
reg := prometheus.NewRegistry()
m := NewManager(
&Options{
QueueCapacity: 10,
DrainOnShutdown: false,
Registerer: reg,
Do: fakeDo,
},
model.UTF8Validation,
nil,
)
am1Cfg := config.DefaultAlertmanagerConfig
am1Cfg.Timeout = model.Duration(time.Second)
m.alertmanagers["1"] = newTestAlertmanagerSet(&am1Cfg, nil, m.opts, m.metrics, server.URL)
m.alertmanagers = make(map[string]*alertmanagerSet)
for _, ams := range m.alertmanagers {
ams.startSendLoops(ams.ams)
}
am1Cfg := config.DefaultAlertmanagerConfig
am1Cfg.Timeout = model.Duration(time.Second)
m.alertmanagers["1"] = newTestAlertmanagerSet(&am1Cfg, nil, m.opts, m.metrics, alertmanagerURL)
notificationManagerStopped := make(chan struct{})
for _, ams := range m.alertmanagers {
ams.startSendLoops(ams.ams)
}
go func() {
defer close(notificationManagerStopped)
m.Run(nil)
}()
// This will be waited on automatically when synctest.Test exits.
go m.Run(nil)
// Queue two alerts. The first should be immediately sent to the receiver, which should block until we release it later.
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-1")})
// Queue two alerts. The first should be immediately sent to the receiver, which should block until we release it later.
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-1")})
select {
case <-receiverReceivedRequest:
// Nothing more to do.
case <-time.After(time.Second):
require.FailNow(t, "gave up waiting for receiver to receive notification of first alert")
}
// Wait for receiver to get the request.
<-handlerStarted
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-2")})
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-2")})
// Stop the notification manager, pause to allow the shutdown to be observed, and then allow the receiver to proceed.
m.Stop()
time.Sleep(time.Second)
close(releaseReceiver)
// Stop the notification manager, then advance time to trigger the request timeout.
m.Stop()
time.Sleep(time.Second)
// Wait for the notification manager to stop and confirm only the first notification was sent.
// The second notification should be dropped.
select {
case <-notificationManagerStopped:
// Nothing more to do.
case <-time.After(time.Second):
require.FailNow(t, "gave up waiting for notification manager to stop")
}
// Allow goroutines to finish.
synctest.Wait()
require.Equal(t, int64(1), alertsReceived.Load())
// Confirm only the first notification was sent. The second notification should be dropped.
require.Equal(t, int64(1), alertsReceived.Load())
})
}
func TestStop_DrainingEnabled(t *testing.T) {
releaseReceiver := make(chan struct{})
receiverReceivedRequest := make(chan struct{}, 2)
alertsReceived := atomic.NewInt64(0)
server := httptest.NewServer(http.HandlerFunc(func(w http.ResponseWriter, r *http.Request) {
var alerts []*Alert
// Let the test know we've received a request.
receiverReceivedRequest <- struct{}{}
synctest.Test(t, func(t *testing.T) {
const alertmanagerURL = "http://alertmanager:9093/api/v2/alerts"
b, err := io.ReadAll(r.Body)
require.NoError(t, err)
handlerStarted := make(chan struct{}, 1)
alertsReceived := atomic.NewInt64(0)
err = json.Unmarshal(b, &alerts)
require.NoError(t, err)
// Fake Do function that simulates alertmanager responding slowly but successfully.
fakeDo := func(_ context.Context, _ *http.Client, req *http.Request) (*http.Response, error) {
var alerts []*Alert
b, err := io.ReadAll(req.Body)
if err != nil {
return nil, fmt.Errorf("read request body: %w", err)
}
if err := json.Unmarshal(b, &alerts); err != nil {
return nil, fmt.Errorf("unmarshal request body: %w", err)
}
alertsReceived.Add(int64(len(alerts)))
alertsReceived.Add(int64(len(alerts)))
// Signal arrival.
handlerStarted <- struct{}{}
// Wait for the test to release us.
<-releaseReceiver
// Block to allow for alert-2 to be queued while this request is in-flight.
time.Sleep(100 * time.Millisecond)
w.WriteHeader(http.StatusOK)
}))
defer func() {
server.Close()
}()
return &http.Response{
StatusCode: http.StatusOK,
Body: io.NopCloser(bytes.NewBuffer(nil)),
}, nil
}
reg := prometheus.NewRegistry()
m := NewManager(
&Options{
QueueCapacity: 10,
DrainOnShutdown: true,
Registerer: reg,
},
model.UTF8Validation,
nil,
)
reg := prometheus.NewRegistry()
m := NewManager(
&Options{
QueueCapacity: 10,
DrainOnShutdown: true,
Registerer: reg,
Do: fakeDo,
},
model.UTF8Validation,
nil,
)
m.alertmanagers = make(map[string]*alertmanagerSet)
m.alertmanagers = make(map[string]*alertmanagerSet)
am1Cfg := config.DefaultAlertmanagerConfig
am1Cfg.Timeout = model.Duration(time.Second)
m.alertmanagers["1"] = newTestAlertmanagerSet(&am1Cfg, nil, m.opts, m.metrics, server.URL)
am1Cfg := config.DefaultAlertmanagerConfig
am1Cfg.Timeout = model.Duration(time.Second)
m.alertmanagers["1"] = newTestAlertmanagerSet(&am1Cfg, nil, m.opts, m.metrics, alertmanagerURL)
for _, ams := range m.alertmanagers {
ams.startSendLoops(ams.ams)
}
for _, ams := range m.alertmanagers {
ams.startSendLoops(ams.ams)
}
notificationManagerStopped := make(chan struct{})
go m.Run(nil)
go func() {
defer close(notificationManagerStopped)
m.Run(nil)
}()
// Queue two alerts. The first should be immediately sent to the receiver.
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-1")})
// Queue two alerts. The first should be immediately sent to the receiver, which should block until we release it later.
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-1")})
// Wait for receiver to get the first request.
<-handlerStarted
select {
case <-receiverReceivedRequest:
// Nothing more to do.
case <-time.After(time.Second):
require.FailNow(t, "gave up waiting for receiver to receive notification of first alert")
}
// Send second alert while first is still being processed (fakeDo has 100ms delay).
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-2")})
m.Send(&Alert{Labels: labels.FromStrings(labels.AlertName, "alert-2")})
// Stop the notification manager. With DrainOnShutdown=true, this should wait
// for the queue to drain, ensuring both alerts are sent.
m.Stop()
// Stop the notification manager and allow the receiver to proceed.
m.Stop()
close(releaseReceiver)
// Advance time so in-flight requests complete.
time.Sleep(time.Second)
// Wait for the notification manager to stop and confirm both notifications were sent.
select {
case <-notificationManagerStopped:
// Nothing more to do.
case <-time.After(200 * time.Millisecond):
require.FailNow(t, "gave up waiting for notification manager to stop")
}
// Allow goroutines to finish.
synctest.Wait()
require.Equal(t, int64(2), alertsReceived.Load())
// Confirm both notifications were sent.
require.Equal(t, int64(2), alertsReceived.Load())
})
}
// TestQueuesDrainingOnApplyConfig ensures that when an alertmanagerSet disappears after an ApplyConfig(), its

Write Preview
Loading…
Cancel
Save