From ac832c157e12f6c7c65f74f73d5ea92541eb7d8f Mon Sep 17 00:00:00 2001 From: Alexander Akhmetov Date: Fri, 20 Jun 2025 14:31:21 +0200 Subject: [PATCH] Alerting: Maintain endsAt for missing alert instances that are not stale yet (#107011) --- pkg/services/ngalert/state/manager.go | 4 ++++ pkg/services/ngalert/state/manager_private_test.go | 14 +++++++------- 2 files changed, 11 insertions(+), 7 deletions(-) diff --git a/pkg/services/ngalert/state/manager.go b/pkg/services/ngalert/state/manager.go index 0549ba54c02..a228f2bc151 100644 --- a/pkg/services/ngalert/state/manager.go +++ b/pkg/services/ngalert/state/manager.go @@ -566,6 +566,10 @@ func (st *Manager) processMissingSeriesStates(logger log.Logger, evaluatedAt tim } staleStatesCount++ + } else if s.State == eval.Alerting { + // We need to update EndsAt for the state so that it will not be resolved by the + // Alertmanager automatically. + s.Maintain(alertRule.IntervalSeconds, evaluatedAt) } record := StateTransition{ diff --git a/pkg/services/ngalert/state/manager_private_test.go b/pkg/services/ngalert/state/manager_private_test.go index ede0bcc0877..471e747d88e 100644 --- a/pkg/services/ngalert/state/manager_private_test.go +++ b/pkg/services/ngalert/state/manager_private_test.go @@ -675,7 +675,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -1055,7 +1055,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -1133,7 +1133,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t3.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -1679,7 +1679,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -2379,7 +2379,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -2962,7 +2962,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { StateReason: ngmodels.StateReasonNoData, LatestResult: newEvaluation(t1, eval.NoData), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1, @@ -3226,7 +3226,7 @@ func TestProcessEvalResults_StateTransitions(t *testing.T) { State: eval.Alerting, LatestResult: newEvaluation(t1, eval.Alerting), StartsAt: t1, - EndsAt: t1.Add(ResendDelay * 4), + EndsAt: t2.Add(ResendDelay * 4), FiredAt: &t1, LastEvaluationTime: t1, LastSentAt: &t1,