diff --git a/pkg/services/alerting/engine.go b/pkg/services/alerting/engine.go index 4d20b13ad73..6061cf42d03 100644 --- a/pkg/services/alerting/engine.go +++ b/pkg/services/alerting/engine.go @@ -95,10 +95,11 @@ func (e *Engine) executeJob(job *AlertJob) { select { case <-time.After(time.Second * 5): e.resultQueue <- &AlertResult{ - State: alertstates.Pending, - Duration: float64(time.Since(now).Nanoseconds()) / float64(1000000), - Error: fmt.Errorf("Timeout"), - AlertJob: job, + State: alertstates.Pending, + Duration: float64(time.Since(now).Nanoseconds()) / float64(1000000), + Error: fmt.Errorf("Timeout"), + AlertJob: job, + ExeuctionTime: time.Now(), } e.log.Debug("Job Execution timeout", "alertRuleId", job.Rule.Id) case result := <-resultChan: diff --git a/pkg/services/alerting/handler.go b/pkg/services/alerting/handler.go index 68a6f08931d..1dfc6cd2cc4 100644 --- a/pkg/services/alerting/handler.go +++ b/pkg/services/alerting/handler.go @@ -2,6 +2,7 @@ package alerting import ( "fmt" + "time" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/log" @@ -28,9 +29,10 @@ func (e *HandlerImpl) Execute(job *AlertJob, resultQueue chan *AlertResult) { timeSeries, err := e.executeQuery(job) if err != nil { resultQueue <- &AlertResult{ - Error: err, - State: alertstates.Pending, - AlertJob: job, + Error: err, + State: alertstates.Pending, + AlertJob: job, + ExeuctionTime: time.Now(), } } @@ -136,5 +138,5 @@ func (e *HandlerImpl) evaluateRule(rule *AlertRule, series tsdb.TimeSeriesSlice) } } - return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert} + return &AlertResult{State: executionState, Description: "Returned " + executionState, TriggeredAlerts: triggeredAlert, ExeuctionTime: time.Now()} } diff --git a/pkg/services/alerting/models.go b/pkg/services/alerting/models.go index 3e0aefd477f..aff42603412 100644 --- a/pkg/services/alerting/models.go +++ b/pkg/services/alerting/models.go @@ -1,5 +1,7 @@ package alerting +import "time" + type AlertJob struct { Offset int64 Delay bool @@ -28,6 +30,7 @@ type AlertResult struct { Description string Error error AlertJob *AlertJob + ExeuctionTime time.Time } type TriggeredAlert struct { diff --git a/pkg/services/alerting/result_handler.go b/pkg/services/alerting/result_handler.go index 97ee346fa87..27f2a4024fe 100644 --- a/pkg/services/alerting/result_handler.go +++ b/pkg/services/alerting/result_handler.go @@ -55,13 +55,15 @@ func (handler *ResultHandlerImpl) shouldUpdateState(result *AlertResult) bool { return false } - now := time.Now() - if query.Result == nil { return true } - olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + //now := time.Now() + //olderThen15Min := query.Result.Created.Before(now.Add(time.Minute * -15)) + lastExecution := query.Result.Created + asdf := result.ExeuctionTime.Add(time.Minute * -15) + olderThen15Min := lastExecution.Before(asdf) changedState := query.Result.NewState != result.State return changedState || olderThen15Min diff --git a/pkg/services/alerting/result_handler_test.go b/pkg/services/alerting/result_handler_test.go index bd492259b84..f44049ecb6d 100644 --- a/pkg/services/alerting/result_handler_test.go +++ b/pkg/services/alerting/result_handler_test.go @@ -51,6 +51,7 @@ func TestAlertResultHandler(t *testing.T) { Created: now.Add(time.Minute * -30), } mockResult.State = alertstates.Critical + mockResult.ExeuctionTime = time.Now() So(resultHandler.shouldUpdateState(mockResult), ShouldBeTrue) }) })