Alerting: new min_interval_seconds options to enforce a minimum eval frequency (#21188)

* add min_interval_seconds setting to alerting config

It will let operator enforce a minimum time for the scheduler to enqueue evaluations

* Introduce UI modifications

* Update docs

Co-authored-by: Martin <uepoch@users.noreply.github.com>

conf/defaults.ini

@@ -593,6 +593,8 @@ notification_timeout_seconds = 30
 # Default setting for max attempts to sending alert notifications. Default value is 3
 max_attempts = 3
 
+# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
+min_interval_seconds = 1
 
 #################################### Explore #############################
 [explore]

conf/sample.ini

@@ -584,6 +584,9 @@
 # Default setting for max attempts to sending alert notifications. Default value is 3
 ;max_attempts = 3
 
+# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
+;min_interval_seconds = 1
+
 #################################### Explore #############################
 [explore]
 # Enable the Explore section

docs/sources/alerting/rules.md

@@ -45,6 +45,7 @@ Currently only the graph panel supports alert rules.
 ### Name and Evaluation interval
 
 Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
+**Note:** You can set a minimum interval in the `alerting.min_interval_seconds` config field, to set a minimum time between evaluations. Check out the [[configuration]]({{< relref "../installation/configuration.md" >}}#min-interval-seconds) page for more information.
 
 ### For
 

docs/sources/installation/configuration.md

@@ -722,6 +722,12 @@ Default setting for alert notification timeout. Default value is `30`
 
 Default setting for max attempts to sending alert notifications. Default value is `3`
 
+### min_interval_seconds
+
+Default setting for minimum interval between rule evaluations. Default value is `1`
+
+> **Note.** This setting has precedence over each individual rule frequency. Therefore, if a rule frequency is lower than this value, this value will be enforced.
+
 ## [rendering]
 
 Options to configure a remote HTTP image rendering service, e.g. using https://github.com/grafana/grafana-image-renderer.

packages/grafana-runtime/src/config.ts

@@ -34,6 +34,7 @@ export class GrafanaBootConfig {
   alertingEnabled = false;
   alertingErrorOrTimeout = '';
   alertingNoDataOrNullValues = '';
+  alertingMinInterval = 1;
   authProxyEnabled = false;
   exploreEnabled = false;
   ldapEnabled = false;

pkg/api/frontendsettings.go

@@ -176,6 +176,7 @@ func (hs *HTTPServer) getFrontendSettingsMap(c *m.ReqContext) (map[string]interf
 		"alertingEnabled":            setting.AlertingEnabled,
 		"alertingErrorOrTimeout":     setting.AlertingErrorOrTimeout,
 		"alertingNoDataOrNullValues": setting.AlertingNoDataOrNullValues,
+		"alertingMinInterval":        setting.AlertingMinInterval,
 		"exploreEnabled":             setting.ExploreEnabled,
 		"googleAnalyticsId":          setting.GoogleAnalyticsId,
 		"disableLoginForm":           setting.DisableLoginForm,

pkg/services/alerting/scheduler.go

@@ -6,6 +6,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 type schedulerImpl struct {
@@ -61,7 +62,13 @@ func (s *schedulerImpl) Tick(tickTime time.Time, execQueue chan *Job) {
 			continue
 		}
 
-		if now%job.Rule.Frequency == 0 {
+		// Check the job frequency against the minimum interval required
+		interval := job.Rule.Frequency
+		if interval < setting.AlertingMinInterval {
+			interval = setting.AlertingMinInterval
+		}
+
+		if now%interval == 0 {
 			if job.Offset > 0 {
 				job.OffsetWait = true
 			} else {

pkg/setting/setting.go

@@ -201,6 +201,7 @@ var (
 	AlertingEvaluationTimeout   time.Duration
 	AlertingNotificationTimeout time.Duration
 	AlertingMaxAttempts         int
+	AlertingMinInterval         int64
 
 	// Explore UI
 	ExploreEnabled bool
@@ -961,6 +962,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
 	notificationTimeoutSeconds := alerting.Key("notification_timeout_seconds").MustInt64(30)
 	AlertingNotificationTimeout = time.Second * time.Duration(notificationTimeoutSeconds)
 	AlertingMaxAttempts = alerting.Key("max_attempts").MustInt(3)
+	AlertingMinInterval = alerting.Key("min_interval_seconds").MustInt64(1)
 
 	explore := iniFile.Section("explore")
 	ExploreEnabled = explore.Key("enabled").MustBool(true)

public/app/features/alerting/AlertTabCtrl.ts

@@ -12,6 +12,7 @@ import { DataQuery, DataSourceApi } from '@grafana/data';
 import { PanelModel } from 'app/features/dashboard/state';
 import { getDefaultCondition } from './getAlertingValidationMessage';
 import { CoreEvents } from 'app/types';
+import kbn from 'app/core/utils/kbn';
 
 export class AlertTabCtrl {
   panel: PanelModel;
@@ -31,6 +32,9 @@ export class AlertTabCtrl {
   appSubUrl: string;
   alertHistory: any;
   newAlertRuleTag: any;
+  alertingMinIntervalSecs: number;
+  alertingMinInterval: string;
+  frequencyWarning: any;
 
   /** @ngInject */
   constructor(
@@ -51,6 +55,8 @@ export class AlertTabCtrl {
     this.executionErrorModes = alertDef.executionErrorModes;
     this.appSubUrl = config.appSubUrl;
     this.panelCtrl._enableAlert = this.enable;
+    this.alertingMinIntervalSecs = config.alertingMinInterval;
+    this.alertingMinInterval = kbn.secondsToHms(config.alertingMinInterval);
   }
 
   $onInit() {
@@ -178,6 +184,8 @@ export class AlertTabCtrl {
       return;
     }
 
+    this.checkFrequency();
+
     alert.conditions = alert.conditions || [];
     if (alert.conditions.length === 0) {
       alert.conditions.push(getDefaultCondition());
@@ -232,6 +240,23 @@ export class AlertTabCtrl {
     this.panelCtrl.render();
   }
 
+  checkFrequency() {
+    this.frequencyWarning = '';
+
+    try {
+      const frequencySecs = kbn.interval_to_seconds(this.alert.frequency);
+      if (frequencySecs < this.alertingMinIntervalSecs) {
+        this.frequencyWarning =
+          'A minimum evaluation interval of ' +
+          this.alertingMinInterval +
+          ' have been configured in Grafana and will be used for this alert rule. ' +
+          'Please contact the administrator to configure a lower interval.';
+      }
+    } catch (err) {
+      this.frequencyWarning = err;
+    }
+  }
+
   graphThresholdChanged(evt: any) {
     for (const condition of this.alert.conditions) {
       if (condition.type === 'query') {

public/app/features/alerting/partials/alert_tab.html

@@ -13,7 +13,7 @@
           </div>
           <div class="gf-form">
             <span class="gf-form-label width-9">Evaluate every</span>
-            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
+            <input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency" ng-blur="ctrl.checkFrequency()">
           </div>
           <div class="gf-form max-width-11">
             <label class="gf-form-label width-5">For</label>
@@ -31,6 +31,12 @@
             </info-popover>
           </div>
         </div>
+        <div class="gf-form" ng-if="ctrl.frequencyWarning">
+          <label class="gf-form-label text-warning">
+            <i
+            class="fa fa-warning"></i> {{ctrl.frequencyWarning}}
+          </label>
+        </div>
       </div>
 
       <div class="gf-form-group">