Merge pull request #13947 from bergquist/alerting_for

Introduce alert debouncing
pull/14121/head
Carl Bergquist 7 years ago committed by GitHub
commit 6049855dc7
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 780
      devenv/dev-dashboards/testdata_alerts.json
  2. 2
      devenv/docker/ha_test/docker-compose.yaml
  3. 1
      devenv/docker/ha_test/grafana/provisioning/alerts.jsonnet
  4. 16
      docs/sources/alerting/rules.md
  5. 2
      pkg/api/alerting.go
  6. 13
      pkg/models/alert.go
  7. 32
      pkg/services/alerting/eval_context.go
  8. 240
      pkg/services/alerting/eval_context_test.go
  9. 12
      pkg/services/alerting/extractor.go
  10. 20
      pkg/services/alerting/extractor_test.go
  11. 47
      pkg/services/alerting/notifiers/alertmanager_test.go
  12. 10
      pkg/services/alerting/notifiers/base.go
  13. 32
      pkg/services/alerting/notifiers/base_test.go
  14. 3
      pkg/services/alerting/result_handler.go
  15. 5
      pkg/services/alerting/rule.go
  16. 0
      pkg/services/alerting/testdata/collapsed-panels.json
  17. 0
      pkg/services/alerting/testdata/dash-without-id.json
  18. 1
      pkg/services/alerting/testdata/graphite-alert.json
  19. 0
      pkg/services/alerting/testdata/influxdb-alert.json
  20. 0
      pkg/services/alerting/testdata/panel-with-id-0.json
  21. 0
      pkg/services/alerting/testdata/panels-missing-id.json
  22. 0
      pkg/services/alerting/testdata/v5-dashboard.json
  23. 9
      pkg/services/sqlstore/alert.go
  24. 4
      pkg/services/sqlstore/alert_test.go
  25. 4
      pkg/services/sqlstore/migrations/alert_mig.go
  26. 1
      public/app/core/utils/colors.ts
  27. 1
      public/app/features/alerting/AlertRuleList.tsx
  28. 4
      public/app/features/alerting/AlertTabCtrl.ts
  29. 12
      public/app/features/alerting/__snapshots__/AlertRuleList.test.tsx.snap
  30. 272
      public/app/features/alerting/partials/alert_tab.html
  31. 7
      public/app/features/alerting/state/alertDef.ts
  32. 2
      public/app/features/annotations/annotation_tooltip.ts
  33. 6
      public/app/features/annotations/event_manager.ts
  34. 6
      public/app/features/panel/panel_directive.ts
  35. 1
      public/app/plugins/panel/alertlist/editor.html
  36. 7
      public/sass/pages/_alerting.scss

@ -1,250 +1,546 @@
{
"revision": 2,
"title": "Alerting with TestData",
"tags": [
"grafana-test"
],
"style": "dark",
"timezone": "browser",
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"hideControls": false,
"sharedCrosshair": false,
"rows": [
"gnetId": null,
"graphTooltip": 0,
"links": [],
"panels": [
{
"collapse": false,
"alert": {
"conditions": [
{
"evaluator": {
"params": [
60
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always OK",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"height": 255.625,
"panels": [
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 0
},
"id": 3,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "1,20,90,30,5,0",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 60
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always OK",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": "125",
"min": "0",
"show": true
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
60
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always OK",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"id": 3,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "1,20,90,30,5,0",
"target": ""
}
],
"thresholds": [
{
"value": 60,
"op": "gt",
"fill": true,
"line": true,
"colorMode": "critical"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always OK",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": "125",
"min": "0",
"show": true
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"executionErrorState": "alerting",
"for": "0m",
"frequency": "60s",
"handler": 1,
"name": "TestData - Always Alerting",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 0
},
"id": 4,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
"query": {
"params": [
"A",
"5m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"enabled": true,
"frequency": "60s",
"handler": 1,
"name": "TestData - Always Alerting",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"id": 4,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"span": 6,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
1
],
"type": "gt"
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
]
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"15m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"for": "5m",
"frequency": "1m",
"handler": 1,
"name": "TestData - No data",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 0,
"y": 7
},
"id": 5,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "no_data_points",
"stringInput": "",
"target": ""
}
],
"title": "New row"
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 1
}
],
"timeFrom": null,
"timeShift": null,
"title": "No data",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"alert": {
"conditions": [
{
"evaluator": {
"params": [
177
],
"type": "gt"
},
"operator": {
"type": "and"
},
"query": {
"params": [
"A",
"15m",
"now"
]
},
"reducer": {
"params": [],
"type": "avg"
},
"type": "query"
}
],
"executionErrorState": "alerting",
"for": "1m",
"frequency": "1m",
"handler": 1,
"name": "TestData - Always Alerting with For",
"noDataState": "no_data",
"notifications": []
},
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-testdata",
"editable": true,
"error": false,
"fill": 1,
"gridPos": {
"h": 7,
"w": 12,
"x": 12,
"y": 7
},
"id": 6,
"isNew": true,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"refId": "A",
"scenario": "random_walk",
"scenarioId": "csv_metric_values",
"stringInput": "200,445,100,150,200,220,190",
"target": ""
}
],
"thresholds": [
{
"colorMode": "critical",
"fill": true,
"line": true,
"op": "gt",
"value": 177
}
],
"timeFrom": null,
"timeShift": null,
"title": "Always Alerting with For",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": "0",
"show": true
},
{
"format": "short",
"label": "",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
}
],
"revision": 2,
"schemaVersion": 16,
"style": "dark",
"tags": [
"grafana-test"
],
"templating": {
"list": []
},
"time": {
"from": "now-6h",
"to": "now"
@ -274,14 +570,8 @@
"30d"
]
},
"templating": {
"list": []
},
"annotations": {
"list": []
},
"schemaVersion": 13,
"version": 4,
"links": [],
"gnetId": null
}
"timezone": "browser",
"title": "Alerting with TestData",
"uid": "7MeksYbmk",
"version": 1
}

@ -9,7 +9,7 @@ services:
- /var/run/docker.sock:/tmp/docker.sock:ro
db:
image: mysql
image: mysql:5.6
environment:
MYSQL_ROOT_PASSWORD: rootpass
MYSQL_DATABASE: grafana

@ -39,6 +39,7 @@ local alertDashboardTemplate = {
"executionErrorState": "alerting",
"frequency": "10s",
"handler": 1,
"for": "1m",
"name": "bulk alerting",
"noDataState": "no_data",
"notifications": [

@ -39,7 +39,7 @@ Currently alerting supports a limited form of high availability. Since v4.2.0 of
## Rule Config
{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
Currently only the graph panel supports alert rules but this will be added to the **Singlestat** and **Table**
panels as well in a future release.
@ -48,6 +48,16 @@ panels as well in a future release.
Here you can specify the name of the alert rule and how often the scheduler should evaluate the alert rule.
### For
> This setting is available in Grafana 5.4 and above.
If an alert rule has a configured `For` and the query violates the configured threshold it will first go from `OK` to `Pending`. Going from `OK` to `Pending` Grafana will not send any notifications. Once the alert rule has been firing for more than `For` duration, it will change to `Alerting` and send alert notifications.
Typically, it's always a good idea to use this setting since its often worse to get false positive than wait a few minutes before the alert notification triggers.
{{< imgbox max-width="40%" img="/img/docs/v4/alerting_conditions.png" caption="Alerting Conditions" >}}
### Conditions
Currently the only condition type that exists is a `Query` condition that allows you to
@ -57,11 +67,11 @@ specify a query letter, time range and an aggregation function.
### Query condition example
```sql
avg() OF query(A, 5m, now) IS BELOW 14
avg() OF query(A, 15m, now) IS BELOW 14
```
- `avg()` Controls how the values for **each** series should be reduced to a value that can be compared against the threshold. Click on the function to change it to another aggregation function.
- `query(A, 5m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `5m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
- `query(A, 15m, now)` The letter defines what query to execute from the **Metrics** tab. The second two parameters define the time range, `15m, now` means 5 minutes ago to now. You can also do `10m, now-2m` to define a time range that will be 10 minutes ago to 2 minutes ago. This is useful if you want to ignore the last 2 minutes of data.
- `IS BELOW 14` Defines the type of threshold and the threshold value. You can click on `IS BELOW` to change the type of threshold.
The query used in an alert rule cannot contain any template variables. Currently we only support `AND` and `OR` operators between conditions and they are executed serially.

@ -295,7 +295,7 @@ func PauseAlert(c *m.ReqContext, dto dtos.PauseAlertCommand) Response {
return Error(500, "", err)
}
var response m.AlertStateType = m.AlertStatePending
var response m.AlertStateType = m.AlertStateUnknown
pausedState := "un-paused"
if cmd.Paused {
response = m.AlertStatePaused

@ -19,6 +19,7 @@ const (
AlertStateAlerting AlertStateType = "alerting"
AlertStateOK AlertStateType = "ok"
AlertStatePending AlertStateType = "pending"
AlertStateUnknown AlertStateType = "unknown"
)
const (
@ -39,7 +40,12 @@ var (
)
func (s AlertStateType) IsValid() bool {
return s == AlertStateOK || s == AlertStateNoData || s == AlertStatePaused || s == AlertStatePending
return s == AlertStateOK ||
s == AlertStateNoData ||
s == AlertStatePaused ||
s == AlertStatePending ||
s == AlertStateAlerting ||
s == AlertStateUnknown
}
func (s NoDataOption) IsValid() bool {
@ -66,12 +72,13 @@ type Alert struct {
PanelId int64
Name string
Message string
Severity string
Severity string //Unused
State AlertStateType
Handler int64
Handler int64 //Unused
Silenced bool
ExecutionError string
Frequency int64
For time.Duration
EvalData *simplejson.Json
NewStateDate time.Time

@ -68,8 +68,13 @@ func (c *EvalContext) GetStateModel() *StateDescription {
Color: "#D63232",
Text: "Alerting",
}
case m.AlertStateUnknown:
return &StateDescription{
Color: "#888888",
Text: "Unknown",
}
default:
panic("Unknown rule state " + c.Rule.State)
panic("Unknown rule state for alert " + c.Rule.State)
}
}
@ -113,7 +118,26 @@ func (c *EvalContext) GetRuleUrl() (string, error) {
return fmt.Sprintf(urlFormat, m.GetFullDashboardUrl(ref.Uid, ref.Slug), c.Rule.PanelId, c.Rule.OrgId), nil
}
// GetNewState returns the new state from the alert rule evaluation
func (c *EvalContext) GetNewState() m.AlertStateType {
ns := getNewStateInternal(c)
if ns != m.AlertStateAlerting || c.Rule.For == 0 {
return ns
}
since := time.Now().Sub(c.Rule.LastStateChange)
if c.PrevAlertState == m.AlertStatePending && since > c.Rule.For {
return m.AlertStateAlerting
}
if c.PrevAlertState == m.AlertStateAlerting {
return m.AlertStateAlerting
}
return m.AlertStatePending
}
func getNewStateInternal(c *EvalContext) m.AlertStateType {
if c.Error != nil {
c.log.Error("Alert Rule Result Error",
"ruleId", c.Rule.Id,
@ -125,11 +149,13 @@ func (c *EvalContext) GetNewState() m.AlertStateType {
return c.PrevAlertState
}
return c.Rule.ExecutionErrorState.ToAlertState()
}
} else if c.Firing {
if c.Firing {
return m.AlertStateAlerting
}
} else if c.NoDataFound {
if c.NoDataFound {
c.log.Info("Alert Rule returned no data",
"ruleId", c.Rule.Id,
"name", c.Rule.Name,

@ -2,11 +2,11 @@ package alerting
import (
"context"
"fmt"
"errors"
"testing"
"time"
"github.com/grafana/grafana/pkg/models"
. "github.com/smartystreets/goconvey/convey"
)
func TestStateIsUpdatedWhenNeeded(t *testing.T) {
@ -31,71 +31,177 @@ func TestStateIsUpdatedWhenNeeded(t *testing.T) {
})
}
func TestAlertingEvalContext(t *testing.T) {
Convey("Should compute and replace properly new rule state", t, func() {
func TestGetStateFromEvalContext(t *testing.T) {
tcs := []struct {
name string
expected models.AlertStateType
applyFn func(ec *EvalContext)
focus bool
}{
{
name: "ok -> alerting",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.Firing = true
ec.PrevAlertState = models.AlertStateOK
},
},
{
name: "ok -> error(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
},
},
{
name: "ok -> pending. since its been firing for less than FOR",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
ec.Rule.For = time.Minute * 5
},
},
{
name: "ok -> pending. since it has to be pending longer than FOR and prev state is ok",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
ec.Rule.For = time.Minute * 2
},
},
{
name: "pending -> alerting. since its been firing for more than FOR and prev state is pending",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-(time.Hour * 5))
ec.Rule.For = time.Minute * 2
},
},
{
name: "alerting -> alerting. should not update regardless of FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateAlerting
ec.Firing = true
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.For = time.Minute * 2
},
},
{
name: "ok -> ok. should not update regardless of FOR",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
ec.Rule.For = time.Minute * 2
},
},
{
name: "ok -> error(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "pending -> error(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Error = errors.New("test error")
ec.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
},
},
{
name: "ok -> no_data(alerting)",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
},
},
{
name: "ok -> no_data(keep_last)",
expected: models.AlertStateOK,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStateOK
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
{
name: "pending -> no_data(keep_last)",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataKeepState
ec.NoDataFound = true
},
},
{
name: "pending -> no_data(alerting) with for duration have not passed",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
ec.Rule.For = time.Minute * 5
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
},
},
{
name: "pending -> no_data(alerting) should set alerting since time passed FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.NoDataState = models.NoDataSetAlerting
ec.NoDataFound = true
ec.Rule.For = time.Minute * 2
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
},
},
{
name: "pending -> error(alerting) with for duration have not passed ",
expected: models.AlertStatePending,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ec.Error = errors.New("test error")
ec.Rule.For = time.Minute * 5
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 2)
},
},
{
name: "pending -> error(alerting) should set alerting since time passed FOR",
expected: models.AlertStateAlerting,
applyFn: func(ec *EvalContext) {
ec.PrevAlertState = models.AlertStatePending
ec.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ec.Error = errors.New("test error")
ec.Rule.For = time.Minute * 2
ec.Rule.LastStateChange = time.Now().Add(-time.Minute * 5)
},
},
}
for _, tc := range tcs {
ctx := NewEvalContext(context.TODO(), &Rule{Conditions: []Condition{&conditionStub{firing: true}}})
dummieError := fmt.Errorf("dummie error")
Convey("ok -> alerting", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Firing = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> error(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorSetAlerting
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> error(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Error = dummieError
ctx.Rule.ExecutionErrorState = models.ExecutionErrorKeepState
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
Convey("ok -> no_data(alerting)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataSetAlerting
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateAlerting)
})
Convey("ok -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStateOK
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStateOK)
})
Convey("pending -> no_data(keep_last)", func() {
ctx.PrevAlertState = models.AlertStatePending
ctx.Rule.NoDataState = models.NoDataKeepState
ctx.NoDataFound = true
ctx.Rule.State = ctx.GetNewState()
So(ctx.Rule.State, ShouldEqual, models.AlertStatePending)
})
})
tc.applyFn(ctx)
have := ctx.GetNewState()
if have != tc.expected {
t.Errorf("failed: %s \n expected '%s' have '%s'\n", tc.name, tc.expected, string(have))
}
}
}

@ -2,8 +2,8 @@ package alerting
import (
"errors"
"fmt"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@ -115,6 +115,15 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
return nil, ValidationError{Reason: "Could not parse frequency"}
}
rawFor := jsonAlert.Get("for").MustString()
var forValue time.Duration
if rawFor != "" {
forValue, err = time.ParseDuration(rawFor)
if err != nil {
return nil, ValidationError{Reason: "Could not parse for"}
}
}
alert := &m.Alert{
DashboardId: e.Dash.Id,
OrgId: e.OrgID,
@ -124,6 +133,7 @@ func (e *DashAlertExtractor) getAlertFromPanels(jsonWithPanels *simplejson.Json,
Handler: jsonAlert.Get("handler").MustInt64(),
Message: jsonAlert.Get("message").MustString(),
Frequency: frequency,
For: forValue,
}
for _, condition := range jsonAlert.Get("conditions").MustArray() {

@ -3,6 +3,7 @@ package alerting
import (
"io/ioutil"
"testing"
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/components/simplejson"
@ -46,7 +47,7 @@ func TestAlertRuleExtraction(t *testing.T) {
return nil
})
json, err := ioutil.ReadFile("./test-data/graphite-alert.json")
json, err := ioutil.ReadFile("./testdata/graphite-alert.json")
So(err, ShouldBeNil)
Convey("Extractor should not modify the original json", func() {
@ -118,6 +119,11 @@ func TestAlertRuleExtraction(t *testing.T) {
So(alerts[1].PanelId, ShouldEqual, 4)
})
Convey("should extract for param", func() {
So(alerts[0].For, ShouldEqual, time.Minute*2)
So(alerts[1].For, ShouldEqual, time.Duration(0))
})
Convey("should extract name and desc", func() {
So(alerts[0].Name, ShouldEqual, "name1")
So(alerts[0].Message, ShouldEqual, "desc1")
@ -140,7 +146,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Panels missing id should return error", func() {
panelWithoutId, err := ioutil.ReadFile("./test-data/panels-missing-id.json")
panelWithoutId, err := ioutil.ReadFile("./testdata/panels-missing-id.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(panelWithoutId)
@ -156,7 +162,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Panel with id set to zero should return error", func() {
panelWithIdZero, err := ioutil.ReadFile("./test-data/panel-with-id-0.json")
panelWithIdZero, err := ioutil.ReadFile("./testdata/panel-with-id-0.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(panelWithIdZero)
@ -172,7 +178,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse alerts from dashboard without rows", func() {
json, err := ioutil.ReadFile("./test-data/v5-dashboard.json")
json, err := ioutil.ReadFile("./testdata/v5-dashboard.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@ -192,7 +198,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse and validate dashboard containing influxdb alert", func() {
json, err := ioutil.ReadFile("./test-data/influxdb-alert.json")
json, err := ioutil.ReadFile("./testdata/influxdb-alert.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@ -221,7 +227,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Should be able to extract collapsed panels", func() {
json, err := ioutil.ReadFile("./test-data/collapsed-panels.json")
json, err := ioutil.ReadFile("./testdata/collapsed-panels.json")
So(err, ShouldBeNil)
dashJson, err := simplejson.NewJson(json)
@ -242,7 +248,7 @@ func TestAlertRuleExtraction(t *testing.T) {
})
Convey("Parse and validate dashboard without id and containing an alert", func() {
json, err := ioutil.ReadFile("./test-data/dash-without-id.json")
json, err := ioutil.ReadFile("./testdata/dash-without-id.json")
So(err, ShouldBeNil)
dashJSON, err := simplejson.NewJson(json)

@ -1,13 +1,60 @@
package notifiers
import (
"context"
"testing"
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/alerting"
. "github.com/smartystreets/goconvey/convey"
)
func TestWhenAlertManagerShouldNotify(t *testing.T) {
tcs := []struct {
prevState m.AlertStateType
newState m.AlertStateType
expect bool
}{
{
prevState: m.AlertStatePending,
newState: m.AlertStateOK,
expect: false,
},
{
prevState: m.AlertStateAlerting,
newState: m.AlertStateOK,
expect: true,
},
{
prevState: m.AlertStateOK,
newState: m.AlertStatePending,
expect: false,
},
{
prevState: m.AlertStateUnknown,
newState: m.AlertStatePending,
expect: false,
},
}
for _, tc := range tcs {
am := &AlertmanagerNotifier{log: log.New("test.logger")}
evalContext := alerting.NewEvalContext(context.TODO(), &alerting.Rule{
State: tc.prevState,
})
evalContext.Rule.State = tc.newState
res := am.ShouldNotify(context.TODO(), evalContext, &m.AlertNotificationState{})
if res != tc.expect {
t.Errorf("got %v expected %v", res, tc.expect)
}
}
}
func TestAlertmanagerNotifier(t *testing.T) {
Convey("Alertmanager notifier tests", t, func() {

@ -67,6 +67,16 @@ func (n *NotifierBase) ShouldNotify(ctx context.Context, context *alerting.EvalC
}
// Do not notify when we become OK for the first time.
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStateOK {
return false
}
// Do not notify when we become OK for the first time.
if context.PrevAlertState == models.AlertStateUnknown && context.Rule.State == models.AlertStatePending {
return false
}
// Do not notify when we become OK from pending
if context.PrevAlertState == models.AlertStatePending && context.Rule.State == models.AlertStateOK {
return false
}

@ -29,7 +29,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStatePending,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@ -38,7 +37,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateAlerting,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: true,
},
@ -47,7 +45,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStatePending,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@ -56,7 +53,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: false,
},
@ -65,7 +61,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateOK,
sendReminder: true,
state: &m.AlertNotificationState{},
expect: false,
},
@ -74,7 +69,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
newState: m.AlertStateOK,
prevState: m.AlertStateAlerting,
sendReminder: false,
state: &m.AlertNotificationState{},
expect: true,
},
@ -94,7 +88,6 @@ func TestShouldSendAlertNotification(t *testing.T) {
prevState: m.AlertStateAlerting,
frequency: time.Minute * 10,
sendReminder: true,
state: &m.AlertNotificationState{},
expect: true,
},
@ -132,6 +125,27 @@ func TestShouldSendAlertNotification(t *testing.T) {
prevState: m.AlertStateOK,
state: &m.AlertNotificationState{State: m.AlertNotificationStatePending, UpdatedAt: tnow.Add(-2 * time.Minute).Unix()},
expect: true,
},
{
name: "unknown -> ok",
prevState: m.AlertStateUnknown,
newState: m.AlertStateOK,
expect: false,
},
{
name: "unknown -> pending",
prevState: m.AlertStateUnknown,
newState: m.AlertStatePending,
expect: false,
},
{
name: "unknown -> alerting",
prevState: m.AlertStateUnknown,
newState: m.AlertStateAlerting,
expect: true,
},
}
@ -141,6 +155,10 @@ func TestShouldSendAlertNotification(t *testing.T) {
State: tc.prevState,
})
if tc.state == nil {
tc.state = &m.AlertNotificationState{}
}
evalContext.Rule.State = tc.newState
nb := &NotifierBase{SendReminder: tc.sendReminder, Frequency: tc.frequency}

@ -73,6 +73,9 @@ func (handler *DefaultResultHandler) Handle(evalContext *EvalContext) error {
// when two servers are raising. This makes sure that the server
// with the last state change always sends a notification.
evalContext.Rule.StateChanges = cmd.Result.StateChanges
// Update the last state change of the alert rule in memory
evalContext.Rule.LastStateChange = time.Now()
}
// save annotation

@ -4,6 +4,7 @@ import (
"fmt"
"regexp"
"strconv"
"time"
"github.com/grafana/grafana/pkg/components/simplejson"
@ -18,6 +19,8 @@ type Rule struct {
Frequency int64
Name string
Message string
LastStateChange time.Time
For time.Duration
NoDataState m.NoDataOption
ExecutionErrorState m.ExecutionErrorOption
State m.AlertStateType
@ -100,6 +103,8 @@ func NewRuleFromDBAlert(ruleDef *m.Alert) (*Rule, error) {
model.Message = ruleDef.Message
model.Frequency = ruleDef.Frequency
model.State = ruleDef.State
model.LastStateChange = ruleDef.NewStateDate
model.For = ruleDef.For
model.NoDataState = m.NoDataOption(ruleDef.Settings.Get("noDataState").MustString("no_data"))
model.ExecutionErrorState = m.ExecutionErrorOption(ruleDef.Settings.Get("executionErrorState").MustString("alerting"))
model.StateChanges = ruleDef.StateChanges

@ -23,6 +23,7 @@
"message": "desc1",
"handler": 1,
"frequency": "60s",
"for": "2m",
"conditions": [
{
"type": "query",

@ -193,7 +193,8 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
if alertToUpdate.ContainsUpdates(alert) {
alert.Updated = timeNow()
alert.State = alertToUpdate.State
sess.MustCols("message")
sess.MustCols("message", "for")
_, err := sess.ID(alert.Id).Update(alert)
if err != nil {
return err
@ -204,7 +205,7 @@ func updateAlerts(existingAlerts []*m.Alert, cmd *m.SaveAlertsCommand, sess *DBS
} else {
alert.Updated = timeNow()
alert.Created = timeNow()
alert.State = m.AlertStatePending
alert.State = m.AlertStateUnknown
alert.NewStateDate = timeNow()
_, err := sess.Insert(alert)
@ -299,7 +300,7 @@ func PauseAlert(cmd *m.PauseAlertCommand) error {
params = append(params, string(m.AlertStatePaused))
params = append(params, timeNow())
} else {
params = append(params, string(m.AlertStatePending))
params = append(params, string(m.AlertStateUnknown))
params = append(params, timeNow())
}
@ -323,7 +324,7 @@ func PauseAllAlerts(cmd *m.PauseAllAlertCommand) error {
if cmd.Paused {
newState = string(m.AlertStatePaused)
} else {
newState = string(m.AlertStatePending)
newState = string(m.AlertStateUnknown)
}
res, err := sess.Exec(`UPDATE alert SET state = ?, new_state_date = ?`, newState, timeNow())

@ -109,7 +109,7 @@ func TestAlertingDataAccess(t *testing.T) {
So(alert.DashboardId, ShouldEqual, testDash.Id)
So(alert.PanelId, ShouldEqual, 1)
So(alert.Name, ShouldEqual, "Alerting title")
So(alert.State, ShouldEqual, "pending")
So(alert.State, ShouldEqual, m.AlertStateUnknown)
So(alert.NewStateDate, ShouldNotBeNil)
So(alert.EvalData, ShouldNotBeNil)
So(alert.EvalData.Get("test").MustString(), ShouldEqual, "test")
@ -154,7 +154,7 @@ func TestAlertingDataAccess(t *testing.T) {
So(query.Result[0].Name, ShouldEqual, "Name")
Convey("Alert state should not be updated", func() {
So(query.Result[0].State, ShouldEqual, "pending")
So(query.Result[0].State, ShouldEqual, m.AlertStateUnknown)
})
})

@ -133,4 +133,8 @@ func addAlertMigrations(mg *Migrator) {
mg.AddMigration("create alert_notification_state table v1", NewAddTableMigration(alert_notification_state))
mg.AddMigration("add index alert_notification_state org_id & alert_id & notifier_id",
NewAddIndexMigration(alert_notification_state, alert_notification_state.Indices[0]))
mg.AddMigration("Add for to alert table", NewAddColumnMigration(alertV1, &Column{
Name: "for", Type: DB_BigInt, Nullable: true,
}))
}

@ -7,6 +7,7 @@ export const DEFAULT_ANNOTATION_COLOR = 'rgba(0, 211, 255, 1)';
export const OK_COLOR = 'rgba(11, 237, 50, 1)';
export const ALERTING_COLOR = 'rgba(237, 46, 24, 1)';
export const NO_DATA_COLOR = 'rgba(150, 150, 150, 1)';
export const PENDING_COLOR = 'rgba(247, 149, 32, 1)';
export const REGION_FILL_ALPHA = 0.09;
const colors = [

@ -29,6 +29,7 @@ export class AlertRuleList extends PureComponent<Props, any> {
{ text: 'Alerting', value: 'alerting' },
{ text: 'No Data', value: 'no_data' },
{ text: 'Paused', value: 'paused' },
{ text: 'Pending', value: 'pending' },
];
componentDidMount() {

@ -169,6 +169,7 @@ export class AlertTabCtrl {
alert.frequency = alert.frequency || '1m';
alert.handler = alert.handler || 1;
alert.notifications = alert.notifications || [];
alert.for = alert.for || '0m';
const defaultName = this.panel.title + ' alert';
alert.name = alert.name || defaultName;
@ -217,7 +218,7 @@ export class AlertTabCtrl {
buildDefaultCondition() {
return {
type: 'query',
query: { params: ['A', '15m', 'now'] },
query: { params: ['A', '5m', 'now'] },
reducer: { type: 'avg', params: [] },
evaluator: { type: 'gt', params: [null] },
operator: { type: 'and' },
@ -354,6 +355,7 @@ export class AlertTabCtrl {
enable() {
this.panel.alert = {};
this.initModel();
this.panel.alert.for = '5m'; //default value for new alerts. for existing alerts we use 0m to avoid breaking changes
}
evaluatorParamsChanged() {

@ -81,6 +81,12 @@ exports[`Render should render alert rules 1`] = `
>
Paused
</option>
<option
key="pending"
value="pending"
>
Pending
</option>
</select>
</div>
</div>
@ -230,6 +236,12 @@ exports[`Render should render component 1`] = `
>
Paused
</option>
<option
key="pending"
value="pending"
>
Pending
</option>
</select>
</div>
</div>

@ -1,147 +1,159 @@
<div class="edit-tab-with-sidemenu" ng-if="ctrl.alert">
<aside class="edit-sidemenu-aside">
<ul class="edit-sidemenu">
<li ng-class="{active: ctrl.subTabIndex === 0}">
<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 1}">
<a ng-click="ctrl.changeTabIndex(1)">
Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 2}">
<a ng-click="ctrl.changeTabIndex(2)">State history</a>
</li>
<aside class="edit-sidemenu-aside">
<ul class="edit-sidemenu">
<li ng-class="{active: ctrl.subTabIndex === 0}">
<a ng-click="ctrl.changeTabIndex(0)">Alert Config</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 1}">
<a ng-click="ctrl.changeTabIndex(1)">
Notifications <span class="muted">({{ctrl.alertNotifications.length}})</span>
</a>
</li>
<li ng-class="{active: ctrl.subTabIndex === 2}">
<a ng-click="ctrl.changeTabIndex(2)">State history</a>
</li>
<li>
<a ng-click="ctrl.delete()">Delete</a>
</li>
</ul>
</aside>
<a ng-click="ctrl.delete()">Delete</a>
</li>
</ul>
</aside>
<div class="edit-tab-content">
<div ng-if="ctrl.subTabIndex === 0">
<div class="alert alert-error m-b-2" ng-show="ctrl.error">
<i class="fa fa-warning"></i> {{ctrl.error}}
</div>
<div class="edit-tab-content">
<div ng-if="ctrl.subTabIndex === 0">
<div class="alert alert-error m-b-2" ng-show="ctrl.error">
<i class="fa fa-warning"></i> {{ctrl.error}}
</div>
<div class="gf-form-group">
<h5 class="section-heading">Alert Config</h5>
<div class="gf-form">
<span class="gf-form-label width-6">Name</span>
<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
<span class="gf-form-label">Evaluate every</span>
<input class="gf-form-input max-width-5" type="text" ng-model="ctrl.alert.frequency"></input>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Alert Config</h5>
<div class="gf-form">
<span class="gf-form-label width-6">Name</span>
<input type="text" class="gf-form-input width-20" ng-model="ctrl.alert.name">
</div>
<div class="gf-form-inline">
<div class="gf-form">
<span class="gf-form-label width-9">Evaluate every</span>
<input class="gf-form-input max-width-6" type="text" ng-model="ctrl.alert.frequency">
</div>
<div class="gf-form max-width-11">
<label class="gf-form-label width-5">For</label>
<input type="text" class="gf-form-input max-width-6" ng-model="ctrl.alert.for" spellcheck='false' placeholder="5m">
<info-popover mode="right-absolute">
If an alert rule has a configured For and the query violates the configured threshold it will first go from OK to Pending.
Going from OK to Pending Grafana will not send any notifications. Once the alert rule has been firing for more than For duration, it will change to Alerting and send alert notifications.
</info-popover>
</div>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Conditions</h5>
<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
<div class="gf-form">
<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
</query-part-editor>
<span class="gf-form-label query-keyword">OF</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
</query-part-editor>
</div>
<div class="gf-form">
<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
</div>
<div class="gf-form">
<label class="gf-form-label">
<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
<i class="fa fa-trash"></i>
</a>
</label>
</div>
</div>
<div class="gf-form-group">
<h5 class="section-heading">Conditions</h5>
<div class="gf-form-inline" ng-repeat="conditionModel in ctrl.conditionModels">
<div class="gf-form">
<metric-segment-model css-class="query-keyword width-5" ng-if="$index" property="conditionModel.operator.type" options="ctrl.evalOperators" custom="false"></metric-segment-model>
<span class="gf-form-label query-keyword width-5" ng-if="$index===0">WHEN</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part width-9" part="conditionModel.reducerPart" handle-event="ctrl.handleReducerPartEvent(conditionModel, $event)">
</query-part-editor>
<span class="gf-form-label query-keyword">OF</span>
</div>
<div class="gf-form">
<query-part-editor class="gf-form-label query-part" part="conditionModel.queryPart" handle-event="ctrl.handleQueryPartEvent(conditionModel, $event)">
</query-part-editor>
</div>
<div class="gf-form">
<metric-segment-model property="conditionModel.evaluator.type" options="ctrl.evalFunctions" custom="false" css-class="query-keyword" on-change="ctrl.evaluatorTypeChanged(conditionModel.evaluator)"></metric-segment-model>
<input class="gf-form-input max-width-9" type="number" step="any" ng-hide="conditionModel.evaluator.params.length === 0" ng-model="conditionModel.evaluator.params[0]" ng-change="ctrl.evaluatorParamsChanged()"></input>
<label class="gf-form-label query-keyword" ng-show="conditionModel.evaluator.params.length === 2">TO</label>
<input class="gf-form-input max-width-9" type="number" step="any" ng-if="conditionModel.evaluator.params.length === 2" ng-model="conditionModel.evaluator.params[1]" ng-change="ctrl.evaluatorParamsChanged()"></input>
</div>
<div class="gf-form">
<label class="gf-form-label">
<a class="pointer" tabindex="1" ng-click="ctrl.removeCondition($index)">
<i class="fa fa-trash"></i>
</a>
</label>
</div>
</div>
<div class="gf-form">
<label class="gf-form-label dropdown">
<a class="pointer dropdown-toggle" data-toggle="dropdown">
<i class="fa fa-plus"></i>
</a>
<ul class="dropdown-menu" role="menu">
<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
</li>
</ul>
</label>
</div>
</div>
<div class="gf-form">
<label class="gf-form-label dropdown">
<a class="pointer dropdown-toggle" data-toggle="dropdown">
<i class="fa fa-plus"></i>
</a>
<ul class="dropdown-menu" role="menu">
<li ng-repeat="ct in ctrl.conditionTypes" role="menuitem">
<a ng-click="ctrl.addCondition(ct.value);">{{ct.text}}</a>
</li>
</ul>
</label>
</div>
</div>
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label width-18">If no data or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
</select>
</div>
</div>
<div class="gf-form-group">
<div class="gf-form">
<span class="gf-form-label width-18">If no data or all values are null</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.noDataState" ng-options="f.value as f.text for f in ctrl.noDataModes">
</select>
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-18">If execution error or timeout</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form">
<span class="gf-form-label width-18">If execution error or timeout</span>
<span class="gf-form-label query-keyword">SET STATE TO</span>
<div class="gf-form-select-wrapper">
<select class="gf-form-input" ng-model="ctrl.alert.executionErrorState" ng-options="f.value as f.text for f in ctrl.executionErrorModes">
</select>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule
</button>
</div>
</div>
<div class="gf-form-button-row">
<button class="btn btn-inverse" ng-click="ctrl.test()">
Test Rule
</button>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.testing">
Evaluating rule <i class="fa fa-spinner fa-spin"></i>
</div>
<div class="gf-form-group" ng-if="ctrl.testing">
Evaluating rule <i class="fa fa-spinner fa-spin"></i>
</div>
<div class="gf-form-group" ng-if="ctrl.testResult">
<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.testResult">
<json-tree root-name="result" object="ctrl.testResult" start-expanded="true"></json-tree>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
<h5 class="section-heading">Notifications</h5>
<div class="gf-form-inline">
<div class="gf-form max-width-30">
<span class="gf-form-label width-8">Send to</span>
<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
</span>
<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
</div>
</div>
<div class="gf-form gf-form--v-stretch">
<span class="gf-form-label width-8">Message</span>
<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message" placeholder="Notification message details..."></textarea>
</div>
</div>
<div class="gf-form-group" ng-if="ctrl.subTabIndex === 1">
<h5 class="section-heading">Notifications</h5>
<div class="gf-form-inline">
<div class="gf-form max-width-30">
<span class="gf-form-label width-8">Send to</span>
<span class="gf-form-label" ng-repeat="nc in ctrl.alertNotifications" ng-style="{'background-color': nc.bgColor }">
<i class="{{nc.iconClass}}"></i>&nbsp;{{nc.name}}&nbsp;
<i class="fa fa-remove pointer muted" ng-click="ctrl.removeNotification($index)" ng-if="nc.isDefault === false"></i>
</span>
<metric-segment segment="ctrl.addNotificationSegment" get-options="ctrl.getNotifications()" on-change="ctrl.notificationAdded()"></metric-segment>
</div>
</div>
<div class="gf-form gf-form--v-stretch">
<span class="gf-form-label width-8">Message</span>
<textarea class="gf-form-input" rows="10" ng-model="ctrl.alert.message" placeholder="Notification message details..."></textarea>
</div>
</div>
<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
<h5 class="section-heading" style="whitespace: nowrap">
State history <span class="muted small">(last 50 state changes)</span>
</h5>
<div class="gf-form-group" style="max-width: 720px;" ng-if="ctrl.subTabIndex === 2">
<button class="btn btn-mini btn-danger pull-right" ng-click="ctrl.clearHistory()"><i class="fa fa-trash"></i>&nbsp;Clear history</button>
<h5 class="section-heading" style="whitespace: nowrap">
State history <span class="muted small">(last 50 state changes)</span>
</h5>
<div ng-show="ctrl.alertHistory.length === 0">
<br>
<i>No state changes recorded</i>
</div>
<div ng-show="ctrl.alertHistory.length === 0">
<br>
<i>No state changes recorded</i>
</div>
<ol class="alert-rule-list" >
<li class="alert-rule-item" ng-repeat="al in ctrl.alertHistory">

@ -99,6 +99,13 @@ function getStateDisplayModel(state) {
stateClass: 'alert-state-warning',
};
}
case 'unknown': {
return {
text: 'UNKNOWN',
iconClass: 'fa fa-question',
stateClass: 'alert-state-paused',
};
}
}
throw { message: 'Unknown alert state' };

@ -32,7 +32,7 @@ export function annotationTooltipDirective($sanitize, dashboardSrv, contextSrv,
if (event.alertId) {
const stateModel = alertDef.getStateDisplayModel(event.newState);
titleStateClass = stateModel.stateClass;
title = `<i class="icon-gf ${stateModel.iconClass}"></i> ${stateModel.text}`;
title = `<i class="${stateModel.iconClass}"></i> ${stateModel.text}`;
text = alertDef.getAlertAnnotationInfo(event);
if (event.text) {
text = text + '<br />' + event.text;

@ -7,6 +7,7 @@ import {
OK_COLOR,
ALERTING_COLOR,
NO_DATA_COLOR,
PENDING_COLOR,
DEFAULT_ANNOTATION_COLOR,
REGION_FILL_ALPHA,
} from 'app/core/utils/colors';
@ -71,6 +72,11 @@ export class EventManager {
position: 'BOTTOM',
markerSize: 5,
},
$__pending: {
color: PENDING_COLOR,
position: 'BOTTOM',
markerSize: 5,
},
$__editing: {
color: DEFAULT_ANNOTATION_COLOR,
position: 'BOTTOM',

@ -161,7 +161,11 @@ module.directive('grafanaPanel', ($rootScope, $document, $timeout) => {
panelContainer.removeClass('panel-alert-state--' + lastAlertState);
}
if (ctrl.alertState.state === 'ok' || ctrl.alertState.state === 'alerting') {
if (
ctrl.alertState.state === 'ok' ||
ctrl.alertState.state === 'alerting' ||
ctrl.alertState.state === 'pending'
) {
panelContainer.addClass('panel-alert-state--' + ctrl.alertState.state);
}

@ -50,5 +50,6 @@
<gf-form-switch class="gf-form" label="No data" label-class="width-10" checked="ctrl.stateFilter['no_data']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Execution error" label-class="width-10" checked="ctrl.stateFilter['execution_error']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Alerting" label-class="width-10" checked="ctrl.stateFilter['alerting']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
<gf-form-switch class="gf-form" label="Pending" label-class="width-10" checked="ctrl.stateFilter['pending']" on-change="ctrl.updateStateFilter()"></gf-form-switch>
</div>
</div>

@ -66,6 +66,13 @@
content: '\e611';
}
}
&--pending {
.panel-alert-icon:before {
color: $warn;
content: '\e611';
}
}
}
@keyframes alerting-panel {

Loading…
Cancel
Save