apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Alerting: Enable Alert rule tags to override PagerDuty Severity setting (#22736)

Browse Source
pull/23011/head^2
Andrew Burian 5 years ago committed by GitHub
parent aeb88015ab
commit 7d9b6a0dee
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 174 additions and 2 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      docs/sources/alerting/notifications.md
  2. 17
      pkg/services/alerting/notifiers/pagerduty.go
  3. 157
      pkg/services/alerting/notifiers/pagerduty_test.go

2
docs/sources/alerting/notifications.md
Unescape View File

@ -110,7 +110,7 @@ Integration Key | Integration key for PagerDuty.
Severity | Level for dynamic notifications, default is `critical`
Auto resolve incidents | Resolve incidents in PagerDuty once the alert goes back to ok
**Note:** The tags `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys they will be transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools.
**Note:** The tags `Severity`, `Class`, `Group`, and `Component` have special meaning in the [Pagerduty Common Event Format - PD-CEF](https://support.pagerduty.com/docs/pd-cef). If an alert panel defines these tag keys, then they are transposed to the root of the event sent to Pagerduty. This means they will be available within the Pagerduty UI and Filtering tools. A Severity tag set on an alert overrides the global Severity set on the notification channel if it's a valid level.
### Webhook

17
pkg/services/alerting/notifiers/pagerduty.go
Unescape View File

@ -98,6 +98,7 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
// set default, override in following case switch if defined
payloadJSON.Set("component", "Grafana")
payloadJSON.Set("severity", pn.Severity)
for _, tag := range evalContext.Rule.AlertRuleTags {
customData.Set(tag.Key, tag.Value)
@ -110,6 +111,21 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
payloadJSON.Set("class", tag.Value)
case "component":
payloadJSON.Set("component", tag.Value)
case "severity":
// Only set severity if it's one of the PD supported enum values
// Info, Warning, Error, or Critical (case insensitive)
switch sev := strings.ToLower(tag.Value); sev {
case "info":
fallthrough
case "warning":
fallthrough
case "error":
fallthrough
case "critical":
payloadJSON.Set("severity", sev)
default:
pn.log.Warn("Ignoring invalid severity tag", "severity", sev)
}
}
}
@ -122,7 +138,6 @@ func (pn *PagerdutyNotifier) buildEventPayload(evalContext *alerting.EvalContext
if hostname, err := os.Hostname(); err == nil {
payloadJSON.Set("source", hostname)
}
payloadJSON.Set("severity", pn.Severity)
payloadJSON.Set("timestamp", time.Now())
payloadJSON.Set("custom_details", customData)
bodyJSON := simplejson.New()

157
pkg/services/alerting/notifiers/pagerduty_test.go
Unescape View File

@ -197,6 +197,7 @@ func TestPagerdutyNotifier(t *testing.T) {
{Key: "group", Value: "aGroup"},
{Key: "class", Value: "aClass"},
{Key: "component", Value: "aComponent"},
{Key: "severity", Value: "warning"},
},
})
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
@ -224,6 +225,162 @@ func TestPagerdutyNotifier(t *testing.T) {
"group": "aGroup",
"class": "aClass",
"component": "aComponent",
"severity": "warning",
"keyOnly": "",
},
"severity": "warning",
"summary": "someRule - someMessage",
"timestamp": "<<PRESENCE>>",
"class": "aClass",
"group": "aGroup",
},
"images": []interface{}{
map[string]interface{}{
"src": "http://somewhere.com/omg_dont_panic.png",
},
},
"routing_key": "abcdefgh0123456789",
}, payload.Interface(), cmp.Comparer(presenceComparer))
So(diff, ShouldBeEmpty)
})
Convey("should support multiple levels of severity", func() {
json := `{
"integrationKey": "abcdefgh0123456789",
"autoResolve": false
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
Type: "pagerduty",
Settings: settingsJSON,
}
not, err := NewPagerdutyNotifier(model)
So(err, ShouldBeNil)
pagerdutyNotifier := not.(*PagerdutyNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateAlerting,
AlertRuleTags: []*models.Tag{
{Key: "keyOnly"},
{Key: "group", Value: "aGroup"},
{Key: "class", Value: "aClass"},
{Key: "component", Value: "aComponent"},
{Key: "severity", Value: "info"},
},
})
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
evalContext.IsTestRun = true
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
payload, err := simplejson.NewJson(payloadJSON)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"client": "Grafana",
"client_url": "",
"dedup_key": "alertId-0",
"event_action": "trigger",
"links": []interface{}{
map[string]interface{}{
"href": "",
},
},
"payload": map[string]interface{}{
"source": "<<PRESENCE>>",
"component": "aComponent",
"custom_details": map[string]interface{}{
"group": "aGroup",
"class": "aClass",
"component": "aComponent",
"severity": "info",
"keyOnly": "",
},
"severity": "info",
"summary": "someRule - someMessage",
"timestamp": "<<PRESENCE>>",
"class": "aClass",
"group": "aGroup",
},
"images": []interface{}{
map[string]interface{}{
"src": "http://somewhere.com/omg_dont_panic.png",
},
},
"routing_key": "abcdefgh0123456789",
}, payload.Interface(), cmp.Comparer(presenceComparer))
So(diff, ShouldBeEmpty)
})
Convey("should ignore invalid severity for PD but keep the tag", func() {
json := `{
"integrationKey": "abcdefgh0123456789",
"autoResolve": false,
"severity": "critical"
}`
settingsJSON, err := simplejson.NewJson([]byte(json))
So(err, ShouldBeNil)
model := &models.AlertNotification{
Name: "pagerduty_testing",
Type: "pagerduty",
Settings: settingsJSON,
}
not, err := NewPagerdutyNotifier(model)
So(err, ShouldBeNil)
pagerdutyNotifier := not.(*PagerdutyNotifier)
evalContext := alerting.NewEvalContext(context.Background(), &alerting.Rule{
ID: 0,
Name: "someRule",
Message: "someMessage",
State: models.AlertStateAlerting,
AlertRuleTags: []*models.Tag{
{Key: "keyOnly"},
{Key: "group", Value: "aGroup"},
{Key: "class", Value: "aClass"},
{Key: "component", Value: "aComponent"},
{Key: "severity", Value: "llama"},
},
})
evalContext.ImagePublicURL = "http://somewhere.com/omg_dont_panic.png"
evalContext.IsTestRun = true
payloadJSON, err := pagerdutyNotifier.buildEventPayload(evalContext)
So(err, ShouldBeNil)
payload, err := simplejson.NewJson(payloadJSON)
So(err, ShouldBeNil)
diff := cmp.Diff(map[string]interface{}{
"client": "Grafana",
"client_url": "",
"dedup_key": "alertId-0",
"event_action": "trigger",
"links": []interface{}{
map[string]interface{}{
"href": "",
},
},
"payload": map[string]interface{}{
"source": "<<PRESENCE>>",
"component": "aComponent",
"custom_details": map[string]interface{}{
"group": "aGroup",
"class": "aClass",
"component": "aComponent",
"severity": "llama",
"keyOnly": "",
},
"severity": "critical",

Write Preview
Loading…
Cancel
Save