mirror of https://github.com/grafana/grafana
Alerting: Add support for Sensu Go notification channel (#28012)
* Add support for Sensu Go notification channel Similar to current support for the older "Core" version of Sensu, this commit add support for the newer version. Closes #19908 Signed-off-by: Todd Campbell <todd@sensu.io> * fix linter errors Signed-off-by: Todd Campbell <todd@sensu.io> * Apply suggestions from code review PR review suggestions Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com> * Fix no new variables error * Replace convey testing with testify Signed-off-by: Todd Campbell <todd@sensu.io> * Wording suggestions Signed-off-by: Todd Campbell <todd@sensu.io> * Add docker compose environment for testing/maintenance Signed-off-by: Todd Campbell <todd@sensu.io> * Renamed and fixed docker-compose.yaml to work in devenv Signed-off-by: Todd Campbell <todd@sensu.io> * Change sensugo web UI port to 3080 so as not to conflict with grafana Signed-off-by: Todd Campbell <todd@sensu.io> * Apply suggestions from code review Set the API key as a secure value. Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com> * Add Sensu Go information to notifications doc Signed-off-by: Todd Campbell <todd@sensu.io> * Update pkg/services/alerting/notifiers/sensugo.go Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com> * change assert to require for Error/NoError tests Signed-off-by: Todd Campbell <todd@sensu.io> Co-authored-by: Arve Knudsen <arve.knudsen@gmail.com> Co-authored-by: Sofia Papagiannaki <papagian@users.noreply.github.com>pull/29445/head
Todd Campbell
5 years ago
committed by
GitHub
parent
cffc1b13ad
commit
643f790753
@ -0,0 +1,63 @@ |
||||
--- |
||||
# Sensu backend configuration |
||||
|
||||
## |
||||
# backend configuration |
||||
## |
||||
state-dir: "/var/lib/sensu/sensu-backend" |
||||
#cache-dir: "/var/cache/sensu/sensu-backend" |
||||
#config-file: "/etc/sensu/backend.yml" |
||||
#debug: false |
||||
#deregistration-handler: "example_handler" |
||||
#log-level: "warn" # available log levels: panic, fatal, error, warn, info, debug |
||||
|
||||
## |
||||
# agent configuration |
||||
## |
||||
#agent-host: "[::]" # listen on all IPv4 and IPv6 addresses |
||||
#agent-port: 8081 |
||||
|
||||
## |
||||
# api configuration |
||||
## |
||||
#api-listen-address: "[::]:8080" # listen on all IPv4 and IPv6 addresses |
||||
#api-url: "http://localhost:8080" |
||||
|
||||
## |
||||
# dashboard configuration |
||||
## |
||||
#dashboard-cert-file: "/path/to/ssl/cert.pem" |
||||
#dashboard-key-file: "/path/to/ssl/key.pem" |
||||
#dashboard-host: "[::]" # listen on all IPv4 and IPv6 addresses |
||||
#dashboard-port: 3000 |
||||
|
||||
## |
||||
# ssl configuration |
||||
## |
||||
#cert-file: "/path/to/ssl/cert.pem" |
||||
#key-file: "/path/to/ssl/key.pem" |
||||
#trusted-ca-file: "/path/to/trusted-certificate-authorities.pem" |
||||
#insecure-skip-tls-verify: false |
||||
|
||||
## |
||||
# store configuration |
||||
## |
||||
#etcd-advertise-client-urls: "http://localhost:2379" |
||||
#etcd-cert-file: "/path/to/ssl/cert.pem" |
||||
#etcd-client-cert-auth: false |
||||
#etcd-initial-advertise-peer-urls: "http://127.0.0.1:2380" |
||||
#etcd-initial-cluster: "default=http://127.0.0.1:2380" |
||||
#etcd-initial-cluster-state: "new" # new or existing |
||||
#etcd-initial-cluster-token: "sensu" |
||||
#etcd-key-file: "/path/to/ssl/key.pem" |
||||
#etcd-listen-client-urls: "http://127.0.0.1:2379" |
||||
#etcd-listen-peer-urls: "http://127.0.0.1:2380" |
||||
#etcd-name: "default" |
||||
#etcd-peer-cert-file: "/path/to/ssl/cert.pem" |
||||
#etcd-peer-client-cert-auth: false |
||||
#etcd-peer-key-file: "/path/to/ssl/key.pem" |
||||
#etcd-peer-trusted-ca-file: "/path/to/ssl/key.pem" |
||||
#etcd-trusted-ca-file: "/path/to/ssl/key.pem" |
||||
#no-embed-etcd: false |
||||
#etcd-cipher-suits |
||||
# - TLS_EXAMPLE |
||||
@ -0,0 +1,18 @@ |
||||
sensu-backend: |
||||
image: sensu/sensu:latest |
||||
container_name: sensu-backend |
||||
ports: |
||||
- "3080:3000" |
||||
- "8080:8080" |
||||
- "8081:8081" |
||||
volumes: |
||||
- ./docker/blocks/sensugo/backend.yml:/etc/sensu/backend.yml |
||||
- sensu-backend-data:/var/lib/sensu/etcd |
||||
environment: |
||||
SENSU_BACKEND_CLUSTER_ADMIN_USERNAME: admin |
||||
SENSU_BACKEND_CLUSTER_ADMIN_PASSWORD: Password123 |
||||
command: "sensu-backend start --log-level info" |
||||
|
||||
volumes: |
||||
sensu-backend-data: {} |
||||
|
||||
@ -0,0 +1,39 @@ |
||||
# Notes on Sensu Go Docker Block |
||||
|
||||
The API Key needed to connect to Sensu Go has to be created manually. |
||||
|
||||
## Create the API Key |
||||
|
||||
`docker exec -it sensu-backend /bin/ash` |
||||
|
||||
Configure the `sensuctl` command using the pre-set username and password: |
||||
|
||||
```bash |
||||
sensuctl configure -n --url http://127.0.0.1:8080 --username admin --password 'Password123' --namespace default |
||||
``` |
||||
|
||||
Generate the API Key: |
||||
|
||||
```bash |
||||
sensuctl api-key grant admin |
||||
``` |
||||
|
||||
The output should look similar to this: |
||||
|
||||
``` |
||||
Created: /api/core/v2/apikeys/0a1b2c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d |
||||
``` |
||||
|
||||
|
||||
## Configuring the notification channel |
||||
|
||||
### Backend URL |
||||
|
||||
The Backend URL is the API port (8080) forwarded to the container, it should be |
||||
`http://localhost:8080` |
||||
|
||||
### API Key |
||||
|
||||
The `0a1b2c3d-4e5f-6a7b-8c9d-0e1f2a3b4c5d` in the output above is the API Key |
||||
to use in configuring the Sensu Go notification channel. |
||||
|
||||
@ -0,0 +1,196 @@ |
||||
package notifiers |
||||
|
||||
import ( |
||||
"fmt" |
||||
"strconv" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana/pkg/bus" |
||||
"github.com/grafana/grafana/pkg/components/simplejson" |
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/grafana/grafana/pkg/services/alerting" |
||||
) |
||||
|
||||
func init() { |
||||
alerting.RegisterNotifier(&alerting.NotifierPlugin{ |
||||
Type: "sensugo", |
||||
Name: "Sensu Go", |
||||
Description: "Sends HTTP POST request to a Sensu Go API", |
||||
Heading: "Sensu Go Settings", |
||||
Factory: NewSensuGoNotifier, |
||||
Options: []alerting.NotifierOption{ |
||||
{ |
||||
Label: "Backend URL", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypeText, |
||||
Placeholder: "http://sensu-api.local:8080", |
||||
PropertyName: "url", |
||||
Required: true, |
||||
}, |
||||
{ |
||||
Label: "API Key", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypePassword, |
||||
Description: "API Key to auth to Sensu Go backend", |
||||
PropertyName: "apikey", |
||||
Required: true, |
||||
Secure: true, |
||||
}, |
||||
{ |
||||
Label: "Proxy entity name", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypeText, |
||||
Description: "If empty, rule name will be used", |
||||
PropertyName: "entity", |
||||
}, |
||||
{ |
||||
Label: "Check name", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypeText, |
||||
Description: "If empty, rule id will be used", |
||||
PropertyName: "check", |
||||
}, |
||||
{ |
||||
Label: "Handler", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypeText, |
||||
PropertyName: "handler", |
||||
}, |
||||
{ |
||||
Label: "Namespace", |
||||
Element: alerting.ElementTypeInput, |
||||
InputType: alerting.InputTypeText, |
||||
Placeholder: "default", |
||||
PropertyName: "namespace", |
||||
}, |
||||
}, |
||||
}) |
||||
} |
||||
|
||||
// NewSensuGoNotifier is the constructor for the Sensu Go Notifier.
|
||||
func NewSensuGoNotifier(model *models.AlertNotification) (alerting.Notifier, error) { |
||||
url := model.Settings.Get("url").MustString() |
||||
apikey := model.DecryptedValue("apikey", model.Settings.Get("apikey").MustString()) |
||||
|
||||
if url == "" { |
||||
return nil, alerting.ValidationError{Reason: "Could not find URL property in settings"} |
||||
} |
||||
if apikey == "" { |
||||
return nil, alerting.ValidationError{Reason: "Could not find the API Key property in settings"} |
||||
} |
||||
|
||||
return &SensuGoNotifier{ |
||||
NotifierBase: NewNotifierBase(model), |
||||
URL: url, |
||||
Entity: model.Settings.Get("entity").MustString(), |
||||
Check: model.Settings.Get("check").MustString(), |
||||
Namespace: model.Settings.Get("namespace").MustString(), |
||||
Handler: model.Settings.Get("handler").MustString(), |
||||
APIKey: apikey, |
||||
log: log.New("alerting.notifier.sensugo"), |
||||
}, nil |
||||
} |
||||
|
||||
// SensuGoNotifier is responsible for sending
|
||||
// alert notifications to Sensu Go.
|
||||
type SensuGoNotifier struct { |
||||
NotifierBase |
||||
URL string |
||||
Entity string |
||||
Check string |
||||
Namespace string |
||||
Handler string |
||||
APIKey string |
||||
log log.Logger |
||||
} |
||||
|
||||
// Notify send alert notification to Sensu Go
|
||||
func (sn *SensuGoNotifier) Notify(evalContext *alerting.EvalContext) error { |
||||
sn.log.Info("Sending Sensu Go result") |
||||
|
||||
var namespace string |
||||
|
||||
bodyJSON := simplejson.New() |
||||
// Sensu Go alerts require an entity and a check. We set it to the user-specified
|
||||
// value (optional), else we fallback and use the grafana rule anme and ruleID.
|
||||
if sn.Entity != "" { |
||||
bodyJSON.SetPath([]string{"entity", "metadata", "name"}, sn.Entity) |
||||
} else { |
||||
// Sensu Go alerts cannot have spaces in them
|
||||
bodyJSON.SetPath([]string{"entity", "metadata", "name"}, strings.ReplaceAll(evalContext.Rule.Name, " ", "_")) |
||||
} |
||||
if sn.Check != "" { |
||||
bodyJSON.SetPath([]string{"check", "metadata", "name"}, sn.Check) |
||||
} else { |
||||
bodyJSON.SetPath([]string{"check", "metadata", "name"}, "grafana_rule_"+strconv.FormatInt(evalContext.Rule.ID, 10)) |
||||
} |
||||
// Sensu Go requires the entity in an event specify its namespace. We set it to
|
||||
// the user-specified value (optional), else we fallback and use default
|
||||
if sn.Namespace != "" { |
||||
bodyJSON.SetPath([]string{"entity", "metadata", "namespace"}, sn.Namespace) |
||||
namespace = sn.Namespace |
||||
} else { |
||||
bodyJSON.SetPath([]string{"entity", "metadata", "namespace"}, "default") |
||||
namespace = "default" |
||||
} |
||||
// Sensu Go needs check output
|
||||
if evalContext.Rule.Message != "" { |
||||
bodyJSON.SetPath([]string{"check", "output"}, evalContext.Rule.Message) |
||||
} else { |
||||
bodyJSON.SetPath([]string{"check", "output"}, "Grafana Metric Condition Met") |
||||
} |
||||
// Sensu GO requires that the check portion of the event have an interval
|
||||
bodyJSON.SetPath([]string{"check", "interval"}, 86400) |
||||
|
||||
switch evalContext.Rule.State { |
||||
case "alerting": |
||||
bodyJSON.SetPath([]string{"check", "status"}, 2) |
||||
case "no_data": |
||||
bodyJSON.SetPath([]string{"check", "status"}, 1) |
||||
default: |
||||
bodyJSON.SetPath([]string{"check", "status"}, 0) |
||||
} |
||||
|
||||
if sn.Handler != "" { |
||||
bodyJSON.SetPath([]string{"check", "handlers"}, []string{sn.Handler}) |
||||
} |
||||
|
||||
ruleURL, err := evalContext.GetRuleURL() |
||||
if err == nil { |
||||
bodyJSON.Set("ruleUrl", ruleURL) |
||||
} |
||||
|
||||
labels := map[string]string{ |
||||
"ruleName": evalContext.Rule.Name, |
||||
"ruleId": strconv.FormatInt(evalContext.Rule.ID, 10), |
||||
"ruleURL": ruleURL, |
||||
} |
||||
|
||||
if sn.NeedsImage() && evalContext.ImagePublicURL != "" { |
||||
labels["imageUrl"] = evalContext.ImagePublicURL |
||||
} |
||||
|
||||
bodyJSON.SetPath([]string{"check", "metadata", "labels"}, labels) |
||||
|
||||
body, err := bodyJSON.MarshalJSON() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
cmd := &models.SendWebhookSync{ |
||||
Url: fmt.Sprintf("%s/api/core/v2/namespaces/%s/events", strings.TrimSuffix(sn.URL, "/"), namespace), |
||||
Body: string(body), |
||||
HttpMethod: "POST", |
||||
HttpHeader: map[string]string{ |
||||
"Content-Type": "application/json", |
||||
"Authorization": fmt.Sprintf("Key %s", sn.APIKey), |
||||
}, |
||||
} |
||||
if err := bus.DispatchCtx(evalContext.Ctx, cmd); err != nil { |
||||
sn.log.Error("Failed to send Sensu Go event", "error", err, "sensugo", sn.Name) |
||||
return err |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
@ -0,0 +1,57 @@ |
||||
package notifiers |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/grafana/pkg/components/simplejson" |
||||
"github.com/grafana/grafana/pkg/models" |
||||
) |
||||
|
||||
func TestSensuGoNotifier(t *testing.T) { |
||||
json := `{ }` |
||||
|
||||
settingsJSON, err := simplejson.NewJson([]byte(json)) |
||||
require.NoError(t, err) |
||||
model := &models.AlertNotification{ |
||||
Name: "Sensu Go", |
||||
Type: "sensugo", |
||||
Settings: settingsJSON, |
||||
} |
||||
|
||||
_, err = NewSensuGoNotifier(model) |
||||
require.Error(t, err) |
||||
|
||||
json = ` |
||||
{ |
||||
"url": "http://sensu-api.example.com:8080", |
||||
"entity": "grafana_instance_01", |
||||
"check": "grafana_rule_0", |
||||
"namespace": "default", |
||||
"handler": "myhandler", |
||||
"apikey": "abcdef0123456789abcdef" |
||||
}` |
||||
|
||||
settingsJSON, err = simplejson.NewJson([]byte(json)) |
||||
require.NoError(t, err) |
||||
model = &models.AlertNotification{ |
||||
Name: "Sensu Go", |
||||
Type: "sensugo", |
||||
Settings: settingsJSON, |
||||
} |
||||
|
||||
not, err := NewSensuGoNotifier(model) |
||||
require.NoError(t, err) |
||||
sensuGoNotifier := not.(*SensuGoNotifier) |
||||
|
||||
assert.Equal(t, "Sensu Go", sensuGoNotifier.Name) |
||||
assert.Equal(t, "sensugo", sensuGoNotifier.Type) |
||||
assert.Equal(t, "http://sensu-api.example.com:8080", sensuGoNotifier.URL) |
||||
assert.Equal(t, "grafana_instance_01", sensuGoNotifier.Entity) |
||||
assert.Equal(t, "grafana_rule_0", sensuGoNotifier.Check) |
||||
assert.Equal(t, "default", sensuGoNotifier.Namespace) |
||||
assert.Equal(t, "myhandler", sensuGoNotifier.Handler) |
||||
assert.Equal(t, "abcdef0123456789abcdef", sensuGoNotifier.APIKey) |
||||
} |
||||
Loading…
Reference in new issue