mirror of https://github.com/grafana/grafana
Alerting: Allow querying of Alerts from notifications (#32614)
* Alerting: Allow querying of Alerts from notifications * Wire everything up * Remove unused functions * Remove duplicate linepull/32759/head^2
gotjosh
5 years ago
committed by
GitHub
parent
2f3ef69b30
commit
fe67680c42
@ -0,0 +1,222 @@ |
||||
package notifier |
||||
|
||||
import ( |
||||
"regexp" |
||||
"sort" |
||||
"time" |
||||
|
||||
apimodels "github.com/grafana/alerting-api/pkg/api" |
||||
"github.com/pkg/errors" |
||||
v2 "github.com/prometheus/alertmanager/api/v2" |
||||
"github.com/prometheus/alertmanager/dispatch" |
||||
"github.com/prometheus/alertmanager/pkg/labels" |
||||
"github.com/prometheus/alertmanager/types" |
||||
prometheus_model "github.com/prometheus/common/model" |
||||
) |
||||
|
||||
var ( |
||||
ErrGetAlertsInternal = errors.New("unable to retrieve alerts(s) due to an internal error") |
||||
ErrGetAlertsBadPayload = errors.New("unable to retrieve alerts") |
||||
ErrGetAlertGroupsBadPayload = errors.New("unable to retrieve alerts groups") |
||||
) |
||||
|
||||
func (am *Alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.GettableAlerts, error) { |
||||
var ( |
||||
// Initialize result slice to prevent api returning `null` when there
|
||||
// are no alerts present
|
||||
res = apimodels.GettableAlerts{} |
||||
) |
||||
|
||||
matchers, err := parseFilter(filter) |
||||
if err != nil { |
||||
am.logger.Error("failed to parse matchers", "err", err) |
||||
return nil, errors.Wrap(ErrGetAlertsBadPayload, err.Error()) |
||||
} |
||||
|
||||
receiverFilter, err := parseReceivers(receivers) |
||||
if err != nil { |
||||
am.logger.Error("failed to parse receiver regex", "err", err) |
||||
return nil, errors.Wrap(ErrGetAlertsBadPayload, err.Error()) |
||||
} |
||||
|
||||
alerts := am.alerts.GetPending() |
||||
defer alerts.Close() |
||||
|
||||
alertFilter := am.alertFilter(matchers, silenced, inhibited, active) |
||||
now := time.Now() |
||||
|
||||
am.reloadConfigMtx.RLock() |
||||
for a := range alerts.Next() { |
||||
if err = alerts.Err(); err != nil { |
||||
break |
||||
} |
||||
|
||||
routes := am.route.Match(a.Labels) |
||||
receivers := make([]string, 0, len(routes)) |
||||
for _, r := range routes { |
||||
receivers = append(receivers, r.RouteOpts.Receiver) |
||||
} |
||||
|
||||
if receiverFilter != nil && !receiversMatchFilter(receivers, receiverFilter) { |
||||
continue |
||||
} |
||||
|
||||
if !alertFilter(a, now) { |
||||
continue |
||||
} |
||||
|
||||
alert := v2.AlertToOpenAPIAlert(a, am.marker.Status(a.Fingerprint()), receivers) |
||||
|
||||
res = append(res, alert) |
||||
} |
||||
am.reloadConfigMtx.RUnlock() |
||||
|
||||
if err != nil { |
||||
am.logger.Error("failed to iterate through the alerts", "err", err) |
||||
return nil, errors.Wrap(ErrGetAlertsInternal, err.Error()) |
||||
} |
||||
sort.Slice(res, func(i, j int) bool { |
||||
return *res[i].Fingerprint < *res[j].Fingerprint |
||||
}) |
||||
|
||||
return res, nil |
||||
} |
||||
|
||||
func (am *Alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.AlertGroups, error) { |
||||
matchers, err := parseFilter(filter) |
||||
if err != nil { |
||||
am.logger.Error("msg", "failed to parse matchers", "err", err) |
||||
return nil, errors.Wrap(ErrGetAlertGroupsBadPayload, err.Error()) |
||||
} |
||||
|
||||
receiverFilter, err := parseReceivers(receivers) |
||||
if err != nil { |
||||
am.logger.Error("msg", "failed to compile receiver regex", "err", err) |
||||
return nil, errors.Wrap(ErrGetAlertGroupsBadPayload, err.Error()) |
||||
} |
||||
|
||||
rf := func(receiverFilter *regexp.Regexp) func(r *dispatch.Route) bool { |
||||
return func(r *dispatch.Route) bool { |
||||
receiver := r.RouteOpts.Receiver |
||||
if receiverFilter != nil && !receiverFilter.MatchString(receiver) { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
}(receiverFilter) |
||||
|
||||
af := am.alertFilter(matchers, silenced, inhibited, active) |
||||
alertGroups, allReceivers := am.dispatcher.Groups(rf, af) |
||||
|
||||
res := make(apimodels.AlertGroups, 0, len(alertGroups)) |
||||
|
||||
for _, alertGroup := range alertGroups { |
||||
ag := &apimodels.AlertGroup{ |
||||
Receiver: &apimodels.Receiver{Name: &alertGroup.Receiver}, |
||||
Labels: v2.ModelLabelSetToAPILabelSet(alertGroup.Labels), |
||||
Alerts: make([]*apimodels.GettableAlert, 0, len(alertGroup.Alerts)), |
||||
} |
||||
|
||||
for _, alert := range alertGroup.Alerts { |
||||
fp := alert.Fingerprint() |
||||
receivers := allReceivers[fp] |
||||
status := am.marker.Status(fp) |
||||
apiAlert := v2.AlertToOpenAPIAlert(alert, status, receivers) |
||||
ag.Alerts = append(ag.Alerts, apiAlert) |
||||
} |
||||
res = append(res, ag) |
||||
} |
||||
|
||||
return res, nil |
||||
} |
||||
|
||||
func (am *Alertmanager) alertFilter(matchers []*labels.Matcher, silenced, inhibited, active bool) func(a *types.Alert, now time.Time) bool { |
||||
return func(a *types.Alert, now time.Time) bool { |
||||
if !a.EndsAt.IsZero() && a.EndsAt.Before(now) { |
||||
return false |
||||
} |
||||
|
||||
// Set alert's current status based on its label set.
|
||||
am.silencer.Mutes(a.Labels) |
||||
|
||||
// Get alert's current status after seeing if it is suppressed.
|
||||
status := am.marker.Status(a.Fingerprint()) |
||||
|
||||
if !active && status.State == types.AlertStateActive { |
||||
return false |
||||
} |
||||
|
||||
if !silenced && len(status.SilencedBy) != 0 { |
||||
return false |
||||
} |
||||
|
||||
if !inhibited && len(status.InhibitedBy) != 0 { |
||||
return false |
||||
} |
||||
|
||||
return alertMatchesFilterLabels(&a.Alert, matchers) |
||||
} |
||||
} |
||||
|
||||
func alertMatchesFilterLabels(a *prometheus_model.Alert, matchers []*labels.Matcher) bool { |
||||
sms := make(map[string]string) |
||||
for name, value := range a.Labels { |
||||
sms[string(name)] = string(value) |
||||
} |
||||
return matchFilterLabels(matchers, sms) |
||||
} |
||||
|
||||
func matchFilterLabels(matchers []*labels.Matcher, sms map[string]string) bool { |
||||
for _, m := range matchers { |
||||
v, prs := sms[m.Name] |
||||
switch m.Type { |
||||
case labels.MatchNotRegexp, labels.MatchNotEqual: |
||||
if m.Value == "" && prs { |
||||
continue |
||||
} |
||||
if !m.Matches(v) { |
||||
return false |
||||
} |
||||
default: |
||||
if m.Value == "" && !prs { |
||||
continue |
||||
} |
||||
if !m.Matches(v) { |
||||
return false |
||||
} |
||||
} |
||||
} |
||||
|
||||
return true |
||||
} |
||||
|
||||
func parseReceivers(receivers string) (*regexp.Regexp, error) { |
||||
if receivers == "" { |
||||
return nil, nil |
||||
} |
||||
|
||||
return regexp.Compile("^(?:" + receivers + ")$") |
||||
} |
||||
|
||||
func parseFilter(filter []string) ([]*labels.Matcher, error) { |
||||
matchers := make([]*labels.Matcher, 0, len(filter)) |
||||
for _, matcherString := range filter { |
||||
matcher, err := labels.ParseMatcher(matcherString) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
matchers = append(matchers, matcher) |
||||
} |
||||
return matchers, nil |
||||
} |
||||
|
||||
func receiversMatchFilter(receivers []string, filter *regexp.Regexp) bool { |
||||
for _, r := range receivers { |
||||
if filter.MatchString(r) { |
||||
return true |
||||
} |
||||
} |
||||
|
||||
return false |
||||
} |
||||
@ -0,0 +1,132 @@ |
||||
package alerting |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"net/http" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/ngalert/models" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
|
||||
"github.com/grafana/grafana/pkg/tests/testinfra" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestAlertAndGroupsQuery(t *testing.T) { |
||||
dir, path := testinfra.CreateGrafDir(t, testinfra.GrafanaOpts{ |
||||
EnableFeatureToggles: []string{"ngalert"}, |
||||
}) |
||||
store := setupDB(t, dir) |
||||
grafanaListedAddr := testinfra.StartGrafana(t, dir, path, store) |
||||
|
||||
// When there are no alerts available, it returns an empty list.
|
||||
{ |
||||
alertsURL := fmt.Sprintf("http://%s/api/alertmanager/grafana/api/v2/alerts", grafanaListedAddr) |
||||
// nolint:gosec
|
||||
resp, err := http.Get(alertsURL) |
||||
require.NoError(t, err) |
||||
t.Cleanup(func() { |
||||
err := resp.Body.Close() |
||||
require.NoError(t, err) |
||||
}) |
||||
b, err := ioutil.ReadAll(resp.Body) |
||||
require.NoError(t, err) |
||||
require.Equal(t, 200, resp.StatusCode) |
||||
require.JSONEq(t, "[]", string(b)) |
||||
} |
||||
|
||||
// When are there no alerts available, it returns an empty list of groups.
|
||||
{ |
||||
alertsURL := fmt.Sprintf("http://%s/api/alertmanager/grafana/api/v2/alerts/groups", grafanaListedAddr) |
||||
// nolint:gosec
|
||||
resp, err := http.Get(alertsURL) |
||||
require.NoError(t, err) |
||||
t.Cleanup(func() { |
||||
err := resp.Body.Close() |
||||
require.NoError(t, err) |
||||
}) |
||||
b, err := ioutil.ReadAll(resp.Body) |
||||
require.NoError(t, err) |
||||
require.NoError(t, err) |
||||
require.Equal(t, 200, resp.StatusCode) |
||||
require.JSONEq(t, "[]", string(b)) |
||||
} |
||||
} |
||||
|
||||
func setupDB(t *testing.T, dir string) *sqlstore.SQLStore { |
||||
store := testinfra.SetUpDatabase(t, dir) |
||||
// Let's make sure we create a default configuration from which we can start.
|
||||
err := store.WithDbSession(context.Background(), func(sess *sqlstore.DBSession) error { |
||||
_, err := sess.Insert(&models.AlertConfiguration{ |
||||
ID: 1, |
||||
AlertmanagerConfiguration: AMConfigFixture, |
||||
ConfigurationVersion: "v1", |
||||
CreatedAt: time.Now(), |
||||
}) |
||||
return err |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
return store |
||||
} |
||||
|
||||
var AMConfigFixture = ` |
||||
{ |
||||
"template_files": {}, |
||||
"alertmanager_config": { |
||||
"global": { |
||||
"resolve_timeout": "4m", |
||||
"http_config": { |
||||
"BasicAuth": null, |
||||
"Authorization": null, |
||||
"BearerToken": "", |
||||
"BearerTokenFile": "", |
||||
"ProxyURL": {}, |
||||
"TLSConfig": { |
||||
"CAFile": "", |
||||
"CertFile": "", |
||||
"KeyFile": "", |
||||
"ServerName": "", |
||||
"InsecureSkipVerify": false |
||||
}, |
||||
"FollowRedirects": true |
||||
}, |
||||
"smtp_from": "youraddress@example.org", |
||||
"smtp_hello": "localhost", |
||||
"smtp_smarthost": "localhost:25", |
||||
"smtp_require_tls": true, |
||||
"pagerduty_url": "https://events.pagerduty.com/v2/enqueue", |
||||
"opsgenie_api_url": "https://api.opsgenie.com/", |
||||
"wechat_api_url": "https://qyapi.weixin.qq.com/cgi-bin/", |
||||
"victorops_api_url": "https://alert.victorops.com/integrations/generic/20131114/alert/" |
||||
}, |
||||
"route": { |
||||
"receiver": "example-email" |
||||
}, |
||||
"templates": [], |
||||
"receivers": [ |
||||
{ |
||||
"name": "example-email", |
||||
"email_configs": [ |
||||
{ |
||||
"send_resolved": false, |
||||
"to": "youraddress@example.org", |
||||
"smarthost": "", |
||||
"html": "{{ template \"email.default.html\" . }}", |
||||
"tls_config": { |
||||
"CAFile": "", |
||||
"CertFile": "", |
||||
"KeyFile": "", |
||||
"ServerName": "", |
||||
"InsecureSkipVerify": false |
||||
} |
||||
} |
||||
] |
||||
} |
||||
] |
||||
} |
||||
} |
||||
` |
||||
Loading…
Reference in new issue