mirror of https://github.com/grafana/grafana
Alerting: Use `alerting.GrafanaAlertmanager` instead of initialising Alertmanager components directly (#61230)
* Alerting: Use `alerting.GrafanaAlertmanager` instead of initialising Alertmanager components directlypull/61506/head
gotjosh
2 years ago
committed by
GitHub
parent
58c4c95e92
commit
e7cd6eb13c
@ -1,223 +1,13 @@ |
||||
package notifier |
||||
|
||||
import ( |
||||
"fmt" |
||||
"regexp" |
||||
"sort" |
||||
"time" |
||||
|
||||
"github.com/grafana/alerting/alerting" |
||||
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
||||
v2 "github.com/prometheus/alertmanager/api/v2" |
||||
"github.com/prometheus/alertmanager/dispatch" |
||||
"github.com/prometheus/alertmanager/pkg/labels" |
||||
"github.com/prometheus/alertmanager/types" |
||||
prometheus_model "github.com/prometheus/common/model" |
||||
) |
||||
|
||||
func (am *Alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.GettableAlerts, error) { |
||||
var ( |
||||
// Initialize result slice to prevent api returning `null` when there
|
||||
// are no alerts present
|
||||
res = apimodels.GettableAlerts{} |
||||
) |
||||
|
||||
if !am.Ready() { |
||||
return res, alerting.ErrGetAlertsUnavailable |
||||
} |
||||
|
||||
matchers, err := parseFilter(filter) |
||||
if err != nil { |
||||
am.logger.Error("failed to parse matchers", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsBadPayload) |
||||
} |
||||
|
||||
receiverFilter, err := parseReceivers(receivers) |
||||
if err != nil { |
||||
am.logger.Error("failed to parse receiver regex", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsBadPayload) |
||||
} |
||||
|
||||
alerts := am.alerts.GetPending() |
||||
defer alerts.Close() |
||||
|
||||
alertFilter := am.alertFilter(matchers, silenced, inhibited, active) |
||||
now := time.Now() |
||||
|
||||
am.reloadConfigMtx.RLock() |
||||
for a := range alerts.Next() { |
||||
if err = alerts.Err(); err != nil { |
||||
break |
||||
} |
||||
|
||||
routes := am.route.Match(a.Labels) |
||||
receivers := make([]string, 0, len(routes)) |
||||
for _, r := range routes { |
||||
receivers = append(receivers, r.RouteOpts.Receiver) |
||||
} |
||||
|
||||
if receiverFilter != nil && !receiversMatchFilter(receivers, receiverFilter) { |
||||
continue |
||||
} |
||||
|
||||
if !alertFilter(a, now) { |
||||
continue |
||||
} |
||||
|
||||
alert := v2.AlertToOpenAPIAlert(a, am.marker.Status(a.Fingerprint()), receivers) |
||||
|
||||
res = append(res, alert) |
||||
} |
||||
am.reloadConfigMtx.RUnlock() |
||||
|
||||
if err != nil { |
||||
am.logger.Error("failed to iterate through the alerts", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsInternal) |
||||
} |
||||
sort.Slice(res, func(i, j int) bool { |
||||
return *res[i].Fingerprint < *res[j].Fingerprint |
||||
}) |
||||
|
||||
return res, nil |
||||
} |
||||
|
||||
func (am *Alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.AlertGroups, error) { |
||||
matchers, err := parseFilter(filter) |
||||
if err != nil { |
||||
am.logger.Error("msg", "failed to parse matchers", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertGroupsBadPayload) |
||||
} |
||||
|
||||
receiverFilter, err := parseReceivers(receivers) |
||||
if err != nil { |
||||
am.logger.Error("msg", "failed to compile receiver regex", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertGroupsBadPayload) |
||||
} |
||||
|
||||
rf := func(receiverFilter *regexp.Regexp) func(r *dispatch.Route) bool { |
||||
return func(r *dispatch.Route) bool { |
||||
receiver := r.RouteOpts.Receiver |
||||
if receiverFilter != nil && !receiverFilter.MatchString(receiver) { |
||||
return false |
||||
} |
||||
return true |
||||
} |
||||
}(receiverFilter) |
||||
|
||||
af := am.alertFilter(matchers, silenced, inhibited, active) |
||||
alertGroups, allReceivers := am.dispatcher.Groups(rf, af) |
||||
|
||||
res := make(apimodels.AlertGroups, 0, len(alertGroups)) |
||||
|
||||
for _, alertGroup := range alertGroups { |
||||
ag := &apimodels.AlertGroup{ |
||||
Receiver: &apimodels.Receiver{Name: &alertGroup.Receiver}, |
||||
Labels: v2.ModelLabelSetToAPILabelSet(alertGroup.Labels), |
||||
Alerts: make([]*apimodels.GettableAlert, 0, len(alertGroup.Alerts)), |
||||
} |
||||
|
||||
for _, alert := range alertGroup.Alerts { |
||||
fp := alert.Fingerprint() |
||||
receivers := allReceivers[fp] |
||||
status := am.marker.Status(fp) |
||||
apiAlert := v2.AlertToOpenAPIAlert(alert, status, receivers) |
||||
ag.Alerts = append(ag.Alerts, apiAlert) |
||||
} |
||||
res = append(res, ag) |
||||
} |
||||
|
||||
return res, nil |
||||
} |
||||
|
||||
func (am *Alertmanager) alertFilter(matchers []*labels.Matcher, silenced, inhibited, active bool) func(a *types.Alert, now time.Time) bool { |
||||
return func(a *types.Alert, now time.Time) bool { |
||||
if !a.EndsAt.IsZero() && a.EndsAt.Before(now) { |
||||
return false |
||||
} |
||||
|
||||
// Set alert's current status based on its label set.
|
||||
am.silencer.Mutes(a.Labels) |
||||
am.inhibitor.Mutes(a.Labels) |
||||
|
||||
// Get alert's current status after seeing if it is suppressed.
|
||||
status := am.marker.Status(a.Fingerprint()) |
||||
|
||||
if !active && status.State == types.AlertStateActive { |
||||
return false |
||||
} |
||||
|
||||
if !silenced && len(status.SilencedBy) != 0 { |
||||
return false |
||||
} |
||||
|
||||
if !inhibited && len(status.InhibitedBy) != 0 { |
||||
return false |
||||
} |
||||
|
||||
return alertMatchesFilterLabels(&a.Alert, matchers) |
||||
} |
||||
} |
||||
|
||||
func alertMatchesFilterLabels(a *prometheus_model.Alert, matchers []*labels.Matcher) bool { |
||||
sms := make(map[string]string) |
||||
for name, value := range a.Labels { |
||||
sms[string(name)] = string(value) |
||||
} |
||||
return matchFilterLabels(matchers, sms) |
||||
} |
||||
|
||||
func matchFilterLabels(matchers []*labels.Matcher, sms map[string]string) bool { |
||||
for _, m := range matchers { |
||||
v, prs := sms[m.Name] |
||||
switch m.Type { |
||||
case labels.MatchNotRegexp, labels.MatchNotEqual: |
||||
if m.Value == "" && prs { |
||||
continue |
||||
} |
||||
if !m.Matches(v) { |
||||
return false |
||||
} |
||||
default: |
||||
if m.Value == "" && !prs { |
||||
continue |
||||
} |
||||
if !m.Matches(v) { |
||||
return false |
||||
} |
||||
} |
||||
} |
||||
|
||||
return true |
||||
func (am *Alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (alerting.GettableAlerts, error) { |
||||
return am.Base.GetAlerts(active, silenced, inhibited, filter, receivers) |
||||
} |
||||
|
||||
func parseReceivers(receivers string) (*regexp.Regexp, error) { |
||||
if receivers == "" { |
||||
return nil, nil |
||||
} |
||||
|
||||
return regexp.Compile("^(?:" + receivers + ")$") |
||||
} |
||||
|
||||
func parseFilter(filter []string) ([]*labels.Matcher, error) { |
||||
matchers := make([]*labels.Matcher, 0, len(filter)) |
||||
for _, matcherString := range filter { |
||||
matcher, err := labels.ParseMatcher(matcherString) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
matchers = append(matchers, matcher) |
||||
} |
||||
return matchers, nil |
||||
} |
||||
|
||||
func receiversMatchFilter(receivers []string, filter *regexp.Regexp) bool { |
||||
for _, r := range receivers { |
||||
if filter.MatchString(r) { |
||||
return true |
||||
} |
||||
} |
||||
|
||||
return false |
||||
func (am *Alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (alerting.AlertGroups, error) { |
||||
return am.Base.GetAlertGroups(active, silenced, inhibited, filter, receivers) |
||||
} |
||||
|
||||
@ -1,113 +1,21 @@ |
||||
package notifier |
||||
|
||||
import ( |
||||
"errors" |
||||
"fmt" |
||||
"time" |
||||
|
||||
"github.com/grafana/alerting/alerting" |
||||
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
||||
v2 "github.com/prometheus/alertmanager/api/v2" |
||||
"github.com/prometheus/alertmanager/silence" |
||||
) |
||||
|
||||
// ListSilences retrieves a list of stored silences. It supports a set of labels as filters.
|
||||
func (am *Alertmanager) ListSilences(filter []string) (apimodels.GettableSilences, error) { |
||||
matchers, err := parseFilter(filter) |
||||
if err != nil { |
||||
am.logger.Error("failed to parse matchers", "error", err) |
||||
return nil, fmt.Errorf("%s: %w", alerting.ErrListSilencesBadPayload.Error(), err) |
||||
} |
||||
|
||||
psils, _, err := am.silences.Query() |
||||
if err != nil { |
||||
am.logger.Error(alerting.ErrGetSilencesInternal.Error(), "error", err) |
||||
return nil, fmt.Errorf("%s: %w", alerting.ErrGetSilencesInternal.Error(), err) |
||||
} |
||||
|
||||
sils := apimodels.GettableSilences{} |
||||
for _, ps := range psils { |
||||
if !v2.CheckSilenceMatchesFilterLabels(ps, matchers) { |
||||
continue |
||||
} |
||||
silence, err := v2.GettableSilenceFromProto(ps) |
||||
if err != nil { |
||||
am.logger.Error("unmarshaling from protobuf failed", "error", err) |
||||
return apimodels.GettableSilences{}, fmt.Errorf("%s: failed to convert internal silence to API silence: %w", |
||||
alerting.ErrGetSilencesInternal.Error(), err) |
||||
} |
||||
sils = append(sils, &silence) |
||||
} |
||||
|
||||
v2.SortSilences(sils) |
||||
|
||||
return sils, nil |
||||
func (am *Alertmanager) ListSilences(filter []string) (alerting.GettableSilences, error) { |
||||
return am.Base.ListSilences(filter) |
||||
} |
||||
|
||||
// GetSilence retrieves a silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
|
||||
func (am *Alertmanager) GetSilence(silenceID string) (apimodels.GettableSilence, error) { |
||||
sils, _, err := am.silences.Query(silence.QIDs(silenceID)) |
||||
if err != nil { |
||||
return apimodels.GettableSilence{}, fmt.Errorf("%s: %w", alerting.ErrGetSilencesInternal.Error(), err) |
||||
} |
||||
|
||||
if len(sils) == 0 { |
||||
am.logger.Error("failed to find silence", "error", err, "id", sils) |
||||
return apimodels.GettableSilence{}, alerting.ErrSilenceNotFound |
||||
} |
||||
|
||||
sil, err := v2.GettableSilenceFromProto(sils[0]) |
||||
if err != nil { |
||||
am.logger.Error("unmarshaling from protobuf failed", "error", err) |
||||
return apimodels.GettableSilence{}, fmt.Errorf("%s: failed to convert internal silence to API silence: %w", |
||||
alerting.ErrGetSilencesInternal.Error(), err) |
||||
} |
||||
|
||||
return sil, nil |
||||
func (am *Alertmanager) GetSilence(silenceID string) (alerting.GettableSilence, error) { |
||||
return am.Base.GetSilence(silenceID) |
||||
} |
||||
|
||||
// CreateSilence persists the provided silence and returns the silence ID if successful.
|
||||
func (am *Alertmanager) CreateSilence(ps *apimodels.PostableSilence) (string, error) { |
||||
sil, err := v2.PostableSilenceToProto(ps) |
||||
if err != nil { |
||||
am.logger.Error("marshaling to protobuf failed", "error", err) |
||||
return "", fmt.Errorf("%s: failed to convert API silence to internal silence: %w", |
||||
alerting.ErrCreateSilenceBadPayload.Error(), err) |
||||
} |
||||
|
||||
if sil.StartsAt.After(sil.EndsAt) || sil.StartsAt.Equal(sil.EndsAt) { |
||||
msg := "start time must be before end time" |
||||
am.logger.Error(msg, "error", "starts_at", sil.StartsAt, "ends_at", sil.EndsAt) |
||||
return "", fmt.Errorf("%s: %w", msg, alerting.ErrCreateSilenceBadPayload) |
||||
} |
||||
|
||||
if sil.EndsAt.Before(time.Now()) { |
||||
msg := "end time can't be in the past" |
||||
am.logger.Error(msg, "ends_at", sil.EndsAt) |
||||
return "", fmt.Errorf("%s: %w", msg, alerting.ErrCreateSilenceBadPayload) |
||||
} |
||||
|
||||
silenceID, err := am.silences.Set(sil) |
||||
if err != nil { |
||||
am.logger.Error("msg", "unable to save silence", "error", err) |
||||
if errors.Is(err, silence.ErrNotFound) { |
||||
return "", alerting.ErrSilenceNotFound |
||||
} |
||||
return "", fmt.Errorf("unable to save silence: %s: %w", err.Error(), alerting.ErrCreateSilenceBadPayload) |
||||
} |
||||
|
||||
return silenceID, nil |
||||
func (am *Alertmanager) CreateSilence(ps *alerting.PostableSilence) (string, error) { |
||||
return am.Base.CreateSilence(ps) |
||||
} |
||||
|
||||
// DeleteSilence looks for and expires the silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
|
||||
func (am *Alertmanager) DeleteSilence(silenceID string) error { |
||||
if err := am.silences.Expire(silenceID); err != nil { |
||||
if errors.Is(err, silence.ErrNotFound) { |
||||
return alerting.ErrSilenceNotFound |
||||
} |
||||
return fmt.Errorf("%s: %w", err.Error(), alerting.ErrDeleteSilenceInternal) |
||||
} |
||||
|
||||
return nil |
||||
return am.Base.DeleteSilence(silenceID) |
||||
} |
||||
|
||||
@ -1,16 +1,22 @@ |
||||
package notifier |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
|
||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
||||
) |
||||
|
||||
// TODO: We no longer do apimodels at this layer, move it to the API.
|
||||
func (am *Alertmanager) GetStatus() apimodels.GettableStatus { |
||||
am.reloadConfigMtx.RLock() |
||||
defer am.reloadConfigMtx.RUnlock() |
||||
config := &apimodels.PostableUserConfig{} |
||||
status := am.Base.GetStatus() |
||||
if status == nil { |
||||
return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) |
||||
} |
||||
|
||||
config := apimodels.PostableApiAlertingConfig{} |
||||
if am.ready() { |
||||
config = am.config.AlertmanagerConfig |
||||
if err := json.Unmarshal(status, config); err != nil { |
||||
am.logger.Error("unable to unmarshall alertmanager config", "Err", err) |
||||
} |
||||
return *apimodels.NewGettableStatus(&config) |
||||
|
||||
return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) |
||||
} |
||||
|
||||
Loading…
Reference in new issue