mirror of https://github.com/grafana/grafana
Alerting: Use `alerting.GrafanaAlertmanager` instead of initialising Alertmanager components directly (#61230)
* Alerting: Use `alerting.GrafanaAlertmanager` instead of initialising Alertmanager components directlypull/61506/head
gotjosh
2 years ago
committed by
GitHub
parent
58c4c95e92
commit
e7cd6eb13c
@ -1,223 +1,13 @@ |
|||||||
package notifier |
package notifier |
||||||
|
|
||||||
import ( |
import ( |
||||||
"fmt" |
|
||||||
"regexp" |
|
||||||
"sort" |
|
||||||
"time" |
|
||||||
|
|
||||||
"github.com/grafana/alerting/alerting" |
"github.com/grafana/alerting/alerting" |
||||||
|
|
||||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
|
||||||
v2 "github.com/prometheus/alertmanager/api/v2" |
|
||||||
"github.com/prometheus/alertmanager/dispatch" |
|
||||||
"github.com/prometheus/alertmanager/pkg/labels" |
|
||||||
"github.com/prometheus/alertmanager/types" |
|
||||||
prometheus_model "github.com/prometheus/common/model" |
|
||||||
) |
) |
||||||
|
|
||||||
func (am *Alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.GettableAlerts, error) { |
func (am *Alertmanager) GetAlerts(active, silenced, inhibited bool, filter []string, receivers string) (alerting.GettableAlerts, error) { |
||||||
var ( |
return am.Base.GetAlerts(active, silenced, inhibited, filter, receivers) |
||||||
// Initialize result slice to prevent api returning `null` when there
|
|
||||||
// are no alerts present
|
|
||||||
res = apimodels.GettableAlerts{} |
|
||||||
) |
|
||||||
|
|
||||||
if !am.Ready() { |
|
||||||
return res, alerting.ErrGetAlertsUnavailable |
|
||||||
} |
|
||||||
|
|
||||||
matchers, err := parseFilter(filter) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("failed to parse matchers", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
receiverFilter, err := parseReceivers(receivers) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("failed to parse receiver regex", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
alerts := am.alerts.GetPending() |
|
||||||
defer alerts.Close() |
|
||||||
|
|
||||||
alertFilter := am.alertFilter(matchers, silenced, inhibited, active) |
|
||||||
now := time.Now() |
|
||||||
|
|
||||||
am.reloadConfigMtx.RLock() |
|
||||||
for a := range alerts.Next() { |
|
||||||
if err = alerts.Err(); err != nil { |
|
||||||
break |
|
||||||
} |
|
||||||
|
|
||||||
routes := am.route.Match(a.Labels) |
|
||||||
receivers := make([]string, 0, len(routes)) |
|
||||||
for _, r := range routes { |
|
||||||
receivers = append(receivers, r.RouteOpts.Receiver) |
|
||||||
} |
|
||||||
|
|
||||||
if receiverFilter != nil && !receiversMatchFilter(receivers, receiverFilter) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
if !alertFilter(a, now) { |
|
||||||
continue |
|
||||||
} |
|
||||||
|
|
||||||
alert := v2.AlertToOpenAPIAlert(a, am.marker.Status(a.Fingerprint()), receivers) |
|
||||||
|
|
||||||
res = append(res, alert) |
|
||||||
} |
|
||||||
am.reloadConfigMtx.RUnlock() |
|
||||||
|
|
||||||
if err != nil { |
|
||||||
am.logger.Error("failed to iterate through the alerts", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertsInternal) |
|
||||||
} |
|
||||||
sort.Slice(res, func(i, j int) bool { |
|
||||||
return *res[i].Fingerprint < *res[j].Fingerprint |
|
||||||
}) |
|
||||||
|
|
||||||
return res, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (am *Alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (apimodels.AlertGroups, error) { |
|
||||||
matchers, err := parseFilter(filter) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("msg", "failed to parse matchers", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertGroupsBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
receiverFilter, err := parseReceivers(receivers) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("msg", "failed to compile receiver regex", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", err.Error(), alerting.ErrGetAlertGroupsBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
rf := func(receiverFilter *regexp.Regexp) func(r *dispatch.Route) bool { |
|
||||||
return func(r *dispatch.Route) bool { |
|
||||||
receiver := r.RouteOpts.Receiver |
|
||||||
if receiverFilter != nil && !receiverFilter.MatchString(receiver) { |
|
||||||
return false |
|
||||||
} |
|
||||||
return true |
|
||||||
} |
|
||||||
}(receiverFilter) |
|
||||||
|
|
||||||
af := am.alertFilter(matchers, silenced, inhibited, active) |
|
||||||
alertGroups, allReceivers := am.dispatcher.Groups(rf, af) |
|
||||||
|
|
||||||
res := make(apimodels.AlertGroups, 0, len(alertGroups)) |
|
||||||
|
|
||||||
for _, alertGroup := range alertGroups { |
|
||||||
ag := &apimodels.AlertGroup{ |
|
||||||
Receiver: &apimodels.Receiver{Name: &alertGroup.Receiver}, |
|
||||||
Labels: v2.ModelLabelSetToAPILabelSet(alertGroup.Labels), |
|
||||||
Alerts: make([]*apimodels.GettableAlert, 0, len(alertGroup.Alerts)), |
|
||||||
} |
|
||||||
|
|
||||||
for _, alert := range alertGroup.Alerts { |
|
||||||
fp := alert.Fingerprint() |
|
||||||
receivers := allReceivers[fp] |
|
||||||
status := am.marker.Status(fp) |
|
||||||
apiAlert := v2.AlertToOpenAPIAlert(alert, status, receivers) |
|
||||||
ag.Alerts = append(ag.Alerts, apiAlert) |
|
||||||
} |
|
||||||
res = append(res, ag) |
|
||||||
} |
|
||||||
|
|
||||||
return res, nil |
|
||||||
} |
|
||||||
|
|
||||||
func (am *Alertmanager) alertFilter(matchers []*labels.Matcher, silenced, inhibited, active bool) func(a *types.Alert, now time.Time) bool { |
|
||||||
return func(a *types.Alert, now time.Time) bool { |
|
||||||
if !a.EndsAt.IsZero() && a.EndsAt.Before(now) { |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
// Set alert's current status based on its label set.
|
|
||||||
am.silencer.Mutes(a.Labels) |
|
||||||
am.inhibitor.Mutes(a.Labels) |
|
||||||
|
|
||||||
// Get alert's current status after seeing if it is suppressed.
|
|
||||||
status := am.marker.Status(a.Fingerprint()) |
|
||||||
|
|
||||||
if !active && status.State == types.AlertStateActive { |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
if !silenced && len(status.SilencedBy) != 0 { |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
if !inhibited && len(status.InhibitedBy) != 0 { |
|
||||||
return false |
|
||||||
} |
|
||||||
|
|
||||||
return alertMatchesFilterLabels(&a.Alert, matchers) |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
func alertMatchesFilterLabels(a *prometheus_model.Alert, matchers []*labels.Matcher) bool { |
|
||||||
sms := make(map[string]string) |
|
||||||
for name, value := range a.Labels { |
|
||||||
sms[string(name)] = string(value) |
|
||||||
} |
|
||||||
return matchFilterLabels(matchers, sms) |
|
||||||
} |
|
||||||
|
|
||||||
func matchFilterLabels(matchers []*labels.Matcher, sms map[string]string) bool { |
|
||||||
for _, m := range matchers { |
|
||||||
v, prs := sms[m.Name] |
|
||||||
switch m.Type { |
|
||||||
case labels.MatchNotRegexp, labels.MatchNotEqual: |
|
||||||
if m.Value == "" && prs { |
|
||||||
continue |
|
||||||
} |
|
||||||
if !m.Matches(v) { |
|
||||||
return false |
|
||||||
} |
|
||||||
default: |
|
||||||
if m.Value == "" && !prs { |
|
||||||
continue |
|
||||||
} |
|
||||||
if !m.Matches(v) { |
|
||||||
return false |
|
||||||
} |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return true |
|
||||||
} |
} |
||||||
|
|
||||||
func parseReceivers(receivers string) (*regexp.Regexp, error) { |
func (am *Alertmanager) GetAlertGroups(active, silenced, inhibited bool, filter []string, receivers string) (alerting.AlertGroups, error) { |
||||||
if receivers == "" { |
return am.Base.GetAlertGroups(active, silenced, inhibited, filter, receivers) |
||||||
return nil, nil |
|
||||||
} |
|
||||||
|
|
||||||
return regexp.Compile("^(?:" + receivers + ")$") |
|
||||||
} |
|
||||||
|
|
||||||
func parseFilter(filter []string) ([]*labels.Matcher, error) { |
|
||||||
matchers := make([]*labels.Matcher, 0, len(filter)) |
|
||||||
for _, matcherString := range filter { |
|
||||||
matcher, err := labels.ParseMatcher(matcherString) |
|
||||||
if err != nil { |
|
||||||
return nil, err |
|
||||||
} |
|
||||||
|
|
||||||
matchers = append(matchers, matcher) |
|
||||||
} |
|
||||||
return matchers, nil |
|
||||||
} |
|
||||||
|
|
||||||
func receiversMatchFilter(receivers []string, filter *regexp.Regexp) bool { |
|
||||||
for _, r := range receivers { |
|
||||||
if filter.MatchString(r) { |
|
||||||
return true |
|
||||||
} |
|
||||||
} |
|
||||||
|
|
||||||
return false |
|
||||||
} |
} |
||||||
|
|||||||
@ -1,113 +1,21 @@ |
|||||||
package notifier |
package notifier |
||||||
|
|
||||||
import ( |
import ( |
||||||
"errors" |
|
||||||
"fmt" |
|
||||||
"time" |
|
||||||
|
|
||||||
"github.com/grafana/alerting/alerting" |
"github.com/grafana/alerting/alerting" |
||||||
|
|
||||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
|
||||||
v2 "github.com/prometheus/alertmanager/api/v2" |
|
||||||
"github.com/prometheus/alertmanager/silence" |
|
||||||
) |
) |
||||||
|
|
||||||
// ListSilences retrieves a list of stored silences. It supports a set of labels as filters.
|
func (am *Alertmanager) ListSilences(filter []string) (alerting.GettableSilences, error) { |
||||||
func (am *Alertmanager) ListSilences(filter []string) (apimodels.GettableSilences, error) { |
return am.Base.ListSilences(filter) |
||||||
matchers, err := parseFilter(filter) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("failed to parse matchers", "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", alerting.ErrListSilencesBadPayload.Error(), err) |
|
||||||
} |
|
||||||
|
|
||||||
psils, _, err := am.silences.Query() |
|
||||||
if err != nil { |
|
||||||
am.logger.Error(alerting.ErrGetSilencesInternal.Error(), "error", err) |
|
||||||
return nil, fmt.Errorf("%s: %w", alerting.ErrGetSilencesInternal.Error(), err) |
|
||||||
} |
|
||||||
|
|
||||||
sils := apimodels.GettableSilences{} |
|
||||||
for _, ps := range psils { |
|
||||||
if !v2.CheckSilenceMatchesFilterLabels(ps, matchers) { |
|
||||||
continue |
|
||||||
} |
|
||||||
silence, err := v2.GettableSilenceFromProto(ps) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("unmarshaling from protobuf failed", "error", err) |
|
||||||
return apimodels.GettableSilences{}, fmt.Errorf("%s: failed to convert internal silence to API silence: %w", |
|
||||||
alerting.ErrGetSilencesInternal.Error(), err) |
|
||||||
} |
|
||||||
sils = append(sils, &silence) |
|
||||||
} |
|
||||||
|
|
||||||
v2.SortSilences(sils) |
|
||||||
|
|
||||||
return sils, nil |
|
||||||
} |
} |
||||||
|
|
||||||
// GetSilence retrieves a silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
|
func (am *Alertmanager) GetSilence(silenceID string) (alerting.GettableSilence, error) { |
||||||
func (am *Alertmanager) GetSilence(silenceID string) (apimodels.GettableSilence, error) { |
return am.Base.GetSilence(silenceID) |
||||||
sils, _, err := am.silences.Query(silence.QIDs(silenceID)) |
|
||||||
if err != nil { |
|
||||||
return apimodels.GettableSilence{}, fmt.Errorf("%s: %w", alerting.ErrGetSilencesInternal.Error(), err) |
|
||||||
} |
|
||||||
|
|
||||||
if len(sils) == 0 { |
|
||||||
am.logger.Error("failed to find silence", "error", err, "id", sils) |
|
||||||
return apimodels.GettableSilence{}, alerting.ErrSilenceNotFound |
|
||||||
} |
|
||||||
|
|
||||||
sil, err := v2.GettableSilenceFromProto(sils[0]) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("unmarshaling from protobuf failed", "error", err) |
|
||||||
return apimodels.GettableSilence{}, fmt.Errorf("%s: failed to convert internal silence to API silence: %w", |
|
||||||
alerting.ErrGetSilencesInternal.Error(), err) |
|
||||||
} |
|
||||||
|
|
||||||
return sil, nil |
|
||||||
} |
} |
||||||
|
|
||||||
// CreateSilence persists the provided silence and returns the silence ID if successful.
|
func (am *Alertmanager) CreateSilence(ps *alerting.PostableSilence) (string, error) { |
||||||
func (am *Alertmanager) CreateSilence(ps *apimodels.PostableSilence) (string, error) { |
return am.Base.CreateSilence(ps) |
||||||
sil, err := v2.PostableSilenceToProto(ps) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("marshaling to protobuf failed", "error", err) |
|
||||||
return "", fmt.Errorf("%s: failed to convert API silence to internal silence: %w", |
|
||||||
alerting.ErrCreateSilenceBadPayload.Error(), err) |
|
||||||
} |
|
||||||
|
|
||||||
if sil.StartsAt.After(sil.EndsAt) || sil.StartsAt.Equal(sil.EndsAt) { |
|
||||||
msg := "start time must be before end time" |
|
||||||
am.logger.Error(msg, "error", "starts_at", sil.StartsAt, "ends_at", sil.EndsAt) |
|
||||||
return "", fmt.Errorf("%s: %w", msg, alerting.ErrCreateSilenceBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
if sil.EndsAt.Before(time.Now()) { |
|
||||||
msg := "end time can't be in the past" |
|
||||||
am.logger.Error(msg, "ends_at", sil.EndsAt) |
|
||||||
return "", fmt.Errorf("%s: %w", msg, alerting.ErrCreateSilenceBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
silenceID, err := am.silences.Set(sil) |
|
||||||
if err != nil { |
|
||||||
am.logger.Error("msg", "unable to save silence", "error", err) |
|
||||||
if errors.Is(err, silence.ErrNotFound) { |
|
||||||
return "", alerting.ErrSilenceNotFound |
|
||||||
} |
|
||||||
return "", fmt.Errorf("unable to save silence: %s: %w", err.Error(), alerting.ErrCreateSilenceBadPayload) |
|
||||||
} |
|
||||||
|
|
||||||
return silenceID, nil |
|
||||||
} |
} |
||||||
|
|
||||||
// DeleteSilence looks for and expires the silence by the provided silenceID. It returns ErrSilenceNotFound if the silence is not present.
|
|
||||||
func (am *Alertmanager) DeleteSilence(silenceID string) error { |
func (am *Alertmanager) DeleteSilence(silenceID string) error { |
||||||
if err := am.silences.Expire(silenceID); err != nil { |
return am.Base.DeleteSilence(silenceID) |
||||||
if errors.Is(err, silence.ErrNotFound) { |
|
||||||
return alerting.ErrSilenceNotFound |
|
||||||
} |
|
||||||
return fmt.Errorf("%s: %w", err.Error(), alerting.ErrDeleteSilenceInternal) |
|
||||||
} |
|
||||||
|
|
||||||
return nil |
|
||||||
} |
} |
||||||
|
|||||||
@ -1,16 +1,22 @@ |
|||||||
package notifier |
package notifier |
||||||
|
|
||||||
import ( |
import ( |
||||||
|
"encoding/json" |
||||||
|
|
||||||
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions" |
||||||
) |
) |
||||||
|
|
||||||
|
// TODO: We no longer do apimodels at this layer, move it to the API.
|
||||||
func (am *Alertmanager) GetStatus() apimodels.GettableStatus { |
func (am *Alertmanager) GetStatus() apimodels.GettableStatus { |
||||||
am.reloadConfigMtx.RLock() |
config := &apimodels.PostableUserConfig{} |
||||||
defer am.reloadConfigMtx.RUnlock() |
status := am.Base.GetStatus() |
||||||
|
if status == nil { |
||||||
|
return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) |
||||||
|
} |
||||||
|
|
||||||
config := apimodels.PostableApiAlertingConfig{} |
if err := json.Unmarshal(status, config); err != nil { |
||||||
if am.ready() { |
am.logger.Error("unable to unmarshall alertmanager config", "Err", err) |
||||||
config = am.config.AlertmanagerConfig |
|
||||||
} |
} |
||||||
return *apimodels.NewGettableStatus(&config) |
|
||||||
|
return *apimodels.NewGettableStatus(&config.AlertmanagerConfig) |
||||||
} |
} |
||||||
|
|||||||
Loading…
Reference in new issue