Alerting: annotation on state change (#36535)

* WIP

* Add annotation on alert state change

* move annotation creation to manager

* praise the linter!

* add debug msg when creating annotation

pkg/services/ngalert/state/manager.go

@@ -1,8 +1,14 @@
 package state
 
 import (
+	"fmt"
+	"strconv"
 	"time"
 
+	"github.com/grafana/grafana/pkg/models"
+	"github.com/grafana/grafana/pkg/services/annotations"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+
 	"github.com/grafana/grafana/pkg/infra/log"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
@@ -151,11 +157,12 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 		EvaluationString: result.EvaluationString,
 	})
 	currentState.TrimResults(alertRule)
+	oldState := currentState.State
 
 	st.log.Debug("setting alert state", "uid", alertRule.UID)
 	switch result.State {
 	case eval.Normal:
-		currentState.resultNormal(result)
+		currentState.resultNormal(alertRule, result)
 	case eval.Alerting:
 		currentState.resultAlerting(alertRule, result)
 	case eval.Error:
@@ -166,6 +173,9 @@ func (st *Manager) setNextState(alertRule *ngModels.AlertRule, result eval.Resul
 	}
 
 	st.set(currentState)
+	if oldState != currentState.State {
+		go st.createAlertAnnotation(currentState.State, alertRule, result)
+	}
 	return currentState
 }
 
@@ -211,3 +221,46 @@ func translateInstanceState(state ngModels.InstanceStateType) eval.State {
 		return eval.Error
 	}
 }
+
+func (st *Manager) createAlertAnnotation(new eval.State, alertRule *ngModels.AlertRule, result eval.Result) {
+	st.log.Debug("alert state changed creating annotation", "alertRuleUID", alertRule.UID, "newState", new.String())
+	dashUid, ok := alertRule.Annotations["__dashboardUid__"]
+	if !ok {
+		return
+	}
+
+	panelUid := alertRule.Annotations["__panelId__"]
+
+	panelId, err := strconv.ParseInt(panelUid, 10, 64)
+	if err != nil {
+		st.log.Error("error parsing panelUID for alert annotation", "panelUID", panelUid, "alertRuleUID", alertRule.UID, "error", err.Error())
+		return
+	}
+
+	query := &models.GetDashboardQuery{
+		Uid:   dashUid,
+		OrgId: alertRule.OrgID,
+	}
+
+	err = sqlstore.GetDashboard(query)
+	if err != nil {
+		st.log.Error("error getting dashboard for alert annotation", "dashboardUID", dashUid, "alertRuleUID", alertRule.UID, "error", err.Error())
+		return
+	}
+
+	annotationText := fmt.Sprintf("%s %s", result.Instance.String(), new.String())
+
+	item := &annotations.Item{
+		OrgId:       alertRule.OrgID,
+		DashboardId: query.Result.Id,
+		PanelId:     panelId,
+		Text:        annotationText,
+		Epoch:       result.EvaluatedAt.UnixNano() / int64(time.Millisecond),
+	}
+
+	annotationRepo := annotations.GetRepository()
+	if err = annotationRepo.Save(item); err != nil {
+		st.log.Error("error saving alert annotation", "alertRuleUID", alertRule.UID, "error", err.Error())
+		return
+	}
+}

pkg/services/ngalert/state/state.go

@@ -30,7 +30,7 @@ type Evaluation struct {
 	EvaluationString string
 }
 
-func (a *State) resultNormal(result eval.Result) {
+func (a *State) resultNormal(alertRule *ngModels.AlertRule, result eval.Result) {
 	if a.State != eval.Normal {
 		a.EndsAt = result.EvaluatedAt
 		a.StartsAt = result.EvaluatedAt