Instrument/ruler api (#33290)

* ruler api histogram instrumentation

* register ruler metrics

go.sum

@@ -1028,6 +1028,7 @@ github.com/jung-kurt/gofpdf v1.0.3-0.20190309125859-24315acbbda5/go.mod h1:7Id9E
 github.com/jung-kurt/gofpdf v1.16.2 h1:jgbatWHfRlPYiK85qgevsZTHviWXKwB1TTiKdz5PtRc=
 github.com/jung-kurt/gofpdf v1.16.2/go.mod h1:1hl7y57EsiPAkLbOwzpzqgx1A30nQCk/YmFV8S2vmK0=
 github.com/jwilder/encoding v0.0.0-20170811194829-b4e1701a28ef/go.mod h1:Ct9fl0F6iIOGgxJ5npU/IUOhOhqlVrGjyIZc8/MagT0=
+github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88 h1:uC1QfSlInpQF+M0ao65imhwqKnz3Q2z/d8PWZRMQvDM=
 github.com/k0kubun/colorstring v0.0.0-20150214042306-9440f1994b88/go.mod h1:3w7q1U84EfirKl04SVQ/s7nPm1ZPhiXd34z40TNz36k=
 github.com/k0kubun/go-ansi v0.0.0-20180517002512-3bf9e2903213/go.mod h1:vNUNkEQ1e29fT/6vq2aBdFsgNPmy8qMdSay1npru+Sw=
 github.com/kardianos/osext v0.0.0-20190222173326-2bc1f35cddc0/go.mod h1:1NbS8ALrpOvjt0rHPNLyCIeMtbizbir8U//inJ+zuB8=

pkg/services/ngalert/api/api.go

@@ -4,6 +4,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
+	"github.com/prometheus/client_golang/prometheus"
 
 	"github.com/go-macaron/binding"
 
@@ -59,6 +60,14 @@ func (api *API) RegisterAPIEndpoints() {
 	proxy := &AlertingProxy{
 		DataProxy: api.DataProxy,
 	}
+
+	var reg prometheus.Registerer
+	// hack, this just assumes that if this histogram is enabled, we should enable others
+	// TODO(owen-d): expose this as a config option (alerting-instrumentation or similar)
+	if api.Cfg.IsHTTPRequestHistogramEnabled() {
+		reg = prometheus.DefaultRegisterer
+	}
+
 	// Register endpoints for proxing to Alertmanager-compatible backends.
 	api.RegisterAlertmanagerApiEndpoints(NewForkedAM(
 		api.DatasourceCache,
@@ -76,6 +85,7 @@ func (api *API) RegisterAPIEndpoints() {
 		api.DatasourceCache,
 		NewLotexRuler(proxy, logger),
 		RulerSrv{store: api.RuleStore, log: logger},
+		reg,
 	))
 	api.RegisterTestingApiEndpoints(TestingApiSrv{
 		AlertingProxy:   proxy,

pkg/services/ngalert/api/fork_ruler.go

@@ -2,26 +2,57 @@ package api
 
 import (
 	"fmt"
+	"time"
 
 	"github.com/grafana/grafana/pkg/api/response"
 	"github.com/grafana/grafana/pkg/models"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
+	"github.com/prometheus/client_golang/prometheus"
+)
+
+const (
+	GrafanaBackend = "grafana"
+	ProxyBackend   = "proxy" // metric cardinality is too high to enumerate all non-grafana backends
 )
 
 // ForkedRuler will validate and proxy requests to the correct backend type depending on the datasource.
 type ForkedRuler struct {
 	LotexRuler, GrafanaRuler RulerApiService
 	DatasourceCache          datasources.CacheService
+
+	// metrics
+	duration *prometheus.HistogramVec
 }
 
 // NewForkedRuler implements a set of routes that proxy to various Cortex Ruler-compatible backends.
-func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService) *ForkedRuler {
-	return &ForkedRuler{
+func NewForkedRuler(datasourceCache datasources.CacheService, lotex, grafana RulerApiService, reg prometheus.Registerer) *ForkedRuler {
+	r := &ForkedRuler{
 		LotexRuler:      lotex,
 		GrafanaRuler:    grafana,
 		DatasourceCache: datasourceCache,
+		duration: prometheus.NewHistogramVec(prometheus.HistogramOpts{
+			Namespace: "grafana",
+			Name:      "alerting_ruler_api_duration_seconds",
+			Help:      "Histogram of latencies affecting the Unified Alerting Ruler API",
+			Buckets:   prometheus.DefBuckets,
+		}, []string{"backend", "status"}),
 	}
+	if reg != nil {
+		reg.MustRegister(
+			r.duration,
+		)
+	}
+	return r
+}
+
+func (r *ForkedRuler) instrument(backend string, fn func() response.Response) response.Response {
+	start := time.Now()
+	resp := fn()
+	r.duration.
+		WithLabelValues(backend, fmt.Sprint(resp.Status())).
+		Observe(time.Since(start).Seconds())
+	return resp
 }
 
 func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) response.Response {
@@ -31,9 +62,9 @@ func (r *ForkedRuler) RouteDeleteNamespaceRulesConfig(ctx *models.ReqContext) re
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteNamespaceRulesConfig(ctx) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteNamespaceRulesConfig(ctx) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
 	}
@@ -46,9 +77,9 @@ func (r *ForkedRuler) RouteDeleteRuleGroupConfig(ctx *models.ReqContext) respons
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteDeleteRuleGroupConfig(ctx) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteDeleteRuleGroupConfig(ctx) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
 	}
@@ -61,9 +92,9 @@ func (r *ForkedRuler) RouteGetNamespaceRulesConfig(ctx *models.ReqContext) respo
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetNamespaceRulesConfig(ctx) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetNamespaceRulesConfig(ctx) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
 	}
@@ -76,9 +107,9 @@ func (r *ForkedRuler) RouteGetRulegGroupConfig(ctx *models.ReqContext) response.
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulegGroupConfig(ctx) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RouteGetRulegGroupConfig(ctx)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulegGroupConfig(ctx) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
 	}
@@ -91,9 +122,9 @@ func (r *ForkedRuler) RouteGetRulesConfig(ctx *models.ReqContext) response.Respo
 	}
 	switch t {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RouteGetRulesConfig(ctx)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RouteGetRulesConfig(ctx) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RouteGetRulesConfig(ctx)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RouteGetRulesConfig(ctx) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", t), nil)
 	}
@@ -120,9 +151,9 @@ func (r *ForkedRuler) RoutePostNameRulesConfig(ctx *models.ReqContext, conf apim
 
 	switch backendType {
 	case apimodels.GrafanaBackend:
-		return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf)
+		return r.instrument(GrafanaBackend, func() response.Response { return r.GrafanaRuler.RoutePostNameRulesConfig(ctx, conf) })
 	case apimodels.LoTexRulerBackend:
-		return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf)
+		return r.instrument(ProxyBackend, func() response.Response { return r.LotexRuler.RoutePostNameRulesConfig(ctx, conf) })
 	default:
 		return response.Error(400, fmt.Sprintf("unexpected backend type (%v)", backendType), nil)
 	}