apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Alerting: Instrument outbound requests for Loki Historian and Remote Alertmanager with tracing (#89185)

Browse Source 
* Add TracedClient

* Handle errors and status codes

* Wire up tracing to normal ASH and loki annotation mapping

* Add tracing to remote alertmanager

* one more spot

* and not or

* More consistency with other grafana traces, lower cardinality name
pull/89260/head
Alexander Weaver 1 year ago committed by GitHub
parent 6262c56132
commit 8491e02caf
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
16 changed files with 124 additions and 47 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 4
      pkg/services/annotations/annotationsimpl/annotations.go
  2. 4
      pkg/services/annotations/annotationsimpl/annotations_test.go
  3. 5
      pkg/services/annotations/annotationsimpl/loki/historian_store.go
  4. 55
      pkg/services/ngalert/client/client.go
  5. 20
      pkg/services/ngalert/ngalert.go
  6. 18
      pkg/services/ngalert/ngalert_test.go
  7. 3
      pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
  8. 7
      pkg/services/ngalert/remote/alertmanager.go
  9. 16
      pkg/services/ngalert/remote/alertmanager_test.go
  10. 8
      pkg/services/ngalert/remote/client/alertmanager.go
  11. 7
      pkg/services/ngalert/remote/client/mimir.go
  12. 5
      pkg/services/ngalert/state/historian/loki.go
  13. 6
      pkg/services/ngalert/state/historian/loki_http.go
  14. 7
      pkg/services/ngalert/state/historian/loki_http_test.go
  15. 3
      pkg/services/ngalert/state/historian/loki_test.go
  16. 3
      pkg/services/publicdashboards/service/common_test.go

4
pkg/services/annotations/annotationsimpl/annotations.go
Unescape View File

@ -8,6 +8,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/tag"
@ -27,6 +28,7 @@ func ProvideService(
cfg *setting.Cfg,
features featuremgmt.FeatureToggles,
tagService tag.Service,
tracer tracing.Tracer,
) *RepositoryImpl {
l := log.New("annotations")
l.Debug("Initializing annotations service")
@ -35,7 +37,7 @@ func ProvideService(
write := xormStore
var read readStore
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"))
historianStore := loki.NewLokiHistorianStore(cfg.UnifiedAlerting.StateHistory, features, db, log.New("annotations.loki"), tracer)
if historianStore != nil {
l.Debug("Using composite read store")
read = NewCompositeStore(log.New("annotations.composite"), xormStore, historianStore)

4
pkg/services/annotations/annotationsimpl/annotations_test.go
Unescape View File

@ -48,7 +48,7 @@ func TestIntegrationAnnotationListingWithRBAC(t *testing.T) {
features := featuremgmt.WithFeatures()
tagService := tagimpl.ProvideService(sql)
repo := ProvideService(sql, cfg, features, tagService)
repo := ProvideService(sql, cfg, features, tagService, tracing.InitializeTracerForTest())
dashboard1 := testutil.CreateDashboard(t, sql, cfg, features, dashboards.SaveDashboardCommand{
UserID: 1,
@ -317,7 +317,7 @@ func TestIntegrationAnnotationListingWithInheritedRBAC(t *testing.T) {
cfg := setting.NewCfg()
cfg.AnnotationMaximumTagsLength = 60
repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql))
repo := ProvideService(sql, cfg, tc.features, tagimpl.ProvideService(sql), tracing.InitializeTracerForTest())
usr.Permissions = map[int64]map[string][]string{1: tc.permissions}
testutil.SetupRBACPermission(t, sql, role, usr)

5
pkg/services/annotations/annotationsimpl/loki/historian_store.go
Unescape View File

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
ngmetrics "github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
"github.com/grafana/grafana/pkg/services/ngalert/state"
@ -53,7 +54,7 @@ type LokiHistorianStore struct {
log log.Logger
}
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger) *LokiHistorianStore {
func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft featuremgmt.FeatureToggles, db db.DB, log log.Logger, tracer tracing.Tracer) *LokiHistorianStore {
if !useStore(cfg, ft) {
return nil
}
@ -64,7 +65,7 @@ func NewLokiHistorianStore(cfg setting.UnifiedAlertingStateHistorySettings, ft f
}
return &LokiHistorianStore{
client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log),
client: historian.NewLokiClient(lokiCfg, historian.NewRequester(), ngmetrics.NewHistorianMetrics(prometheus.DefaultRegisterer, subsystem), log, tracer),
db: db,
log: log,
}

55
pkg/services/ngalert/client/client.go
Unescape View File

@ -7,6 +7,11 @@ import (
"strconv"
"github.com/grafana/dskit/instrument"
"github.com/grafana/grafana/pkg/infra/tracing"
"go.opentelemetry.io/otel/attribute"
"go.opentelemetry.io/otel/codes"
semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
"go.opentelemetry.io/otel/trace"
)
// Requester executes an HTTP request.
@ -14,7 +19,7 @@ type Requester interface {
Do(req *http.Request) (*http.Response, error)
}
// TimedClient instruments a request. It implements Requester.
// TimedClient instruments a request with metrics. It implements Requester.
type TimedClient struct {
client Requester
collector instrument.Collector
@ -70,3 +75,51 @@ func TimeRequest(ctx context.Context, operation string, coll instrument.Collecto
coll, toStatusCode, doRequest)
return response, err
}
// TracedClient instruments a request with tracing. It implements Requester.
type TracedClient struct {
client Requester
tracer tracing.Tracer
name string
}
func NewTracedClient(client Requester, tracer tracing.Tracer, name string) *TracedClient {
return &TracedClient{
client: client,
tracer: tracer,
name: name,
}
}
// Do executes the request.
func (c TracedClient) Do(r *http.Request) (*http.Response, error) {
ctx, span := c.tracer.Start(r.Context(), c.name, trace.WithSpanKind(trace.SpanKindClient))
defer span.End()
span.SetAttributes(semconv.HTTPURL(r.URL.String()))
span.SetAttributes(semconv.HTTPMethod(r.Method))
c.tracer.Inject(ctx, r.Header, span)
r = r.WithContext(ctx)
resp, err := c.client.Do(r)
if err != nil {
span.SetStatus(codes.Error, "request failed")
span.RecordError(err)
} else {
if resp.ContentLength > 0 {
span.SetAttributes(attribute.Int64("http.content_length", resp.ContentLength))
}
span.SetAttributes(semconv.HTTPStatusCode(resp.StatusCode))
if resp.StatusCode >= 400 && resp.StatusCode < 600 {
span.RecordError(fmt.Errorf("error with HTTP status code %d", resp.StatusCode))
}
}
return resp, err
}
// RoundTrip implements the RoundTripper interface.
func (c TracedClient) RoundTrip(r *http.Request) (*http.Response, error) {
return c.Do(r)
}

20
pkg/services/ngalert/ngalert.go
Unescape View File

@ -200,7 +200,7 @@ func (ng *AlertNG) init() error {
PromoteConfig: true,
SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager", "err", err)
return nil, err
@ -234,7 +234,7 @@ func (ng *AlertNG) init() error {
TenantID: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.TenantID,
URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
return internalAM, nil
@ -270,7 +270,7 @@ func (ng *AlertNG) init() error {
URL: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.URL,
SyncInterval: ng.Cfg.UnifiedAlerting.RemoteAlertmanager.SyncInterval,
}
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m)
remoteAM, err := createRemoteAlertmanager(cfg, ng.KVStore, ng.SecretsService.Decrypt, autogenFn, m, ng.tracer)
if err != nil {
moaLogger.Error("Failed to create remote Alertmanager, falling back to using only the internal one", "err", err)
return internalAM, nil
@ -359,7 +359,7 @@ func (ng *AlertNG) init() error {
// There are a set of feature toggles available that act as short-circuits for common configurations.
// If any are set, override the config accordingly.
ApplyStateHistoryFeatureToggles(&ng.Cfg.UnifiedAlerting.StateHistory, ng.FeatureToggles, ng.Log)
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log)
history, err := configureHistorianBackend(initCtx, ng.Cfg.UnifiedAlerting.StateHistory, ng.annotationsRepo, ng.dashboardService, ng.store, ng.Metrics.GetHistorianMetrics(), ng.Log, ng.tracer)
if err != nil {
return err
}
@ -523,7 +523,7 @@ type Historian interface {
state.Historian
}
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger) (Historian, error) {
func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingStateHistorySettings, ar annotations.Repository, ds dashboards.DashboardService, rs historian.RuleStore, met *metrics.Historian, l log.Logger, tracer tracing.Tracer) (Historian, error) {
if !cfg.Enabled {
met.Info.WithLabelValues("noop").Set(0)
return historian.NewNopHistorian(), nil
@ -538,7 +538,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
if backend == historian.BackendTypeMultiple {
primaryCfg := cfg
primaryCfg.Backend = cfg.MultiPrimary
primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l)
primary, err := configureHistorianBackend(ctx, primaryCfg, ar, ds, rs, met, l, tracer)
if err != nil {
return nil, fmt.Errorf("multi-backend target \"%s\" was misconfigured: %w", cfg.MultiPrimary, err)
}
@ -547,7 +547,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
for _, b := range cfg.MultiSecondaries {
secCfg := cfg
secCfg.Backend = b
sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l)
sec, err := configureHistorianBackend(ctx, secCfg, ar, ds, rs, met, l, tracer)
if err != nil {
return nil, fmt.Errorf("multi-backend target \"%s\" was miconfigured: %w", b, err)
}
@ -569,7 +569,7 @@ func configureHistorianBackend(ctx context.Context, cfg setting.UnifiedAlertingS
}
req := historian.NewRequester()
lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met)
backend := historian.NewRemoteLokiBackend(lokiBackendLogger, lcfg, req, met, tracer)
testConnCtx, cancelFunc := context.WithTimeout(ctx, 10*time.Second)
defer cancelFunc()
@ -627,8 +627,8 @@ func ApplyStateHistoryFeatureToggles(cfg *setting.UnifiedAlertingStateHistorySet
}
}
func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager) (*remote.Alertmanager, error) {
return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m)
func createRemoteAlertmanager(cfg remote.AlertmanagerConfig, kvstore kvstore.KVStore, decryptFn remote.DecryptFn, autogenFn remote.AutogenFn, m *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*remote.Alertmanager, error) {
return remote.NewAlertmanager(cfg, notifier.NewFileStore(cfg.OrgID, kvstore), decryptFn, autogenFn, m, tracer)
}
func createRecordingWriter(featureToggles featuremgmt.FeatureToggles, settings setting.RecordingRuleSettings) (schedule.RecordingWriter, error) {

18
pkg/services/ngalert/ngalert_test.go
Unescape View File

@ -62,12 +62,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid backend", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "invalid-backend",
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "unrecognized")
})
@ -75,13 +76,14 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid multi-backend primary", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "multiple",
MultiPrimary: "invalid-backend",
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "multi-backend target")
require.ErrorContains(t, err, "unrecognized")
@ -90,6 +92,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("fail initialization if invalid multi-backend secondary", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "multiple",
@ -97,7 +100,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
MultiSecondaries: []string{"annotations", "invalid-backend"},
}
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
_, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.ErrorContains(t, err, "multi-backend target")
require.ErrorContains(t, err, "unrecognized")
@ -106,6 +109,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
t.Run("do not fail initialization if pinging Loki fails", func(t *testing.T) {
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "loki",
@ -114,7 +118,7 @@ func TestConfigureHistorianBackend(t *testing.T) {
LokiWriteURL: "http://gone.invalid",
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)
@ -124,12 +128,13 @@ func TestConfigureHistorianBackend(t *testing.T) {
reg := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: true,
Backend: "annotations",
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)
@ -146,11 +151,12 @@ grafana_alerting_state_history_info{backend="annotations"} 1
reg := prometheus.NewRegistry()
met := metrics.NewHistorianMetrics(reg, metrics.Subsystem)
logger := log.NewNopLogger()
tracer := tracing.InitializeTracerForTest()
cfg := setting.UnifiedAlertingStateHistorySettings{
Enabled: false,
}
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger)
h, err := configureHistorianBackend(context.Background(), cfg, nil, nil, nil, met, logger, tracer)
require.NotNil(t, h)
require.NoError(t, err)

3
pkg/services/ngalert/notifier/multiorg_alertmanager_remote_test.go
Unescape View File

@ -12,6 +12,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@ -66,7 +67,7 @@ func TestMultiorgAlertmanager_RemoteSecondaryMode(t *testing.T) {
DefaultConfig: setting.GetAlertmanagerDefaultConfiguration(),
}
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m)
remoteAM, err := remote.NewAlertmanager(externalAMCfg, notifier.NewFileStore(orgID, kvStore), secretsService.Decrypt, remote.NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// Use both Alertmanager implementations in the forked Alertmanager.

7
pkg/services/ngalert/remote/alertmanager.go
Unescape View File

@ -25,6 +25,7 @@ import (
"gopkg.in/yaml.v3"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/services/ngalert/models"
@ -100,7 +101,7 @@ func (cfg *AlertmanagerConfig) Validate() error {
return nil
}
func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn DecryptFn, autogenFn AutogenFn, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
if err := cfg.Validate(); err != nil {
return nil, err
}
@ -118,7 +119,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
URL: u,
PromoteConfig: cfg.PromoteConfig,
}
mc, err := remoteClient.New(mcCfg, metrics)
mc, err := remoteClient.New(mcCfg, metrics, tracer)
if err != nil {
return nil, err
}
@ -129,7 +130,7 @@ func NewAlertmanager(cfg AlertmanagerConfig, store stateStore, decryptFn Decrypt
Password: cfg.BasicAuthPassword,
Logger: logger,
}
amc, err := remoteClient.NewAlertmanager(amcCfg, metrics)
amc, err := remoteClient.NewAlertmanager(amcCfg, metrics, tracer)
if err != nil {
return nil, err
}

16
pkg/services/ngalert/remote/alertmanager_test.go
Unescape View File

@ -23,6 +23,7 @@ import (
"github.com/grafana/alerting/definition"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
ngmodels "github.com/grafana/grafana/pkg/services/ngalert/models"
@ -105,7 +106,7 @@ func TestNewAlertmanager(t *testing.T) {
DefaultConfig: defaultGrafanaConfig,
}
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
if test.expErr != "" {
require.EqualError(tt, err, test.expErr)
return
@ -177,7 +178,7 @@ func TestApplyConfig(t *testing.T) {
// An error response from the remote Alertmanager should result in the readiness check failing.
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
config := &ngmodels.AlertConfiguration{
@ -305,6 +306,7 @@ func TestCompareAndSendConfiguration(t *testing.T) {
decryptFn,
test.autogenFn,
m,
tracing.InitializeTracerForTest(),
)
require.NoError(t, err)
@ -364,7 +366,7 @@ func TestIntegrationRemoteAlertmanagerConfiguration(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, database.ProvideSecretsStore(db.InitTestDB(t)))
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, fstore, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
encodedFullState, err := am.getFullState(ctx)
@ -521,7 +523,7 @@ func TestIntegrationRemoteAlertmanagerGetStatus(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should get the default Cloud Alertmanager configuration.
@ -555,7 +557,7 @@ func TestIntegrationRemoteAlertmanagerSilences(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should have no silences at first.
@ -640,7 +642,7 @@ func TestIntegrationRemoteAlertmanagerAlerts(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// Wait until the Alertmanager is ready to send alerts.
@ -709,7 +711,7 @@ func TestIntegrationRemoteAlertmanagerReceivers(t *testing.T) {
secretsService := secretsManager.SetupTestService(t, fakes.NewFakeSecretsStore())
m := metrics.NewRemoteAlertmanagerMetrics(prometheus.NewRegistry())
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m)
am, err := NewAlertmanager(cfg, nil, secretsService.Decrypt, NoopAutogenFn, m, tracing.InitializeTracerForTest())
require.NoError(t, err)
// We should start with the default config.

8
pkg/services/ngalert/remote/client/alertmanager.go
Unescape View File

@ -9,6 +9,7 @@ import (
httptransport "github.com/go-openapi/runtime/client"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
amclient "github.com/prometheus/alertmanager/api/v2/client"
@ -31,7 +32,7 @@ type Alertmanager struct {
logger log.Logger
}
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager) (*Alertmanager, error) {
func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Alertmanager, error) {
// First, add the authentication middleware.
c := &http.Client{Transport: &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
@ -40,14 +41,15 @@ func NewAlertmanager(cfg *AlertmanagerConfig, metrics *metrics.RemoteAlertmanage
}}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
apiEndpoint := *cfg.URL
// Next, make sure you set the right path.
u := apiEndpoint.JoinPath(alertmanagerAPIMountPath, amclient.DefaultBasePath)
// Create an Alertmanager client using the timed client as the transport.
// Create an Alertmanager client using the instrumented client as the transport.
r := httptransport.New(u.Host, u.Path, []string{u.Scheme})
r.Transport = tc
r.Transport = trc
return &Alertmanager{
logger: cfg.Logger,

7
pkg/services/ngalert/remote/client/mimir.go
Unescape View File

@ -12,6 +12,7 @@ import (
"strings"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
apimodels "github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -68,7 +69,7 @@ func (e *errorResponse) Error() string {
return e.Error2
}
func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
func New(cfg *Config, metrics *metrics.RemoteAlertmanager, tracer tracing.Tracer) (*Mimir, error) {
rt := &MimirAuthRoundTripper{
TenantID: cfg.TenantID,
Password: cfg.Password,
@ -78,10 +79,12 @@ func New(cfg *Config, metrics *metrics.RemoteAlertmanager) (*Mimir, error) {
c := &http.Client{
Transport: rt,
}
tc := client.NewTimedClient(c, metrics.RequestLatency)
trc := client.NewTracedClient(tc, tracer, "remote.alertmanager.client")
return &Mimir{
endpoint: cfg.URL,
client: client.NewTimedClient(c, metrics.RequestLatency),
client: trc,
logger: cfg.Logger,
metrics: metrics,
promoteConfig: cfg.PromoteConfig,

5
pkg/services/ngalert/state/historian/loki.go
Unescape View File

@ -14,6 +14,7 @@ import (
"github.com/grafana/grafana/pkg/components/simplejson"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -55,9 +56,9 @@ type RemoteLokiBackend struct {
log log.Logger
}
func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian) *RemoteLokiBackend {
func NewRemoteLokiBackend(logger log.Logger, cfg LokiConfig, req client.Requester, metrics *metrics.Historian, tracer tracing.Tracer) *RemoteLokiBackend {
return &RemoteLokiBackend{
client: NewLokiClient(cfg, req, metrics, logger),
client: NewLokiClient(cfg, req, metrics, logger, tracer),
externalLabels: cfg.ExternalLabels,
clock: clock.New(),
metrics: metrics,

6
pkg/services/ngalert/state/historian/loki_http.go
Unescape View File

@ -12,6 +12,7 @@ import (
"time"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
"github.com/grafana/grafana/pkg/setting"
@ -103,10 +104,11 @@ const (
NeqRegEx Operator = "!~"
)
func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger) *HttpLokiClient {
func NewLokiClient(cfg LokiConfig, req client.Requester, metrics *metrics.Historian, logger log.Logger, tracer tracing.Tracer) *HttpLokiClient {
tc := client.NewTimedClient(req, metrics.WriteDuration)
trc := client.NewTracedClient(tc, tracer, "ngalert.historian.client")
return &HttpLokiClient{
client: tc,
client: trc,
encoder: cfg.Encoder,
cfg: cfg,
metrics: metrics,

7
pkg/services/ngalert/state/historian/loki_http_test.go
Unescape View File

@ -18,6 +18,7 @@ import (
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
)
func TestLokiConfig(t *testing.T) {
@ -227,7 +228,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
ReadPathURL: url,
WritePathURL: url,
Encoder: JsonEncoder{},
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
// Unauthorized request should fail against Grafana Cloud.
err = client.Ping(context.Background())
@ -255,7 +256,7 @@ func TestLokiHTTPClient_Manual(t *testing.T) {
BasicAuthUser: "<your_username>",
BasicAuthPassword: "<your_password>",
Encoder: JsonEncoder{},
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger())
}, NewRequester(), metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem), log.NewNopLogger(), tracing.InitializeTracerForTest())
// When running on prem, you might need to set the tenant id,
// so the x-scope-orgid header is set.
@ -389,7 +390,7 @@ func createTestLokiClient(req client.Requester) *HttpLokiClient {
Encoder: JsonEncoder{},
}
met := metrics.NewHistorianMetrics(prometheus.NewRegistry(), metrics.Subsystem)
return NewLokiClient(cfg, req, met, log.NewNopLogger())
return NewLokiClient(cfg, req, met, log.NewNopLogger(), tracing.InitializeTracerForTest())
}
func reqBody(t *testing.T, req *http.Request) string {

3
pkg/services/ngalert/state/historian/loki_test.go
Unescape View File

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/ngalert/client"
"github.com/grafana/grafana/pkg/services/ngalert/eval"
"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@ -514,7 +515,7 @@ func createTestLokiBackend(req client.Requester, met *metrics.Historian) *Remote
ExternalLabels: map[string]string{"externalLabelKey": "externalLabelValue"},
}
lokiBackendLogger := log.New("ngalert.state.historian", "backend", "loki")
return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met)
return NewRemoteLokiBackend(lokiBackendLogger, cfg, req, met, tracing.InitializeTracerForTest())
}
func singleFromNormal(st *state.State) []state.StateTransition {

3
pkg/services/publicdashboards/service/common_test.go
Unescape View File

@ -5,6 +5,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/services/annotations/annotationsimpl"
"github.com/grafana/grafana/pkg/services/dashboards"
@ -29,7 +30,7 @@ func newPublicDashboardServiceImpl(
db, cfg := db.InitTestDBWithCfg(t)
tagService := tagimpl.ProvideService(db)
if annotationsRepo == nil {
annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService)
annotationsRepo = annotationsimpl.ProvideService(db, cfg, featuremgmt.WithFeatures(), tagService, tracing.InitializeTracerForTest())
}
if publicDashboardStore == nil {

Write Preview
Loading…
Cancel
Save