diff --git a/pkg/api/http_server.go b/pkg/api/http_server.go index bc45ab50557..cb1c37cc3ad 100644 --- a/pkg/api/http_server.go +++ b/pkg/api/http_server.go @@ -213,6 +213,7 @@ type HTTPServer struct { authnService authn.Service starApi *starApi.API promRegister prometheus.Registerer + promGatherer prometheus.Gatherer clientConfigProvider grafanaapiserver.DirectRestConfigProvider namespacer request.NamespaceMapper anonService anonymous.Service @@ -256,7 +257,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi loginAttemptService loginAttempt.Service, orgService org.Service, teamService team.Service, accesscontrolService accesscontrol.Service, navTreeService navtree.Service, annotationRepo annotations.Repository, tagService tag.Service, searchv2HTTPService searchV2.SearchHTTPService, oauthTokenService oauthtoken.OAuthTokenService, - statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service, + statsService stats.Service, authnService authn.Service, pluginsCDNService *pluginscdn.Service, promGatherer prometheus.Gatherer, starApi *starApi.API, promRegister prometheus.Registerer, clientConfigProvider grafanaapiserver.DirectRestConfigProvider, anonService anonymous.Service, ) (*HTTPServer, error) { web.Env = cfg.Env @@ -356,6 +357,7 @@ func ProvideHTTPServer(opts ServerOptions, cfg *setting.Cfg, routeRegister routi pluginsCDNService: pluginsCDNService, starApi: starApi, promRegister: promRegister, + promGatherer: promGatherer, clientConfigProvider: clientConfigProvider, namespacer: request.GetNamespaceMapper(cfg), anonService: anonService, @@ -720,7 +722,7 @@ func (hs *HTTPServer) metricsEndpoint(ctx *web.Context) { } promhttp. - HandlerFor(prometheus.DefaultGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}). + HandlerFor(hs.promGatherer, promhttp.HandlerOpts{EnableOpenMetrics: true}). ServeHTTP(ctx.Resp, ctx.Req) } diff --git a/pkg/cmd/grafana-server/commands/buildinfo.go b/pkg/cmd/grafana-server/commands/buildinfo.go index 96bf7185cdd..d7097915f4e 100644 --- a/pkg/cmd/grafana-server/commands/buildinfo.go +++ b/pkg/cmd/grafana-server/commands/buildinfo.go @@ -5,22 +5,23 @@ import ( "time" "github.com/grafana/grafana/pkg/extensions" - "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/setting" ) -func setBuildInfo(opts ServerOptions) { +func getBuildstamp(opts ServerOptions) int64 { buildstampInt64, err := strconv.ParseInt(opts.BuildStamp, 10, 64) if err != nil || buildstampInt64 == 0 { buildstampInt64 = time.Now().Unix() } + return buildstampInt64 +} + +func setBuildInfo(opts ServerOptions) { setting.BuildVersion = opts.Version setting.BuildCommit = opts.Commit setting.EnterpriseBuildCommit = opts.EnterpriseCommit - setting.BuildStamp = buildstampInt64 + setting.BuildStamp = getBuildstamp(opts) setting.BuildBranch = opts.BuildBranch setting.IsEnterprise = extensions.IsEnterprise setting.Packaging = validPackaging(Packaging) - - metrics.SetBuildInformation(opts.Version, opts.Commit, opts.BuildBranch, buildstampInt64) } diff --git a/pkg/cmd/grafana-server/commands/cli.go b/pkg/cmd/grafana-server/commands/cli.go index 37ff71f7c76..ea329ea1b73 100644 --- a/pkg/cmd/grafana-server/commands/cli.go +++ b/pkg/cmd/grafana-server/commands/cli.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/api" gcli "github.com/grafana/grafana/pkg/cmd/grafana-cli/commands" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/infra/process" "github.com/grafana/grafana/pkg/server" _ "github.com/grafana/grafana/pkg/services/alerting/conditions" @@ -111,6 +112,8 @@ func RunServer(opts ServerOptions) error { return err } + metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) + s, err := server.Initialize( cfg, server.Options{ diff --git a/pkg/cmd/grafana-server/commands/target.go b/pkg/cmd/grafana-server/commands/target.go index 5843927a4c1..ae908ac9c18 100644 --- a/pkg/cmd/grafana-server/commands/target.go +++ b/pkg/cmd/grafana-server/commands/target.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/api" "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/server" "github.com/grafana/grafana/pkg/setting" ) @@ -88,6 +89,8 @@ func RunTargetServer(opts ServerOptions) error { return err } + metrics.SetBuildInformation(metrics.ProvideRegisterer(cfg), opts.Version, opts.Commit, opts.BuildBranch, getBuildstamp(opts)) + s, err := server.InitializeModuleServer( cfg, server.Options{ diff --git a/pkg/infra/metrics/frontendmetrics.go b/pkg/infra/metrics/frontendmetrics.go index 399b084e0a4..00e4bff7939 100644 --- a/pkg/infra/metrics/frontendmetrics.go +++ b/pkg/infra/metrics/frontendmetrics.go @@ -19,7 +19,7 @@ type FrontendMetricsRecorder func(event FrontendMetricEvent) // FrontendMetrics contains all the valid frontend metrics and a handler function for recording events var FrontendMetrics map[string]FrontendMetricsRecorder = map[string]FrontendMetricsRecorder{} -func registerFrontendHistogram(name string, help string) { +func registerFrontendHistogram(reg prometheus.Registerer, name string, help string) { defBuckets := []float64{.1, .25, .5, 1, 1.5, 2, 5, 10, 20, 40} histogram := prometheus.NewHistogram(prometheus.HistogramOpts{ @@ -33,14 +33,14 @@ func registerFrontendHistogram(name string, help string) { histogram.Observe(event.Value) } - prometheus.MustRegister(histogram) + reg.MustRegister(histogram) } -func initFrontendMetrics() { - registerFrontendHistogram("frontend_boot_load_time_seconds", "Frontend boot time measurement") - registerFrontendHistogram("frontend_boot_first_paint_time_seconds", "Frontend boot first paint") - registerFrontendHistogram("frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint") - registerFrontendHistogram("frontend_boot_js_done_time_seconds", "Frontend boot initial js load") - registerFrontendHistogram("frontend_boot_css_time_seconds", "Frontend boot initial css load") - registerFrontendHistogram("frontend_plugins_preload_ms", "Frontend preload plugin time measurement") +func initFrontendMetrics(r prometheus.Registerer) { + registerFrontendHistogram(r, "frontend_boot_load_time_seconds", "Frontend boot time measurement") + registerFrontendHistogram(r, "frontend_boot_first_paint_time_seconds", "Frontend boot first paint") + registerFrontendHistogram(r, "frontend_boot_first_contentful_paint_time_seconds", "Frontend boot first contentful paint") + registerFrontendHistogram(r, "frontend_boot_js_done_time_seconds", "Frontend boot initial js load") + registerFrontendHistogram(r, "frontend_boot_css_time_seconds", "Frontend boot initial css load") + registerFrontendHistogram(r, "frontend_plugins_preload_ms", "Frontend preload plugin time measurement") } diff --git a/pkg/infra/metrics/metrics.go b/pkg/infra/metrics/metrics.go index 6c0d1ad706d..4ed18149f8b 100644 --- a/pkg/infra/metrics/metrics.go +++ b/pkg/infra/metrics/metrics.go @@ -613,7 +613,7 @@ func init() { } // SetBuildInformation sets the build information for this binary -func SetBuildInformation(version, revision, branch string, buildTimestamp int64) { +func SetBuildInformation(reg prometheus.Registerer, version, revision, branch string, buildTimestamp int64) { edition := "oss" if setting.IsEnterprise { edition = "enterprise" @@ -631,7 +631,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64) Namespace: ExporterName, }, []string{"version", "revision", "branch", "goversion", "edition"}) - prometheus.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp) + reg.MustRegister(grafanaBuildVersion, grafanaBuildTimestamp) grafanaBuildVersion.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(1) grafanaBuildTimestamp.WithLabelValues(version, revision, branch, runtime.Version(), edition).Set(float64(buildTimestamp)) @@ -639,7 +639,7 @@ func SetBuildInformation(version, revision, branch string, buildTimestamp int64) // SetEnvironmentInformation exposes environment values provided by the operators as an `_info` metric. // If there are no environment metrics labels configured, this metric will not be exposed. -func SetEnvironmentInformation(labels map[string]string) error { +func SetEnvironmentInformation(reg prometheus.Registerer, labels map[string]string) error { if len(labels) == 0 { return nil } @@ -651,7 +651,7 @@ func SetEnvironmentInformation(labels map[string]string) error { ConstLabels: labels, }) - prometheus.MustRegister(grafanaEnvironmentInfo) + reg.MustRegister(grafanaEnvironmentInfo) grafanaEnvironmentInfo.Set(1) return nil @@ -661,8 +661,8 @@ func SetPluginBuildInformation(pluginID, pluginType, version, signatureStatus st grafanaPluginBuildInfoDesc.WithLabelValues(pluginID, pluginType, version, signatureStatus).Set(1) } -func initMetricVars() { - prometheus.MustRegister( +func initMetricVars(reg prometheus.Registerer) { + reg.MustRegister( MInstanceStart, MPageStatus, MApiStatus, diff --git a/pkg/infra/metrics/service.go b/pkg/infra/metrics/service.go index 2b24c2a665d..13767508a48 100644 --- a/pkg/infra/metrics/service.go +++ b/pkg/infra/metrics/service.go @@ -2,11 +2,17 @@ package metrics import ( "context" + "errors" + "regexp" + + "github.com/prometheus/client_golang/prometheus" + dto "github.com/prometheus/client_model/go" + "k8s.io/component-base/metrics/legacyregistry" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/infra/metrics/graphitebridge" + "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/setting" - "github.com/prometheus/client_golang/prometheus" ) var metricsLogger log.Logger = log.New("metrics") @@ -19,12 +25,10 @@ func (lw *logWrapper) Println(v ...any) { lw.logger.Info("graphite metric bridge", v...) } -func init() { - initMetricVars() - initFrontendMetrics() -} +func ProvideService(cfg *setting.Cfg, reg prometheus.Registerer) (*InternalMetricsService, error) { + initMetricVars(reg) + initFrontendMetrics(reg) -func ProvideService(cfg *setting.Cfg) (*InternalMetricsService, error) { s := &InternalMetricsService{ Cfg: cfg, } @@ -55,5 +59,66 @@ func (im *InternalMetricsService) Run(ctx context.Context) error { return ctx.Err() } -func ProvideRegisterer() prometheus.Registerer { return prometheus.DefaultRegisterer } -func ProvideRegistererForTest() prometheus.Registerer { return prometheus.NewRegistry() } +func ProvideRegisterer(cfg *setting.Cfg) prometheus.Registerer { + if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) { + return legacyregistry.Registerer() + } + return prometheus.DefaultRegisterer +} + +func ProvideGatherer(cfg *setting.Cfg) prometheus.Gatherer { + if cfg.IsFeatureToggleEnabled(featuremgmt.FlagGrafanaAPIServer) { + return newAddPrefixWrapper(legacyregistry.DefaultGatherer) + } + return prometheus.DefaultGatherer +} + +func ProvideRegistererForTest() prometheus.Registerer { + return prometheus.NewRegistry() +} + +func ProvideGathererForTest(reg prometheus.Registerer) prometheus.Gatherer { + // the registerer provided by ProvideRegistererForTest + // is a *prometheus.Registry, so it also implements prometheus.Gatherer + return reg.(*prometheus.Registry) +} + +var _ prometheus.Gatherer = (*addPrefixWrapper)(nil) + +// addPrefixWrapper wraps a prometheus.Gatherer, and ensures that all metric names are prefixed with `grafana_`. +// metrics with the prefix `grafana_` or `go_` are not modified. +type addPrefixWrapper struct { + orig prometheus.Gatherer + reg *regexp.Regexp +} + +func newAddPrefixWrapper(orig prometheus.Gatherer) *addPrefixWrapper { + return &addPrefixWrapper{ + orig: orig, + reg: regexp.MustCompile("^((?:grafana_|go_).*)"), + } +} + +func (g *addPrefixWrapper) Gather() ([]*dto.MetricFamily, error) { + mf, err := g.orig.Gather() + if err != nil { + return nil, err + } + + names := make(map[string]struct{}) + + for i := 0; i < len(mf); i++ { + m := mf[i] + if m.Name != nil && !g.reg.MatchString(*m.Name) { + *m.Name = "grafana_" + *m.Name + // since we are modifying the name, we need to check for duplicates in the gatherer + if _, exists := names[*m.Name]; exists { + return nil, errors.New("duplicate metric name: " + *m.Name) + } + } + // keep track of names to detect duplicates + names[*m.Name] = struct{}{} + } + + return mf, nil +} diff --git a/pkg/infra/metrics/service_test.go b/pkg/infra/metrics/service_test.go new file mode 100644 index 00000000000..b5c9ea92f50 --- /dev/null +++ b/pkg/infra/metrics/service_test.go @@ -0,0 +1,61 @@ +package metrics + +import ( + "testing" + + dto "github.com/prometheus/client_model/go" + "github.com/stretchr/testify/require" +) + +func TestK8sGathererWrapper_Gather(t *testing.T) { + orig := &mockGatherer{} + g := newAddPrefixWrapper(orig) + + t.Run("metrics with grafana and go prefix are not modified", func(t *testing.T) { + originalMF := []*dto.MetricFamily{ + {Name: strptr("grafana_metric1")}, + {Name: strptr("metric2")}, + {Name: strptr("go_metric1")}, + } + + orig.GatherFunc = func() ([]*dto.MetricFamily, error) { + return originalMF, nil + } + + expectedMF := []*dto.MetricFamily{ + {Name: strptr("grafana_metric1")}, + {Name: strptr("grafana_metric2")}, + {Name: strptr("go_metric1")}, + } + + mf, err := g.Gather() + require.NoError(t, err) + require.Equal(t, expectedMF, mf) + }) + + t.Run("duplicate metrics result in an error", func(t *testing.T) { + originalMF := []*dto.MetricFamily{ + {Name: strptr("grafana_metric1")}, + {Name: strptr("metric1")}, + } + + orig.GatherFunc = func() ([]*dto.MetricFamily, error) { + return originalMF, nil + } + + _, err := g.Gather() + require.Error(t, err) + }) +} + +type mockGatherer struct { + GatherFunc func() ([]*dto.MetricFamily, error) +} + +func (m *mockGatherer) Gather() ([]*dto.MetricFamily, error) { + return m.GatherFunc() +} + +func strptr(s string) *string { + return &s +} diff --git a/pkg/infra/metrics/wireset.go b/pkg/infra/metrics/wireset.go new file mode 100644 index 00000000000..489c995badc --- /dev/null +++ b/pkg/infra/metrics/wireset.go @@ -0,0 +1,17 @@ +package metrics + +import ( + "github.com/google/wire" +) + +var WireSet = wire.NewSet( + ProvideService, + ProvideRegisterer, + ProvideGatherer, +) + +var WireSetForTest = wire.NewSet( + ProvideService, + ProvideRegistererForTest, + ProvideGathererForTest, +) diff --git a/pkg/server/server.go b/pkg/server/server.go index a9de82ef902..ca1263d9e4d 100644 --- a/pkg/server/server.go +++ b/pkg/server/server.go @@ -13,6 +13,8 @@ import ( "golang.org/x/sync/errgroup" + "github.com/prometheus/client_golang/prometheus" + "github.com/grafana/grafana/pkg/api" _ "github.com/grafana/grafana/pkg/extensions" "github.com/grafana/grafana/pkg/infra/log" @@ -38,9 +40,10 @@ type Options struct { func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry, provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry, usageStatsProvidersRegistry registry.UsageStatsProvidersRegistry, statsCollectorService *statscollector.Service, + promReg prometheus.Registerer, ) (*Server, error) { statsCollectorService.RegisterProviders(usageStatsProvidersRegistry.GetServices()) - s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider) + s, err := newServer(opts, cfg, httpServer, roleRegistry, provisioningService, backgroundServiceProvider, promReg) if err != nil { return nil, err } @@ -54,11 +57,13 @@ func New(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistr func newServer(opts Options, cfg *setting.Cfg, httpServer *api.HTTPServer, roleRegistry accesscontrol.RoleRegistry, provisioningService provisioning.ProvisioningService, backgroundServiceProvider registry.BackgroundServiceRegistry, + promReg prometheus.Registerer, ) (*Server, error) { rootCtx, shutdownFn := context.WithCancel(context.Background()) childRoutines, childCtx := errgroup.WithContext(rootCtx) s := &Server{ + promReg: promReg, context: childCtx, childRoutines: childRoutines, HTTPServer: httpServer, @@ -101,6 +106,7 @@ type Server struct { HTTPServer *api.HTTPServer roleRegistry accesscontrol.RoleRegistry provisioningService provisioning.ProvisioningService + promReg prometheus.Registerer } // Init initializes the server and its services. @@ -117,7 +123,7 @@ func (s *Server) Init() error { return err } - if err := metrics.SetEnvironmentInformation(s.cfg.MetricsGrafanaEnvironmentInfo); err != nil { + if err := metrics.SetEnvironmentInformation(s.promReg, s.cfg.MetricsGrafanaEnvironmentInfo); err != nil { return err } diff --git a/pkg/server/server_test.go b/pkg/server/server_test.go index 1c45a5a82f2..5e1d57b2fa3 100644 --- a/pkg/server/server_test.go +++ b/pkg/server/server_test.go @@ -7,6 +7,7 @@ import ( "testing" "time" + "github.com/prometheus/client_golang/prometheus" "github.com/stretchr/testify/require" "github.com/grafana/grafana/pkg/registry" @@ -48,7 +49,7 @@ func (s *testService) IsDisabled() bool { func testServer(t *testing.T, services ...registry.BackgroundService) *Server { t.Helper() - s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...)) + s, err := newServer(Options{}, setting.NewCfg(), nil, &acimpl.Service{}, nil, backgroundsvcs.NewBackgroundServiceRegistry(services...), prometheus.NewRegistry()) require.NoError(t, err) // Required to skip configuration initialization that causes // DI errors in this test. diff --git a/pkg/server/wire.go b/pkg/server/wire.go index c02c5dc7db3..3e2bbf5ba00 100644 --- a/pkg/server/wire.go +++ b/pkg/server/wire.go @@ -254,7 +254,6 @@ var wireBasicSet = wire.NewSet( notifications.ProvideSmtpService, tracing.ProvideService, wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)), - metrics.ProvideService, testdatasource.ProvideService, ldapapi.ProvideService, opentsdb.ProvideService, @@ -389,7 +388,7 @@ var wireBasicSet = wire.NewSet( var wireSet = wire.NewSet( wireBasicSet, - metrics.ProvideRegisterer, + metrics.WireSet, sqlstore.ProvideService, ngmetrics.ProvideService, wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), @@ -404,6 +403,7 @@ var wireSet = wire.NewSet( var wireCLISet = wire.NewSet( NewRunner, wireBasicSet, + metrics.WireSet, sqlstore.ProvideService, ngmetrics.ProvideService, wire.Bind(new(notifications.Service), new(*notifications.NotificationService)), @@ -418,7 +418,7 @@ var wireCLISet = wire.NewSet( var wireTestSet = wire.NewSet( wireBasicSet, ProvideTestEnv, - metrics.ProvideRegistererForTest, + metrics.WireSetForTest, sqlstore.ProvideServiceForTests, ngmetrics.ProvideServiceForTest, notifications.MockNotificationService, diff --git a/pkg/server/wireexts_oss.go b/pkg/server/wireexts_oss.go index 366cc425646..5efd58cd50e 100644 --- a/pkg/server/wireexts_oss.go +++ b/pkg/server/wireexts_oss.go @@ -7,6 +7,7 @@ package server import ( "github.com/google/wire" + "github.com/grafana/grafana/pkg/infra/metrics" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/plugins/manager" "github.com/grafana/grafana/pkg/registry" @@ -123,6 +124,7 @@ var wireExtsTestSet = wire.NewSet( var wireExtsBaseCLISet = wire.NewSet( NewModuleRunner, + metrics.WireSet, featuremgmt.ProvideManagerService, featuremgmt.ProvideToggles, hooks.ProvideService,