Init dualwriter metrics (#89003)

* Pass prometheus registerer to the dual writer

* Fix tests

* Remove unused var

* Fix tests

* Uncomment test

* Remove leading line

* Fix tests. Reuse registerer if there's already one

* Lint

* Improve double registering logic

* Rebase main

pkg/apiserver/builder/common.go

@@ -4,6 +4,7 @@ import (
 	"net/http"
 
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
+	"github.com/prometheus/client_golang/prometheus"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
 	"k8s.io/apimachinery/pkg/runtime/serializer"
@@ -29,6 +30,7 @@ type APIGroupBuilder interface {
 		codecs serializer.CodecFactory,
 		optsGetter generic.RESTOptionsGetter,
 		desiredMode grafanarest.DualWriterMode,
+		reg prometheus.Registerer,
 	) (*genericapiserver.APIGroupInfo, error)
 
 	// Get OpenAPI definitions

pkg/apiserver/builder/helper.go

@@ -25,6 +25,7 @@ import (
 
 	"github.com/grafana/grafana/pkg/apiserver/endpoints/filters"
 	"github.com/grafana/grafana/pkg/services/apiserver/options"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 // TODO: this is a temporary hack to make rest.Connecter work with resource level routes
@@ -128,6 +129,7 @@ func InstallAPIs(
 	optsGetter generic.RESTOptionsGetter,
 	builders []APIGroupBuilder,
 	storageOpts *options.StorageOptions,
+	reg prometheus.Registerer,
 ) error {
 	// dual writing is only enabled when the storage type is not legacy.
 	// this is needed to support setting a default RESTOptionsGetter for new APIs that don't
@@ -136,7 +138,7 @@ func InstallAPIs(
 
 	for _, b := range builders {
 		mode := b.GetDesiredDualWriterMode(dualWriteEnabled, storageOpts.DualWriterDesiredModes)
-		g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode)
+		g, err := b.GetAPIGroupInfo(scheme, codecs, optsGetter, mode, reg)
 		if err != nil {
 			return err
 		}

pkg/apiserver/rest/dualwriter.go

@@ -6,6 +6,7 @@ import (
 	"fmt"
 
 	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apiserver/pkg/registry/rest"
@@ -95,25 +96,28 @@ const (
 	Mode4
 )
 
+// TODO: make this function private as there should only be one public way of setting the dual writing mode
 // NewDualWriter returns a new DualWriter.
-func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage) DualWriter {
+func NewDualWriter(mode DualWriterMode, legacy LegacyStorage, storage Storage, reg prometheus.Registerer) DualWriter {
+	metrics := &dualWriterMetrics{}
+	metrics.init(reg)
 	switch mode {
 	// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
 	// writing to legacy storage without `unifiedStorage` enabled.
 	case Mode1:
 		// read and write only from legacy storage
-		return newDualWriterMode1(legacy, storage)
+		return newDualWriterMode1(legacy, storage, metrics)
 	case Mode2:
 		// write to both, read from storage but use legacy as backup
-		return newDualWriterMode2(legacy, storage)
+		return newDualWriterMode2(legacy, storage, metrics)
 	case Mode3:
 		// write to both, read from storage only
-		return newDualWriterMode3(legacy, storage)
+		return newDualWriterMode3(legacy, storage, metrics)
 	case Mode4:
 		// read and write only from storage
-		return newDualWriterMode4(legacy, storage)
+		return newDualWriterMode4(legacy, storage, metrics)
 	default:
-		return newDualWriterMode1(legacy, storage)
+		return newDualWriterMode1(legacy, storage, metrics)
 	}
 }
 
@@ -142,6 +146,7 @@ func SetDualWritingMode(
 	storage Storage,
 	entity string,
 	desiredMode DualWriterMode,
+	reg prometheus.Registerer,
 ) (DualWriter, error) {
 	toMode := map[string]DualWriterMode{
 		// It is not possible to initialize a mode 0 dual writer. Mode 0 represents
@@ -200,5 +205,5 @@ func SetDualWritingMode(
 
 	// 	#TODO add support for other combinations of desired and current modes
 
-	return NewDualWriter(currentMode, legacy, storage), nil
+	return NewDualWriter(currentMode, legacy, storage, reg), nil
 }

pkg/apiserver/rest/dualwriter_mode1.go

@@ -24,10 +24,8 @@ const mode1Str = "1"
 
 // NewDualWriterMode1 returns a new DualWriter in mode 1.
 // Mode 1 represents writing to and reading from LegacyStorage.
-func newDualWriterMode1(legacy LegacyStorage, storage Storage) *DualWriterMode1 {
-	metrics := &dualWriterMetrics{}
-	metrics.init()
-	return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: metrics}
+func newDualWriterMode1(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode1 {
+	return &DualWriterMode1{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode1"), dualWriterMetrics: dwm}
 }
 
 // Mode returns the mode of the dual writer.

pkg/apiserver/rest/dualwriter_mode1_test.go

@@ -5,6 +5,7 @@ import (
 	"errors"
 	"testing"
 
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 	"k8s.io/apimachinery/pkg/api/meta"
@@ -22,6 +23,8 @@ var failingObj = &example.Pod{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ObjectMeta
 var exampleList = &example.PodList{TypeMeta: metav1.TypeMeta{Kind: "foo"}, ListMeta: metav1.ListMeta{}, Items: []example.Pod{*exampleObj}}
 var anotherList = &example.PodList{Items: []example.Pod{*anotherObj}}
 
+var p = prometheus.NewRegistry()
+
 func TestMode1_Create(t *testing.T) {
 	type testCase struct {
 		input          runtime.Object
@@ -68,7 +71,7 @@ func TestMode1_Create(t *testing.T) {
 				tt.setupStorageFn(m)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			obj, err := dw.Create(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.CreateOptions{})
 
@@ -131,7 +134,8 @@ func TestMode1_Get(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			p := prometheus.NewRegistry()
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
 
@@ -182,7 +186,7 @@ func TestMode1_List(t *testing.T) {
 				tt.setupStorageFn(m)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			_, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
 
@@ -237,7 +241,7 @@ func TestMode1_Delete(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			obj, _, err := dw.Delete(context.Background(), tt.input, func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{})
 
@@ -296,7 +300,7 @@ func TestMode1_DeleteCollection(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, tt.input, &metainternalversion.ListOptions{})
 
@@ -372,7 +376,7 @@ func TestMode1_Update(t *testing.T) {
 				tt.setupGetFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode1, ls, us)
+			dw := NewDualWriter(Mode1, ls, us, p)
 
 			obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
 

pkg/apiserver/rest/dualwriter_mode2.go

@@ -4,6 +4,7 @@ import (
 	"context"
 	"time"
 
+	"github.com/grafana/grafana/pkg/apimachinery/utils"
 	apierrors "k8s.io/apimachinery/pkg/api/errors"
 	"k8s.io/apimachinery/pkg/api/meta"
 	metainternalversion "k8s.io/apimachinery/pkg/apis/meta/internalversion"
@@ -13,8 +14,6 @@ import (
 	"k8s.io/apimachinery/pkg/selection"
 	"k8s.io/apiserver/pkg/registry/rest"
 	"k8s.io/klog/v2"
-
-	"github.com/grafana/grafana/pkg/apimachinery/utils"
 )
 
 type DualWriterMode2 struct {
@@ -28,10 +27,8 @@ const mode2Str = "2"
 
 // NewDualWriterMode2 returns a new DualWriter in mode 2.
 // Mode 2 represents writing to LegacyStorage and Storage and reading from LegacyStorage.
-func newDualWriterMode2(legacy LegacyStorage, storage Storage) *DualWriterMode2 {
-	metrics := &dualWriterMetrics{}
-	metrics.init()
-	return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: metrics}
+func newDualWriterMode2(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode2 {
+	return &DualWriterMode2{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode2"), dualWriterMetrics: dwm}
 }
 
 // Mode returns the mode of the dual writer.

pkg/apiserver/rest/dualwriter_mode2_test.go

@@ -67,7 +67,7 @@ func TestMode2_Create(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, err := dw.Create(context.Background(), tt.input, createFn, &metav1.CreateOptions{})
 
@@ -143,7 +143,7 @@ func TestMode2_Get(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, err := dw.Get(context.Background(), tt.input, &metav1.GetOptions{})
 
@@ -196,7 +196,7 @@ func TestMode2_List(t *testing.T) {
 				tt.setupStorageFn(m)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, err := dw.List(context.Background(), &metainternalversion.ListOptions{})
 
@@ -289,7 +289,7 @@ func TestMode2_Delete(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, _, err := dw.Delete(context.Background(), tt.input, func(context.Context, runtime.Object) error { return nil }, &metav1.DeleteOptions{})
 
@@ -361,7 +361,7 @@ func TestMode2_DeleteCollection(t *testing.T) {
 				tt.setupStorageFn(m)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, err := dw.DeleteCollection(context.Background(), func(ctx context.Context, obj runtime.Object) error { return nil }, &metav1.DeleteOptions{TypeMeta: metav1.TypeMeta{Kind: tt.input}}, &metainternalversion.ListOptions{})
 
@@ -469,7 +469,7 @@ func TestMode2_Update(t *testing.T) {
 				tt.setupStorageFn(m, tt.input)
 			}
 
-			dw := NewDualWriter(Mode2, ls, us)
+			dw := NewDualWriter(Mode2, ls, us, p)
 
 			obj, _, err := dw.Update(context.Background(), tt.input, updatedObjInfoObj{}, func(ctx context.Context, obj runtime.Object) error { return nil }, func(ctx context.Context, obj, old runtime.Object) error { return nil }, false, &metav1.UpdateOptions{})
 

pkg/apiserver/rest/dualwriter_mode3.go

@@ -20,10 +20,8 @@ type DualWriterMode3 struct {
 
 // newDualWriterMode3 returns a new DualWriter in mode 3.
 // Mode 3 represents writing to LegacyStorage and Storage and reading from Storage.
-func newDualWriterMode3(legacy LegacyStorage, storage Storage) *DualWriterMode3 {
-	metrics := &dualWriterMetrics{}
-	metrics.init()
-	return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: metrics}
+func newDualWriterMode3(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode3 {
+	return &DualWriterMode3{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode3"), dualWriterMetrics: dwm}
 }
 
 // Mode returns the mode of the dual writer.

pkg/apiserver/rest/dualwriter_mode4.go

@@ -19,10 +19,8 @@ type DualWriterMode4 struct {
 
 // newDualWriterMode4 returns a new DualWriter in mode 4.
 // Mode 4 represents writing and reading from Storage.
-func newDualWriterMode4(legacy LegacyStorage, storage Storage) *DualWriterMode4 {
-	metrics := &dualWriterMetrics{}
-	metrics.init()
-	return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: metrics}
+func newDualWriterMode4(legacy LegacyStorage, storage Storage, dwm *dualWriterMetrics) *DualWriterMode4 {
+	return &DualWriterMode4{Legacy: legacy, Storage: storage, Log: klog.NewKlogr().WithName("DualWriterMode4"), dualWriterMetrics: dwm}
 }
 
 // Mode returns the mode of the dual writer.

pkg/apiserver/rest/dualwriter_test.go

@@ -7,6 +7,7 @@ import (
 
 	playlist "github.com/grafana/grafana/pkg/apis/playlist/v0alpha1"
 	"github.com/grafana/grafana/pkg/infra/kvstore"
+	"github.com/prometheus/client_golang/prometheus"
 	"github.com/stretchr/testify/assert"
 	"github.com/stretchr/testify/mock"
 )
@@ -45,7 +46,8 @@ func TestSetDualWritingMode(t *testing.T) {
 
 		kvStore := kvstore.WithNamespace(kvstore.NewFakeKVStore(), 0, "storage.dualwriting."+tt.stackID)
 
-		dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode)
+		p := prometheus.NewRegistry()
+		dw, err := SetDualWritingMode(context.Background(), kvStore, ls, us, playlist.GROUPRESOURCE, tt.desiredMode, p)
 		assert.NoError(t, err)
 		assert.Equal(t, tt.expectedMode, dw.Mode())
 

pkg/apiserver/rest/metrics.go

@@ -5,6 +5,7 @@ import (
 	"time"
 
 	"github.com/prometheus/client_golang/prometheus"
+	"k8s.io/klog/v2"
 )
 
 type dualWriterMetrics struct {
@@ -37,10 +38,17 @@ var DualWriterOutcome = prometheus.NewHistogramVec(prometheus.HistogramOpts{
 	NativeHistogramBucketFactor: 1.1,
 }, []string{"mode", "name", "method"})
 
-func (m *dualWriterMetrics) init() {
+func (m *dualWriterMetrics) init(reg prometheus.Registerer) {
+	log := klog.NewKlogr()
 	m.legacy = DualWriterLegacyDuration
 	m.storage = DualWriterStorageDuration
 	m.outcome = DualWriterOutcome
+	errLegacy := reg.Register(m.legacy)
+	errStorage := reg.Register(m.storage)
+	errOutcome := reg.Register(m.outcome)
+	if errLegacy != nil || errStorage != nil || errOutcome != nil {
+		log.Info("cloud migration metrics already registered")
+	}
 }
 
 func (m *dualWriterMetrics) recordLegacyDuration(isError bool, mode string, name string, method string, startFrom time.Time) {

pkg/cmd/grafana/apiserver/server.go

@@ -165,7 +165,7 @@ func (o *APIServerOptions) RunAPIServer(config *genericapiserver.RecommendedConf
 
 	// Install the API Group+version
 	// #TODO figure out how to configure storage type in o.Options.StorageOptions
-	err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions)
+	err = builder.InstallAPIs(grafanaAPIServer.Scheme, grafanaAPIServer.Codecs, server, config.RESTOptionsGetter, o.builders, o.Options.StorageOptions, o.Options.MetricsOptions.MetricsRegisterer)
 	if err != nil {
 		return err
 	}

pkg/registry/apis/dashboard/register.go

@@ -25,6 +25,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/provisioning"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*DashboardsAPIBuilder)(nil)
@@ -49,6 +50,7 @@ func RegisterAPIService(cfg *setting.Cfg, features featuremgmt.FeatureToggles,
 	accessControl accesscontrol.AccessControl,
 	provisioning provisioning.ProvisioningService,
 	dashStore dashboards.Store,
+	reg prometheus.Registerer,
 	sql db.DB,
 ) *DashboardsAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
@@ -115,6 +117,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	optsGetter generic.RESTOptionsGetter,
 	desiredMode grafanarest.DualWriterMode,
+	reg prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs)
 
@@ -145,7 +148,7 @@ func (b *DashboardsAPIBuilder) GetAPIGroupInfo(
 		if err := store.CompleteWithOptions(options); err != nil {
 			return nil, err
 		}
-		storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store)
+		storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store, reg)
 	}
 
 	// Summary

pkg/registry/apis/dashboardsnapshot/register.go

@@ -8,6 +8,7 @@ import (
 	"time"
 
 	"github.com/gorilla/mux"
+	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -71,6 +72,7 @@ func RegisterAPIService(
 	cfg *setting.Cfg,
 	features featuremgmt.FeatureToggles,
 	sql db.DB,
+	reg prometheus.Registerer,
 ) *SnapshotsAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
 		return nil // skip registration unless opting into experimental apis
@@ -129,6 +131,7 @@ func (b *SnapshotsAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	optsGetter generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(dashboardsnapshot.GROUP, scheme, metav1.ParameterCodec, codecs)
 	storage := map[string]rest.Storage{}

pkg/registry/apis/datasource/register.go

@@ -7,6 +7,7 @@ import (
 	"time"
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
+	"github.com/prometheus/client_golang/prometheus"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/apis/meta/v1/unstructured"
 	"k8s.io/apimachinery/pkg/runtime"
@@ -56,6 +57,7 @@ func RegisterAPIService(
 	contextProvider PluginContextWrapper,
 	pluginStore pluginstore.Store,
 	accessControl accesscontrol.AccessControl,
+	reg prometheus.Registerer,
 ) (*DataSourceAPIBuilder, error) {
 	// This requires devmode!
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
@@ -205,6 +207,7 @@ func (b *DataSourceAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	_ generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	storage := map[string]rest.Storage{}
 

pkg/registry/apis/example/register.go

@@ -25,6 +25,7 @@ import (
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/infra/appcontext"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*TestingAPIBuilder)(nil)
@@ -41,7 +42,7 @@ func NewTestingAPIBuilder() *TestingAPIBuilder {
 	}
 }
 
-func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *TestingAPIBuilder {
+func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *TestingAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
 		return nil // skip registration unless opting into experimental apis
 	}
@@ -92,6 +93,7 @@ func (b *TestingAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	_ generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	b.codecs = codecs
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(b.gv.Group, scheme, metav1.ParameterCodec, codecs)

pkg/registry/apis/featuretoggle/register.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/accesscontrol"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*FeatureFlagAPIBuilder)(nil)
@@ -40,6 +41,7 @@ func RegisterAPIService(features *featuremgmt.FeatureManager,
 	accessControl accesscontrol.AccessControl,
 	apiregistration builder.APIRegistrar,
 	cfg *setting.Cfg,
+	registerer prometheus.Registerer,
 ) *FeatureFlagAPIBuilder {
 	builder := NewFeatureFlagAPIBuilder(features, accessControl, cfg)
 	apiregistration.RegisterAPI(builder)
@@ -89,6 +91,7 @@ func (b *FeatureFlagAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	_ generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs)
 

pkg/registry/apis/folders/register.go

@@ -27,6 +27,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 	"github.com/grafana/grafana/pkg/services/folder"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*FolderAPIBuilder)(nil)
@@ -47,6 +48,7 @@ func RegisterAPIService(cfg *setting.Cfg,
 	apiregistration builder.APIRegistrar,
 	folderSvc folder.Service,
 	accessControl accesscontrol.AccessControl,
+	registerer prometheus.Registerer,
 ) *FolderAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
 		return nil // skip registration unless opting into experimental apis
@@ -106,6 +108,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	optsGetter generic.RESTOptionsGetter,
 	desiredMode grafanarest.DualWriterMode,
+	reg prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(v0alpha1.GROUP, scheme, metav1.ParameterCodec, codecs)
 
@@ -145,7 +148,7 @@ func (b *FolderAPIBuilder) GetAPIGroupInfo(
 		if err != nil {
 			return nil, err
 		}
-		storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store)
+		storage[resourceInfo.StoragePath()] = grafanarest.NewDualWriter(grafanarest.Mode1, legacyStore, store, reg)
 	}
 
 	apiGroupInfo.VersionedResourcesStorageMap[v0alpha1.VERSION] = storage

pkg/registry/apis/peakq/register.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*PeakQAPIBuilder)(nil)
@@ -28,7 +29,7 @@ func NewPeakQAPIBuilder() *PeakQAPIBuilder {
 	return &PeakQAPIBuilder{}
 }
 
-func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *PeakQAPIBuilder {
+func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *PeakQAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
 		return nil // skip registration unless opting into experimental apis
 	}
@@ -73,6 +74,7 @@ func (b *PeakQAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory,
 	optsGetter generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode, // dual write desired mode (not relevant)
+	_ prometheus.Registerer, // prometheus registerer
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(peakq.GROUP, scheme, metav1.ParameterCodec, codecs)
 

pkg/registry/apis/playlist/register.go

@@ -23,6 +23,7 @@ import (
 	gapiutil "github.com/grafana/grafana/pkg/services/apiserver/utils"
 	playlistsvc "github.com/grafana/grafana/pkg/services/playlist"
 	"github.com/grafana/grafana/pkg/setting"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*PlaylistAPIBuilder)(nil)
@@ -39,12 +40,14 @@ func RegisterAPIService(p playlistsvc.Service,
 	apiregistration builder.APIRegistrar,
 	cfg *setting.Cfg,
 	kvStore kvstore.KVStore,
+	registerer prometheus.Registerer,
 ) *PlaylistAPIBuilder {
 	builder := &PlaylistAPIBuilder{
 		service:    p,
 		namespacer: request.GetNamespaceMapper(cfg),
 		gv:         playlist.PlaylistResourceInfo.GroupVersion(),
 		kvStore:    kvstore.WithNamespace(kvStore, 0, "storage.dualwriting"),
+		// register:  newMetrics(registerer),
 	}
 	apiregistration.RegisterAPI(builder)
 	return builder
@@ -94,6 +97,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	optsGetter generic.RESTOptionsGetter,
 	desiredMode grafanarest.DualWriterMode,
+	reg prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(playlist.GROUP, scheme, metav1.ParameterCodec, codecs)
 	storage := map[string]rest.Storage{}
@@ -133,7 +137,7 @@ func (b *PlaylistAPIBuilder) GetAPIGroupInfo(
 			return nil, err
 		}
 
-		dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, legacyStore, store, playlist.GROUPRESOURCE, desiredMode)
+		dualWriter, err := grafanarest.SetDualWritingMode(context.Background(), b.kvStore, legacyStore, store, playlist.GROUPRESOURCE, desiredMode, reg)
 		if err != nil {
 			return nil, err
 		}

pkg/registry/apis/query/metrics.go

@@ -9,15 +9,15 @@ const (
 	metricsNamespace = "grafana"
 )
 
-type metrics struct {
+type queryMetrics struct {
 	dsRequests *prometheus.CounterVec
 
 	// older metric
 	expressionsQuerySummary *prometheus.SummaryVec
 }
 
-func newMetrics(reg prometheus.Registerer) *metrics {
-	m := &metrics{
+func newQueryMetrics(reg prometheus.Registerer) *queryMetrics {
+	m := &queryMetrics{
 		dsRequests: prometheus.NewCounterVec(prometheus.CounterOpts{
 			Namespace: metricsNamespace,
 			Subsystem: metricsSubSystem,

pkg/registry/apis/query/register.go

@@ -42,7 +42,7 @@ type QueryAPIBuilder struct {
 	features               featuremgmt.FeatureToggles
 
 	tracer     tracing.Tracer
-	metrics    *metrics
+	metrics    *queryMetrics
 	parser     *queryParser
 	client     DataSourceClientSupplier
 	registry   query.DataSourceApiServerRegistry
@@ -81,7 +81,7 @@ func NewQueryAPIBuilder(features featuremgmt.FeatureToggles,
 		client:               client,
 		registry:             registry,
 		parser:               newQueryParser(reader, legacy, tracer),
-		metrics:              newMetrics(registerer),
+		metrics:              newQueryMetrics(registerer),
 		tracer:               tracer,
 		features:             features,
 		queryTypes:           queryTypes,
@@ -151,6 +151,7 @@ func (b *QueryAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory, // pointer?
 	optsGetter generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	gv := query.SchemeGroupVersion
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(gv.Group, scheme, metav1.ParameterCodec, codecs)

pkg/registry/apis/scope/register.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*ScopeAPIBuilder)(nil)
@@ -30,7 +31,7 @@ func NewScopeAPIBuilder() *ScopeAPIBuilder {
 	return &ScopeAPIBuilder{}
 }
 
-func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ScopeAPIBuilder {
+func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, reg prometheus.Registerer) *ScopeAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagGrafanaAPIServerWithExperimentalAPIs) {
 		return nil // skip registration unless opting into experimental apis
 	}
@@ -121,6 +122,7 @@ func (b *ScopeAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory,
 	optsGetter generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode, // dual write desired mode (not relevant)
+	_ prometheus.Registerer, // prometheus registerer
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(scope.GROUP, scheme, metav1.ParameterCodec, codecs)
 

pkg/registry/apis/service/register.go

@@ -15,6 +15,7 @@ import (
 	"github.com/grafana/grafana/pkg/apiserver/builder"
 	grafanarest "github.com/grafana/grafana/pkg/apiserver/rest"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
+	"github.com/prometheus/client_golang/prometheus"
 )
 
 var _ builder.APIGroupBuilder = (*ServiceAPIBuilder)(nil)
@@ -26,7 +27,7 @@ func NewServiceAPIBuilder() *ServiceAPIBuilder {
 	return &ServiceAPIBuilder{}
 }
 
-func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar) *ServiceAPIBuilder {
+func RegisterAPIService(features featuremgmt.FeatureToggles, apiregistration builder.APIRegistrar, registerer prometheus.Registerer) *ServiceAPIBuilder {
 	if !features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
 		return nil // skip registration unless opting into aggregator mode
 	}
@@ -79,6 +80,7 @@ func (b *ServiceAPIBuilder) GetAPIGroupInfo(
 	codecs serializer.CodecFactory,
 	optsGetter generic.RESTOptionsGetter,
 	_ grafanarest.DualWriterMode,
+	_ prometheus.Registerer,
 ) (*genericapiserver.APIGroupInfo, error) {
 	apiGroupInfo := genericapiserver.NewDefaultAPIGroupInfo(service.GROUP, scheme, metav1.ParameterCodec, codecs)
 

pkg/services/apiserver/aggregator/aggregator.go

@@ -24,6 +24,7 @@ import (
 
 	servicev0alpha1 "github.com/grafana/grafana/pkg/apis/service/v0alpha1"
 	"github.com/grafana/grafana/pkg/registry/apis/service"
+	"github.com/prometheus/client_golang/prometheus"
 	"gopkg.in/yaml.v3"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
 	"k8s.io/apimachinery/pkg/runtime/schema"
@@ -172,7 +173,7 @@ func CreateAggregatorConfig(commandOptions *options.Options, sharedConfig generi
 	return NewConfig(aggregatorConfig, sharedInformerFactory, []builder.APIGroupBuilder{serviceAPIBuilder}, remoteServicesConfig), nil
 }
 
-func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.DelegationTarget) (*aggregatorapiserver.APIAggregator, error) {
+func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.DelegationTarget, reg prometheus.Registerer) (*aggregatorapiserver.APIAggregator, error) {
 	aggregatorConfig := config.KubeAggregatorConfig
 	sharedInformerFactory := config.Informers
 	remoteServicesConfig := config.RemoteServicesConfig
@@ -285,7 +286,13 @@ func CreateAggregatorServer(config *Config, delegateAPIServer genericapiserver.D
 	})
 
 	for _, b := range config.Builders {
-		serviceAPIGroupInfo, err := b.GetAPIGroupInfo(aggregatorscheme.Scheme, aggregatorscheme.Codecs, aggregatorConfig.GenericConfig.RESTOptionsGetter, grafanarest.Mode0)
+		serviceAPIGroupInfo, err := b.GetAPIGroupInfo(
+			aggregatorscheme.Scheme,
+			aggregatorscheme.Codecs,
+			aggregatorConfig.GenericConfig.RESTOptionsGetter,
+			grafanarest.Mode0,
+			reg,
+		)
 		if err != nil {
 			return nil, err
 		}

pkg/services/apiserver/service.go

@@ -7,6 +7,7 @@ import (
 	"path"
 
 	"github.com/grafana/dskit/services"
+	"github.com/prometheus/client_golang/prometheus"
 	"google.golang.org/grpc"
 	"google.golang.org/grpc/credentials/insecure"
 	metav1 "k8s.io/apimachinery/pkg/apis/meta/v1"
@@ -24,6 +25,7 @@ import (
 	filestorage "github.com/grafana/grafana/pkg/apiserver/storage/file"
 	"github.com/grafana/grafana/pkg/infra/appcontext"
 	"github.com/grafana/grafana/pkg/infra/db"
+	"github.com/grafana/grafana/pkg/infra/metrics"
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/middleware"
 	"github.com/grafana/grafana/pkg/modules"
@@ -109,6 +111,7 @@ type service struct {
 	builders []builder.APIGroupBuilder
 
 	tracing *tracing.TracingService
+	metrics prometheus.Registerer
 
 	authorizer *authorizer.GrafanaAuthorizer
 }
@@ -131,6 +134,7 @@ func ProvideService(
 		authorizer: authorizer.NewGrafanaAuthorizer(cfg, orgService),
 		tracing:    tracing,
 		db:         db, // For Unified storage
+		metrics:    metrics.ProvideRegisterer(),
 	}
 
 	// This will be used when running as a dskit service
@@ -314,7 +318,7 @@ func (s *service) start(ctx context.Context) error {
 	}
 
 	// Install the API group+version
-	err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions)
+	err = builder.InstallAPIs(Scheme, Codecs, server, serverConfig.RESTOptionsGetter, builders, o.StorageOptions, s.metrics)
 	if err != nil {
 		return err
 	}
@@ -324,7 +328,7 @@ func (s *service) start(ctx context.Context) error {
 
 	var runningServer *genericapiserver.GenericAPIServer
 	if s.features.IsEnabledGlobally(featuremgmt.FlagKubernetesAggregator) {
-		runningServer, err = s.startAggregator(transport, serverConfig, server)
+		runningServer, err = s.startAggregator(transport, serverConfig, server, s.metrics)
 		if err != nil {
 			return err
 		}
@@ -374,6 +378,7 @@ func (s *service) startAggregator(
 	transport *roundTripperFunc,
 	serverConfig *genericapiserver.RecommendedConfig,
 	server *genericapiserver.GenericAPIServer,
+	reg prometheus.Registerer,
 ) (*genericapiserver.GenericAPIServer, error) {
 	namespaceMapper := request.GetNamespaceMapper(s.cfg)
 
@@ -382,7 +387,7 @@ func (s *service) startAggregator(
 		return nil, err
 	}
 
-	aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, server)
+	aggregatorServer, err := aggregator.CreateAggregatorServer(aggregatorConfig, server, reg)
 	if err != nil {
 		return nil, err
 	}

pkg/services/apiserver/standalone/options/metrics.go

@@ -15,7 +15,7 @@ type MetricsOptions struct {
 	MetricsRegisterer prometheus.Registerer
 }
 
-func NewMetrcicsOptions(logger log.Logger) *MetricsOptions {
+func NewMetricsOptions(logger log.Logger) *MetricsOptions {
 	return &MetricsOptions{
 		logger: logger,
 	}

pkg/services/apiserver/standalone/options/options.go

@@ -25,7 +25,7 @@ func New(logger log.Logger, codec runtime.Codec) *Options {
 		ExtraOptions:       options.NewExtraOptions(),
 		RecommendedOptions: options.NewRecommendedOptions(codec),
 		TracingOptions:     NewTracingOptions(logger),
-		MetricsOptions:     NewMetrcicsOptions(logger),
+		MetricsOptions:     NewMetricsOptions(logger),
 		ServerRunOptions:   genericoptions.NewServerRunOptions(),
 		StorageOptions:     options.NewStorageOptions(),
 	}