Instrumentation: Add support for instrumenting database queries (#66022)

Signed-off-by: bergquist <carl.bergquist@gmail.com>
Co-authored-by: Christopher Moyer <35463610+chri2547@users.noreply.github.com>

conf/defaults.ini

@@ -159,6 +159,9 @@ query_retries = 0
 # For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5.
 transaction_retries = 5
 
+# Set to true to add metrics and tracing for database queries. 
+instrument_queries = false
+
 #################################### Cache server #############################
 [remote_cache]
 # Either "redis", "memcached" or "database" default is "database"

conf/sample.ini

@@ -161,6 +161,9 @@
 # For "sqlite" only. How many times to retry transaction in case of database is locked failures. Default is 5.
 ;transaction_retries = 5
 
+# Set to true to add metrics and tracing for database queries. 
+;instrument_queries = false
+
 ################################### Data sources #########################
 [datasources]
 # Upper limit of data sources that Grafana will return. This limit is a temporary configuration and it will be deprecated when pagination will be introduced on the list data sources API.

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -404,6 +404,10 @@ This setting applies to `sqlite` only and controls the number of times the syste
 
 This setting applies to `sqlite` only and controls the number of times the system retries a transaction when the database is locked. The default value is `5`.
 
+### instrument_queries
+
+Set to `true` to add metrics and tracing for database queries. The default value is `false`.
+
 <hr />
 
 ## [remote_cache]

docs/sources/setup-grafana/configure-grafana/feature-toggles/index.md

@@ -22,7 +22,6 @@ Some stable features are enabled by default. You can disable a stable feature by
 | Feature toggle name                 | Description                                                                                                                                                                                         | Enabled by default |
 | ----------------------------------- | --------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------- | ------------------ |
 | `disableEnvelopeEncryption`         | Disable envelope encryption (emergency only)                                                                                                                                                        |                    |
-| `database_metrics`                  | Add Prometheus metrics for database tables                                                                                                                                                          |                    |
 | `featureHighlights`                 | Highlight Grafana Enterprise features                                                                                                                                                               |                    |
 | `cloudWatchDynamicLabels`           | Use dynamic labels instead of alias patterns in CloudWatch datasource                                                                                                                               | Yes                |
 | `dataConnectionsConsole`            | Enables a new top-level page called Connections. This page is an experiment that provides a better experience when you install and configure data sources and other plugins.                        | Yes                |

packages/grafana-data/src/types/featureToggles.gen.ts

@@ -20,7 +20,6 @@
 export interface FeatureToggles {
   trimDefaults?: boolean;
   disableEnvelopeEncryption?: boolean;
-  database_metrics?: boolean;
   ['live-service-web-worker']?: boolean;
   queryOverLive?: boolean;
   panelTitleSearch?: boolean;

pkg/middleware/loggermw/logger.go

@@ -119,7 +119,7 @@ func (l *loggerImpl) prepareLogParams(c *contextmodel.ReqContext, duration time.
 		lvl = lvl.HighestOf(errutil.LevelWarn)
 	}
 
-	if l.flags.IsEnabled(featuremgmt.FlagDatabaseMetrics) {
+	if l.cfg.DatabaseInstrumentQueries {
 		logParams = append(logParams, "db_call_count", log.TotalDBCallCount(c.Req.Context()))
 	}
 

pkg/services/featuremgmt/registry.go

@@ -21,12 +21,6 @@ var (
 			State:       FeatureStateStable,
 			Owner:       grafanaAsCodeSquad,
 		},
-		{
-			Name:        "database_metrics",
-			Description: "Add Prometheus metrics for database tables",
-			State:       FeatureStateStable,
-			Owner:       hostedGrafanaTeam,
-		},
 		{
 			Name:         "live-service-web-worker",
 			Description:  "This will use a webworker thread to processes events rather than the main thread",

pkg/services/featuremgmt/toggles_gen.csv

@@ -1,7 +1,6 @@
 Name,State,Owner,requiresDevMode,RequiresLicense,RequiresRestart,FrontendOnly
 trimDefaults,beta,@grafana/grafana-as-code,false,false,false,false
 disableEnvelopeEncryption,stable,@grafana/grafana-as-code,false,false,false,false
-database_metrics,stable,@grafana/hosted-grafana-team,false,false,false,false
 live-service-web-worker,alpha,@grafana/grafana-app-platform-squad,false,false,false,true
 queryOverLive,alpha,@grafana/grafana-app-platform-squad,false,false,false,true
 panelTitleSearch,beta,@grafana/grafana-app-platform-squad,false,false,false,false

pkg/services/featuremgmt/toggles_gen.go

@@ -15,10 +15,6 @@ const (
 	// Disable envelope encryption (emergency only)
 	FlagDisableEnvelopeEncryption = "disableEnvelopeEncryption"
 
-	// FlagDatabaseMetrics
-	// Add Prometheus metrics for database tables
-	FlagDatabaseMetrics = "database_metrics"
-
 	// FlagLiveServiceWebWorker
 	// This will use a webworker thread to processes events rather than the main thread
 	FlagLiveServiceWebWorker = "live-service-web-worker"

pkg/services/searchV2/index.go

@@ -591,10 +591,6 @@ func (i *searchIndex) withCtxData(ctx context.Context, params ...interface{}) []
 		params = append(params, "traceID", traceID)
 	}
 
-	if i.features.IsEnabled(featuremgmt.FlagDatabaseMetrics) {
-		params = append(params, "db_call_count", log.TotalDBCallCount(ctx))
-	}
-
 	return params
 }
 

pkg/services/sqlstore/sqlstore.go

@@ -362,7 +362,7 @@ func (ss *SQLStore) initEngine(engine *xorm.Engine) error {
 		return err
 	}
 
-	if ss.Cfg.IsFeatureToggleEnabled(featuremgmt.FlagDatabaseMetrics) {
+	if ss.Cfg.DatabaseInstrumentQueries {
 		ss.dbCfg.Type = WrapDatabaseDriverWithHooks(ss.dbCfg.Type, ss.tracer)
 	}
 

pkg/setting/setting.go

@@ -519,6 +519,12 @@ type Cfg struct {
 	GRPCServerTLSConfig *tls.Config
 
 	CustomResponseHeaders map[string]string
+
+	// DatabaseInstrumentQueries is used to decide if database queries
+	// should be instrumented with metrics, logs and traces.
+	// This needs to be on the global object since its used in the
+	// sqlstore package and HTTP middlewares.
+	DatabaseInstrumentQueries bool
 }
 
 // AddChangePasswordLink returns if login form is disabled or not since
@@ -1189,6 +1195,9 @@ func (cfg *Cfg) Load(args CommandLineArgs) error {
 
 	cfg.LogConfigSources()
 
+	databaseSection := iniFile.Section("database")
+	cfg.DatabaseInstrumentQueries = databaseSection.Key("instrument_queries").MustBool(false)
+
 	return nil
 }