Tracing: Standardize on otel tracing (#75528)

contribute/backend/instrumentation.md

@@ -156,7 +156,7 @@ A distributed trace is data that tracks an application request as it flows throu
 
 ### Usage
 
-Grafana currently supports two tracing implementations, [OpenTelemetry](https://opentelemetry.io/) and [OpenTracing](https://opentracing.io/). OpenTracing is deprecated, but still supported until we remove it. The two different implementations implements the `Tracer` and `Span` interfaces, defined in the _pkg/infra/tracing_ package, which you can use to create traces and spans. To get a hold of a `Tracer` you would need to get it injected as dependency into your service, see [Services](services.md) for more details.
+Grafana uses [OpenTelemetry](https://opentelemetry.io/) for distributed tracing. There's an interface `Tracer` in the _pkg/infra/tracing_ package that implements the [OpenTelemetry Tracer interface](go.opentelemetry.io/otel/trace), which you can use to create traces and spans. To get a hold of a `Tracer` you would need to get it injected as dependency into your service, see [Services](services.md) for more details. For more information, see https://opentelemetry.io/docs/instrumentation/go/manual/.
 
 Example:
 
@@ -166,6 +166,7 @@ import (
 
    "github.com/grafana/grafana/pkg/infra/tracing"
    "go.opentelemetry.io/otel/attribute"
+   "go.opentelemetry.io/otel/trace"
 )
 
 type MyService struct {
@@ -179,36 +180,36 @@ func ProvideService(tracer tracing.Tracer) *MyService {
 }
 
 func (s *MyService) Hello(ctx context.Context, name string) (string, error) {
-   ctx, span := s.tracer.Start(ctx, "MyService.Hello")
+   ctx, span := s.tracer.Start(ctx, "MyService.Hello", trace.WithAttributes(
+      attribute.String("my_attribute", "val"),
+   ))
    // this make sure the span is marked as finished when this
    // method ends to allow the span to be flushed and sent to
    // storage backend.
    defer span.End()
 
    // Add some event to show Events usage
-   span.AddEvents(
+   span.AddEvent("checking name...")
-      []string{"message"},
-      []tracing.EventValue{
-         {Str: "checking name..."},
-      })
 
    if name == "" {
       err := fmt.Errorf("name cannot be empty")
 
+      // sets the span’s status to Error to make the span tracking
+      // a failed operation as an error span.
+      span.SetStatus(codes.Error, "failed to check name")
       // record err as an exception span event for this span
       span.RecordError(err)
       return "", err
    }
 
    // Add some other event to show Events usage
-   span.AddEvents(
+   span.AddEvent("name checked")
-      []string{"message"},
-      []tracing.EventValue{
-         {Str: "name checked"},
-      })
 
    // Add attribute to show Attributes usage
-   span.SetAttributes("my_service.name", name, attribute.Key("my_service.name").String(name))
+   span.SetAttributes(
+      attribute.String("my_service.name", name),
+      attribute.Int64("my_service.some_other", int64(1337)),
+   )
 
    return fmt.Sprintf("Hello %s", name), nil
 }
@@ -243,6 +244,22 @@ If span names, attribute or event values originates from user input they **shoul
 
 Be **careful** to not expose any sensitive information in span names, attribute or event values, e.g. secrets, credentials etc.
 
+### Span attributes
+
+Consider using `attributes.<Type>("<key>", <value>)` in favor of `attributes.Key("<key>").<Type>(<value>)` since it requires less characters and thereby reads easier.
+
+Example:
+
+```go
+attribute.String("datasource_name", proxy.ds.Name)
+// vs
+attribute.Key("datasource_name").String(proxy.ds.Name)
+
+attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID)
+// vs
+attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID)
+```
+
 ### How to collect, visualize and query traces (and correlate logs with traces) locally
 
 #### 1. Start Jaeger
@@ -255,20 +272,11 @@ make devenv sources=jaeger
 
 To enable tracing in Grafana, you must set the address in your config.ini file
 
-opentelemetry tracing (recommended):
-
 ```ini
 [tracing.opentelemetry.jaeger]
 address = http://localhost:14268/api/traces
 ```
 
-opentracing tracing (deprecated/not recommended):
-
-```ini
-[tracing.jaeger]
-address = localhost:6831
-```
-
 #### 3. Search/browse collected logs and traces in Grafana Explore
 
 You need provisioned gdev-jaeger and gdev-loki datasources, see [developer dashboard and data sources](https://github.com/grafana/grafana/tree/main/devenv#developer-dashboards-and-data-sources) for setup instructions.

go.mod

@@ -121,7 +121,7 @@ require (
 	gopkg.in/mail.v2 v2.3.1 // @grafana/backend-platform
 	gopkg.in/yaml.v2 v2.4.0 // indirect
 	gopkg.in/yaml.v3 v3.0.1 // @grafana/alerting-squad-backend
-	xorm.io/builder v0.3.6 // indirect; @grafana/backend-platform
+	xorm.io/builder v0.3.6 // @grafana/backend-platform
 	xorm.io/core v0.7.3 // @grafana/backend-platform
 	xorm.io/xorm v0.8.2 // @grafana/alerting-squad-backend
 )
@@ -174,7 +174,7 @@ require (
 	github.com/grpc-ecosystem/go-grpc-prometheus v1.2.1-0.20191002090509-6af20e3a5340 // indirect
 	github.com/hashicorp/errwrap v1.1.0 // indirect
 	github.com/hashicorp/go-msgpack v0.5.5 // indirect
-	github.com/hashicorp/go-multierror v1.1.1 // indirect; @grafana/grafana-as-code
+	github.com/hashicorp/go-multierror v1.1.1 // @grafana/grafana-as-code
 	github.com/hashicorp/go-sockaddr v1.0.2 // indirect
 	github.com/hashicorp/golang-lru v0.6.0 // indirect
 	github.com/hashicorp/yamux v0.1.1 // indirect

pkg/api/pluginproxy/ds_proxy.go

@@ -12,6 +12,7 @@ import (
 	"time"
 
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/api/datasource"
 	"github.com/grafana/grafana/pkg/infra/httpclient"
@@ -142,10 +143,12 @@ func (proxy *DataSourceProxy) HandleRequest() {
 
 	proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)
 
-	span.SetAttributes("datasource_name", proxy.ds.Name, attribute.Key("datasource_name").String(proxy.ds.Name))
+	span.SetAttributes(
-	span.SetAttributes("datasource_type", proxy.ds.Type, attribute.Key("datasource_type").String(proxy.ds.Type))
+		attribute.String("datasource_name", proxy.ds.Name),
-	span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
+		attribute.String("datasource_type", proxy.ds.Type),
-	span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgID, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID))
+		attribute.String("user", proxy.ctx.SignedInUser.Login),
+		attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID),
+	)
 
 	proxy.addTraceFromHeaderValue(span, "X-Panel-Id", "panel_id")
 	proxy.addTraceFromHeaderValue(span, "X-Dashboard-Id", "dashboard_id")
@@ -155,11 +158,11 @@ func (proxy *DataSourceProxy) HandleRequest() {
 	reverseProxy.ServeHTTP(proxy.ctx.Resp, proxy.ctx.Req)
 }
 
-func (proxy *DataSourceProxy) addTraceFromHeaderValue(span tracing.Span, headerName string, tagName string) {
+func (proxy *DataSourceProxy) addTraceFromHeaderValue(span trace.Span, headerName string, tagName string) {
 	panelId := proxy.ctx.Req.Header.Get(headerName)
 	dashId, err := strconv.Atoi(panelId)
 	if err == nil {
-		span.SetAttributes(tagName, dashId, attribute.Key(tagName).Int(dashId))
+		span.SetAttributes(attribute.Int(tagName, dashId))
 	}
 }
 

pkg/api/pluginproxy/pluginproxy.go

@@ -109,8 +109,10 @@ func (proxy *PluginProxy) HandleRequest() {
 
 	proxy.ctx.Req = proxy.ctx.Req.WithContext(ctx)
 
-	span.SetAttributes("user", proxy.ctx.SignedInUser.Login, attribute.Key("user").String(proxy.ctx.SignedInUser.Login))
+	span.SetAttributes(
-	span.SetAttributes("org_id", proxy.ctx.SignedInUser.OrgID, attribute.Key("org_id").Int64(proxy.ctx.SignedInUser.OrgID))
+		attribute.String("user", proxy.ctx.SignedInUser.Login),
+		attribute.Int64("org_id", proxy.ctx.SignedInUser.OrgID),
+	)
 
 	proxy.tracer.Inject(ctx, proxy.ctx.Req.Header, span)
 

pkg/bus/bus.go

@@ -55,7 +55,7 @@ func (b *InProcBus) Publish(ctx context.Context, msg Msg) error {
 	_, span := b.tracer.Start(ctx, "bus - "+msgName)
 	defer span.End()
 
-	span.SetAttributes("msg", msgName, attribute.Key("msg").String(msgName))
+	span.SetAttributes(attribute.String("msg", msgName))
 
 	return nil
 }

pkg/expr/commands.go

@@ -70,7 +70,7 @@ func (gm *MathCommand) NeedsVars() []string {
 // failed to execute.
 func (gm *MathCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.Vars, tracer tracing.Tracer) (mathexp.Results, error) {
 	_, span := tracer.Start(ctx, "SSE.ExecuteMath")
-	span.SetAttributes("expression", gm.RawExpression, attribute.Key("expression").String(gm.RawExpression))
+	span.SetAttributes(attribute.String("expression", gm.RawExpression))
 	defer span.End()
 	return gm.Expression.Execute(gm.refID, vars, tracer)
 }
@@ -163,7 +163,7 @@ func (gr *ReduceCommand) Execute(ctx context.Context, _ time.Time, vars mathexp.
 	_, span := tracer.Start(ctx, "SSE.ExecuteReduce")
 	defer span.End()
 
-	span.SetAttributes("reducer", gr.Reducer, attribute.Key("reducer").String(gr.Reducer))
+	span.SetAttributes(attribute.String("reducer", gr.Reducer))
 
 	newRes := mathexp.Results{}
 	for i, val := range vars[gr.VarToReduce].Values {

pkg/expr/dataplane.go

@@ -61,7 +61,7 @@ func shouldUseDataplane(frames data.Frames, logger log.Logger, disable bool) (dt
 func handleDataplaneFrames(ctx context.Context, tracer tracing.Tracer, t data.FrameType, frames data.Frames) (mathexp.Results, error) {
 	_, span := tracer.Start(ctx, "SSE.HandleDataPlaneData")
 	defer span.End()
-	span.SetAttributes("dataplane.type", t, attribute.Key("dataplane.type").String(string(t)))
+	span.SetAttributes(attribute.String("dataplane.type", string(t)))
 
 	switch t.Kind() {
 	case data.KindUnknown:

pkg/expr/graph.go

@@ -99,10 +99,10 @@ func (dp *DataPipeline) execute(c context.Context, now time.Time, s *Service) (m
 		}
 
 		c, span := s.tracer.Start(c, "SSE.ExecuteNode")
-		span.SetAttributes("node.refId", node.RefID(), attribute.Key("node.refId").String(node.RefID()))
+		span.SetAttributes(attribute.String("node.refId", node.RefID()))
 		if len(node.NeedsVars()) > 0 {
 			inputRefIDs := node.NeedsVars()
-			span.SetAttributes("node.inputRefIDs", inputRefIDs, attribute.Key("node.inputRefIDs").StringSlice(inputRefIDs))
+			span.SetAttributes(attribute.StringSlice("node.inputRefIDs", inputRefIDs))
 		}
 		defer span.End()
 

pkg/expr/nodes.go

@@ -10,12 +10,12 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	"gonum.org/v1/gonum/graph/simple"
 
 	"github.com/grafana/grafana/pkg/expr/classic"
 	"github.com/grafana/grafana/pkg/expr/mathexp"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/datasources"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
 )
@@ -236,8 +236,10 @@ func executeDSNodesGrouped(ctx context.Context, now time.Time, vars mathexp.Vars
 				"datasourceVersion", firstNode.datasource.Version,
 			)
 
-			span.SetAttributes("datasource.type", firstNode.datasource.Type, attribute.Key("datasource.type").String(firstNode.datasource.Type))
+			span.SetAttributes(
-			span.SetAttributes("datasource.uid", firstNode.datasource.UID, attribute.Key("datasource.uid").String(firstNode.datasource.UID))
+				attribute.String("datasource.type", firstNode.datasource.Type),
+				attribute.String("datasource.uid", firstNode.datasource.UID),
+			)
 
 			req := &backend.QueryDataRequest{
 				PluginContext: pCtx,
@@ -261,11 +263,8 @@ func executeDSNodesGrouped(ctx context.Context, now time.Time, vars mathexp.Vars
 				if e != nil {
 					responseType = "error"
 					respStatus = "failure"
-					span.AddEvents([]string{"error", "message"},
+					span.SetStatus(codes.Error, "failed to query data source")
-						[]tracing.EventValue{
+					span.RecordError(e)
-							{Str: fmt.Sprintf("%v", err)},
-							{Str: "failed to query data source"},
-						})
 				}
 				logger.Debug("Data source queried", "responseType", responseType)
 				useDataplane := strings.HasPrefix(responseType, "dataplane-")
@@ -313,8 +312,10 @@ func (dn *DSNode) Execute(ctx context.Context, now time.Time, _ mathexp.Vars, s
 	if err != nil {
 		return mathexp.Results{}, err
 	}
-	span.SetAttributes("datasource.type", dn.datasource.Type, attribute.Key("datasource.type").String(dn.datasource.Type))
+	span.SetAttributes(
-	span.SetAttributes("datasource.uid", dn.datasource.UID, attribute.Key("datasource.uid").String(dn.datasource.UID))
+		attribute.String("datasource.type", dn.datasource.Type),
+		attribute.String("datasource.uid", dn.datasource.UID),
+	)
 
 	req := &backend.QueryDataRequest{
 		PluginContext: pCtx,
@@ -337,11 +338,8 @@ func (dn *DSNode) Execute(ctx context.Context, now time.Time, _ mathexp.Vars, s
 		if e != nil {
 			responseType = "error"
 			respStatus = "failure"
-			span.AddEvents([]string{"error", "message"},
+			span.SetStatus(codes.Error, "failed to query data source")
-				[]tracing.EventValue{
+			span.RecordError(e)
-					{Str: fmt.Sprintf("%v", err)},
-					{Str: "failed to query data source"},
-				})
 		}
 		logger.Debug("Data source queried", "responseType", responseType)
 		useDataplane := strings.HasPrefix(responseType, "dataplane-")

pkg/infra/httpclient/httpclientprovider/tracing_middleware.go

@@ -10,6 +10,7 @@ import (
 	"go.opentelemetry.io/contrib/instrumentation/net/http/httptrace/otelhttptrace"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -30,17 +31,18 @@ func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Midd
 			ctx = httptrace.WithClientTrace(ctx, otelhttptrace.NewClientTrace(ctx, otelhttptrace.WithoutSubSpans(), otelhttptrace.WithoutHeaders()))
 			req = req.WithContext(ctx)
 			for k, v := range opts.Labels {
-				span.SetAttributes(k, v, attribute.Key(k).String(v))
+				span.SetAttributes(attribute.String(k, v))
 			}
 
 			tracer.Inject(ctx, req.Header, span)
 			res, err := next.RoundTrip(req)
 
-			span.SetAttributes("http.url", req.URL.String(), attribute.String("http.url", req.URL.String()))
+			span.SetAttributes(semconv.HTTPURL(req.URL.String()))
-			span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
+			span.SetAttributes(semconv.HTTPMethod(req.Method))
 			// ext.SpanKind.Set(span, ext.SpanKindRPCClientEnum)
 
 			if err != nil {
+				span.SetStatus(codes.Error, "request failed")
 				span.RecordError(err)
 				return res, err
 			}
@@ -49,10 +51,10 @@ func TracingMiddleware(logger log.Logger, tracer tracing.Tracer) httpclient.Midd
 				// we avoid measuring contentlength less than zero because it indicates
 				// that the content size is unknown. https://godoc.org/github.com/badu/http#Response
 				if res.ContentLength > 0 {
-					span.SetAttributes(httpContentLengthTagKey, res.ContentLength, attribute.Key(httpContentLengthTagKey).Int64(res.ContentLength))
+					span.SetAttributes(attribute.Int64(httpContentLengthTagKey, res.ContentLength))
 				}
 
-				span.SetAttributes("http.status_code", res.StatusCode, attribute.Int("http.status_code", res.StatusCode))
+				span.SetAttributes(semconv.HTTPStatusCode(res.StatusCode))
 				if res.StatusCode >= 400 {
 					span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(res.StatusCode)))
 				}

pkg/infra/serverlock/serverlock.go

@@ -34,7 +34,7 @@ type ServerLockService struct {
 func (sl *ServerLockService) LockAndExecute(ctx context.Context, actionName string, maxInterval time.Duration, fn func(ctx context.Context)) error {
 	start := time.Now()
 	ctx, span := sl.tracer.Start(ctx, "ServerLockService.LockAndExecute")
-	span.SetAttributes("serverlock.actionName", actionName, attribute.Key("serverlock.actionName").String(actionName))
+	span.SetAttributes(attribute.String("serverlock.actionName", actionName))
 	defer span.End()
 
 	ctxLogger := sl.log.FromContext(ctx)
@@ -138,7 +138,7 @@ func (sl *ServerLockService) getOrCreate(ctx context.Context, actionName string)
 func (sl *ServerLockService) LockExecuteAndRelease(ctx context.Context, actionName string, maxInterval time.Duration, fn func(ctx context.Context)) error {
 	start := time.Now()
 	ctx, span := sl.tracer.Start(ctx, "ServerLockService.LockExecuteAndRelease")
-	span.SetAttributes("serverlock.actionName", actionName, attribute.Key("serverlock.actionName").String(actionName))
+	span.SetAttributes(attribute.String("serverlock.actionName", actionName))
 	defer span.End()
 
 	ctxLogger := sl.log.FromContext(ctx)

pkg/infra/tracing/test_helper.go

@@ -24,7 +24,7 @@ func InitializeTracerForTest(opts ...TracerForTestOption) Tracer {
 
 	otel.SetTracerProvider(tp)
 
-	ots := &Opentelemetry{Propagation: "jaeger,w3c", tracerProvider: tp}
+	ots := &TracingService{Propagation: "jaeger,w3c", tracerProvider: tp}
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }

pkg/infra/tracing/tracing.go

@@ -15,14 +15,13 @@ import (
 	"go.opentelemetry.io/contrib/samplers/jaegerremote"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
-	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
 	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	trace "go.opentelemetry.io/otel/trace"
 
 	"github.com/go-kit/log/level"
@@ -44,7 +43,7 @@ const (
 	w3cPropagator    string = "w3c"
 )
 
-type Opentelemetry struct {
+type TracingService struct {
 	enabled       string
 	Address       string
 	Propagation   string
@@ -57,7 +56,7 @@ type Opentelemetry struct {
 	log log.Logger
 
 	tracerProvider tracerProvider
-	tracer         trace.Tracer
+	trace.Tracer
 
 	Cfg *setting.Cfg
 }
@@ -68,24 +67,10 @@ type tracerProvider interface {
 	Shutdown(ctx context.Context) error
 }
 
-type OpentelemetrySpan struct {
-	span trace.Span
-}
-
-type EventValue struct {
-	Str string
-	Num int64
-}
-
 // Tracer defines the service used to create new spans.
 type Tracer interface {
-	// Run implements registry.BackgroundService.
+	trace.Tracer
-	Run(context.Context) error
+
-	// Start creates a new [Span] and places trace metadata on the
-	// [context.Context] passed to the method.
-	// Chose a low cardinality spanName and use [Span.SetAttributes]
-	// or [Span.AddEvents] for high cardinality data.
-	Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
 	// Inject adds identifying information for the span to the
 	// headers defined in [http.Header] map (this mutates http.Header).
 	//
@@ -94,47 +79,10 @@ type Tracer interface {
 	// information passed as [Span] is preferred.
 	// Both the context and span must be derived from the same call to
 	// [Tracer.Start].
-	Inject(context.Context, http.Header, Span)
+	Inject(context.Context, http.Header, trace.Span)
-
-	// OtelTracer returns the trace.Tracer if available or nil.
-	OtelTracer() trace.Tracer
-}
-
-// Span defines a time range for an operation. This is equivalent to a
-// single line in a flame graph.
-type Span interface {
-	// End finalizes the Span and adds its end timestamp.
-	// Any further operations on the Span are not permitted after
-	// End has been called.
-	End()
-	// SetAttributes adds additional data to a span.
-	// SetAttributes repeats the key value pair with [string] and [any]
-	// used for OpenTracing and [attribute.KeyValue] used for
-	// OpenTelemetry.
-	SetAttributes(key string, value any, kv attribute.KeyValue)
-	// SetName renames the span.
-	SetName(name string)
-	// SetStatus can be used to indicate whether the span was
-	// successfully or unsuccessfully executed.
-	//
-	// Only useful for OpenTelemetry.
-	SetStatus(code codes.Code, description string)
-	// RecordError adds an error to the span.
-	//
-	// Only useful for OpenTelemetry.
-	RecordError(err error, options ...trace.EventOption)
-	// AddEvents adds additional data with a temporal dimension to the
-	// span.
-	//
-	// Panics if the length of keys is shorter than the length of values.
-	AddEvents(keys []string, values []EventValue)
-
-	// contextWithSpan returns a context.Context that holds the parent
-	// context plus a reference to this span.
-	ContextWithSpan(ctx context.Context) context.Context
 }
 
-func ProvideService(cfg *setting.Cfg) (Tracer, error) {
+func ProvideService(cfg *setting.Cfg) (*TracingService, error) {
 	ots, err := ParseSettings(cfg)
 	if err != nil {
 		return nil, err
@@ -153,8 +101,8 @@ func ProvideService(cfg *setting.Cfg) (Tracer, error) {
 	return ots, nil
 }
 
-func ParseSettings(cfg *setting.Cfg) (*Opentelemetry, error) {
+func ParseSettings(cfg *setting.Cfg) (*TracingService, error) {
-	ots := &Opentelemetry{
+	ots := &TracingService{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
@@ -162,38 +110,13 @@ func ParseSettings(cfg *setting.Cfg) (*Opentelemetry, error) {
 	return ots, err
 }
 
-type traceKey struct{}
+func TraceIDFromContext(ctx context.Context, requireSampled bool) string {
-type traceValue struct {
+	spanCtx := trace.SpanContextFromContext(ctx)
-	ID        string
+	if !spanCtx.HasTraceID() || !spanCtx.IsValid() || (requireSampled && !spanCtx.IsSampled()) {
-	IsSampled bool
+		return ""
-}
-
-func TraceIDFromContext(c context.Context, requireSampled bool) string {
-	v := c.Value(traceKey{})
-	// Return traceID if a) it is present and b) it is sampled when requireSampled param is true
-	if trace, ok := v.(traceValue); ok && (!requireSampled || trace.IsSampled) {
-		return trace.ID
 	}
-	return ""
-}
 
-// SpanFromContext returns the Span previously associated with ctx, or nil, if no such span could be found.
+	return spanCtx.TraceID().String()
-// It is the equivalent of opentracing.SpanFromContext and trace.SpanFromContext.
-func SpanFromContext(ctx context.Context) Span {
-	if span := trace.SpanFromContext(ctx); span != nil {
-		return OpentelemetrySpan{span: span}
-	}
-	return nil
-}
-
-// ContextWithSpan returns a new context.Context that holds a reference to the given span.
-// If span is nil, a new context without an active span is returned.
-// It is the equivalent of opentracing.ContextWithSpan and trace.ContextWithSpan.
-func ContextWithSpan(ctx context.Context, span Span) context.Context {
-	if span != nil {
-		return span.ContextWithSpan(ctx)
-	}
-	return ctx
 }
 
 type noopTracerProvider struct {
@@ -204,7 +127,7 @@ func (noopTracerProvider) Shutdown(ctx context.Context) error {
 	return nil
 }
 
-func (ots *Opentelemetry) parseSettings() error {
+func (ots *TracingService) parseSettings() error {
 	legacyAddress, legacyTags := "", ""
 	if section, err := ots.Cfg.Raw.GetSection("tracing.jaeger"); err == nil {
 		legacyAddress = section.Key("address").MustString("")
@@ -263,7 +186,7 @@ func (ots *Opentelemetry) parseSettings() error {
 	return nil
 }
 
-func (ots *Opentelemetry) OTelExporterEnabled() bool {
+func (ots *TracingService) OTelExporterEnabled() bool {
 	return ots.enabled == otlpExporter
 }
 
@@ -283,7 +206,7 @@ func splitCustomAttribs(s string) ([]attribute.KeyValue, error) {
 	return res, nil
 }
 
-func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
+func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
 	var ep jaeger.EndpointOption
 	// Create the Jaeger exporter: address can be either agent address (host:port) or collector URL
 	if strings.HasPrefix(ots.Address, "http://") || strings.HasPrefix(ots.Address, "https://") {
@@ -328,7 +251,7 @@ func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider,
 	return tp, nil
 }
 
-func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
+func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
 	client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.Address), otlptracegrpc.WithInsecure())
 	exp, err := otlptrace.New(context.Background(), client)
 	if err != nil {
@@ -343,7 +266,7 @@ func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, er
 	return initTracerProvider(exp, ots.Cfg.BuildVersion, sampler, ots.customAttribs...)
 }
 
-func (ots *Opentelemetry) initSampler() (tracesdk.Sampler, error) {
+func (ots *TracingService) initSampler() (tracesdk.Sampler, error) {
 	switch ots.sampler {
 	case "const", "":
 		if ots.samplerParam >= 1 {
@@ -390,11 +313,11 @@ func initTracerProvider(exp tracesdk.SpanExporter, version string, sampler trace
 	return tp, nil
 }
 
-func (ots *Opentelemetry) initNoopTracerProvider() (tracerProvider, error) {
+func (ots *TracingService) initNoopTracerProvider() (tracerProvider, error) {
 	return &noopTracerProvider{TracerProvider: trace.NewNoopTracerProvider()}, nil
 }
 
-func (ots *Opentelemetry) initOpentelemetryTracer() error {
+func (ots *TracingService) initOpentelemetryTracer() error {
 	var tp tracerProvider
 	var err error
 	switch ots.enabled {
@@ -450,12 +373,12 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
 		ots.tracerProvider = tp
 	}
 
-	ots.tracer = otel.GetTracerProvider().Tracer("component-main")
+	ots.Tracer = otel.GetTracerProvider().Tracer("component-main")
 
 	return nil
 }
 
-func (ots *Opentelemetry) Run(ctx context.Context) error {
+func (ots *TracingService) Run(ctx context.Context) error {
 	otel.SetErrorHandler(otel.ErrorHandlerFunc(func(err error) {
 		err = level.Error(ots.log).Log("msg", "OpenTelemetry handler returned an error", "err", err)
 		if err != nil {
@@ -478,68 +401,12 @@ func (ots *Opentelemetry) Run(ctx context.Context) error {
 	return nil
 }
 
-func (ots *Opentelemetry) Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span) {
+func (ots *TracingService) Inject(ctx context.Context, header http.Header, _ trace.Span) {
-	ctx, span := ots.tracer.Start(ctx, spanName, opts...)
-	opentelemetrySpan := OpentelemetrySpan{
-		span: span,
-	}
-
-	if traceID := span.SpanContext().TraceID(); traceID.IsValid() {
-		ctx = context.WithValue(ctx, traceKey{}, traceValue{traceID.String(), span.SpanContext().IsSampled()})
-	}
-
-	return ctx, opentelemetrySpan
-}
-
-func (ots *Opentelemetry) Inject(ctx context.Context, header http.Header, _ Span) {
 	otel.GetTextMapPropagator().Inject(ctx, propagation.HeaderCarrier(header))
 }
 
-func (ots *Opentelemetry) OtelTracer() trace.Tracer {
+func (ots *TracingService) OtelTracer() trace.Tracer {
-	return ots.tracer
+	return ots
-}
-
-func (s OpentelemetrySpan) End() {
-	s.span.End()
-}
-
-func (s OpentelemetrySpan) SetAttributes(key string, value any, kv attribute.KeyValue) {
-	s.span.SetAttributes(kv)
-}
-
-func (s OpentelemetrySpan) SetName(name string) {
-	s.span.SetName(name)
-}
-
-func (s OpentelemetrySpan) SetStatus(code codes.Code, description string) {
-	s.span.SetStatus(code, description)
-}
-
-func (s OpentelemetrySpan) RecordError(err error, options ...trace.EventOption) {
-	s.span.RecordError(err, options...)
-}
-
-func (s OpentelemetrySpan) AddEvents(keys []string, values []EventValue) {
-	for i, v := range values {
-		if v.Str != "" {
-			s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).String(v.Str)))
-		}
-		if v.Num != 0 {
-			s.span.AddEvent(keys[i], trace.WithAttributes(attribute.Key(keys[i]).Int64(v.Num)))
-		}
-	}
-}
-
-func (s OpentelemetrySpan) ContextWithSpan(ctx context.Context) context.Context {
-	if s.span != nil {
-		ctx = trace.ContextWithSpan(ctx, s.span)
-		// Grafana also manages its own separate traceID in the context in addition to what opentracing handles.
-		// It's derived from the span. Ensure that we propagate this too.
-		if traceID := s.span.SpanContext().TraceID(); traceID.IsValid() {
-			ctx = context.WithValue(ctx, traceKey{}, traceValue{traceID.String(), s.span.SpanContext().IsSampled()})
-		}
-	}
-	return ctx
 }
 
 type rateLimiter struct {

pkg/infra/tracing/tracing_test.go

@@ -176,23 +176,22 @@ func TestTracingConfig(t *testing.T) {
 			tracer, err := ProvideService(cfg)
 			assert.NoError(t, err)
 			// make sure tracker is properly configured
-			otel := tracer.(*Opentelemetry)
+			assert.Equal(t, test.ExpectedExporter, tracer.enabled)
-			assert.Equal(t, test.ExpectedExporter, otel.enabled)
+			assert.Equal(t, test.ExpectedAddress, tracer.Address)
-			assert.Equal(t, test.ExpectedAddress, otel.Address)
+			assert.Equal(t, test.ExpectedPropagator, tracer.Propagation)
-			assert.Equal(t, test.ExpectedPropagator, otel.Propagation)
+			assert.Equal(t, test.ExpectedAttrs, tracer.customAttribs)
-			assert.Equal(t, test.ExpectedAttrs, otel.customAttribs)
 
 			if test.ExpectedSampler != "" {
-				assert.Equal(t, test.ExpectedSampler, otel.sampler)
+				assert.Equal(t, test.ExpectedSampler, tracer.sampler)
-				assert.Equal(t, test.ExpectedSamplerParam, otel.samplerParam)
+				assert.Equal(t, test.ExpectedSamplerParam, tracer.samplerParam)
-				assert.Equal(t, test.ExpectedSamplingServerURL, otel.samplerRemoteURL)
+				assert.Equal(t, test.ExpectedSamplingServerURL, tracer.samplerRemoteURL)
 			}
 		})
 	}
 }
 
 func TestInitSampler(t *testing.T) {
-	otel := &Opentelemetry{}
+	otel := &TracingService{}
 	sampler, err := otel.initSampler()
 	require.NoError(t, err)
 	assert.Equal(t, "AlwaysOffSampler", sampler.Description())

pkg/infra/usagestats/service/usage_stats.go

@@ -77,7 +77,7 @@ func (uss *UsageStats) runMetricsFunc(ctx context.Context, fn usagestats.Metrics
 	start := time.Now()
 	ctx, span := uss.tracer.Start(ctx, "UsageStats.Gather")
 	fnName := runtime.FuncForPC(reflect.ValueOf(fn).Pointer()).Name()
-	span.SetAttributes("usageStats.function", fnName, attribute.Key("usageStats.function").String(fnName))
+	span.SetAttributes(attribute.String("usageStats.function", fnName))
 	defer span.End()
 
 	fnMetrics, err := fn(ctx)

pkg/middleware/request_tracing.go

@@ -9,9 +9,9 @@ import (
 	"strings"
 
 	"go.opentelemetry.io/otel"
-	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/propagation"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -97,9 +97,9 @@ func RequestTracing(tracer tracing.Tracer) web.Middleware {
 
 			status := rw.Status()
 
-			span.SetAttributes("http.status_code", status, attribute.Int("http.status_code", status))
+			span.SetAttributes(semconv.HTTPStatusCode(status))
-			span.SetAttributes("http.url", req.RequestURI, attribute.String("http.url", req.RequestURI))
+			span.SetAttributes(semconv.HTTPURL(req.RequestURI))
-			span.SetAttributes("http.method", req.Method, attribute.String("http.method", req.Method))
+			span.SetAttributes(semconv.HTTPMethod(req.Method))
 			if status >= 400 {
 				span.SetStatus(codes.Error, fmt.Sprintf("error with HTTP status code %s", strconv.Itoa(status)))
 			}

pkg/plugins/backendplugin/coreplugin/registry.go

@@ -87,9 +87,8 @@ func ProvideCoreRegistry(tracer tracing.Tracer, am *azuremonitor.Service, cw *cl
 	es *elasticsearch.Service, grap *graphite.Service, idb *influxdb.Service, lk *loki.Service, otsdb *opentsdb.Service,
 	pr *prometheus.Service, t *tempo.Service, td *testdatasource.Service, pg *postgres.Service, my *mysql.Service,
 	ms *mssql.Service, graf *grafanads.Service, pyroscope *pyroscope.Service, parca *parca.Service) *Registry {
-	if otelTracer := tracer.OtelTracer(); otelTracer != nil {
+	// Non-optimal global solution to replace plugin SDK default tracer for core plugins.
-		sdktracing.InitDefaultTracer(otelTracer)
+	sdktracing.InitDefaultTracer(tracer)
-	}
 
 	return NewRegistry(map[string]backendplugin.PluginFactoryFunc{
 		CloudWatch:      asBackendPlugin(cw.Executor),

pkg/registry/backgroundsvcs/background_services.go

@@ -46,7 +46,7 @@ import (
 func ProvideBackgroundServiceRegistry(
 	httpServer *api.HTTPServer, ng *ngalert.AlertNG, cleanup *cleanup.CleanUpService, live *live.GrafanaLive,
 	pushGateway *pushhttp.Gateway, notifications *notifications.NotificationService, pluginStore *pluginStore.Service,
-	rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing tracing.Tracer,
+	rendering *rendering.RenderingService, tokenService auth.UserTokenBackgroundService, tracing *tracing.TracingService,
 	provisioning *provisioning.ProvisioningServiceImpl, alerting *alerting.AlertEngine, usageStats *uss.UsageStats,
 	statsCollector *statscollector.Service, grafanaUpdateChecker *updatechecker.GrafanaService,
 	pluginsUpdateChecker *updatechecker.PluginsService, metrics *metrics.InternalMetricsService,

pkg/server/wire.go

@@ -244,6 +244,7 @@ var wireBasicSet = wire.NewSet(
 	notifications.ProvideService,
 	notifications.ProvideSmtpService,
 	tracing.ProvideService,
+	wire.Bind(new(tracing.Tracer), new(*tracing.TracingService)),
 	metrics.ProvideService,
 	testdatasource.ProvideService,
 	ldapapi.ProvideService,

pkg/services/alerting/engine.go

@@ -9,6 +9,7 @@ import (
 	"github.com/benbjohnson/clock"
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/grafana/pkg/infra/localcache"
@@ -206,13 +207,8 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 		defer func() {
 			if err := recover(); err != nil {
 				e.log.Error("Alert Panic", "error", err, "stack", log.Stack(1))
+				span.SetStatus(codes.Error, "failed to execute alert rule. panic was recovered.")
 				span.RecordError(fmt.Errorf("%v", err))
-				span.AddEvents(
-					[]string{"error", "message"},
-					[]tracing.EventValue{
-						{Str: fmt.Sprintf("%v", err)},
-						{Str: "failed to execute alert rule. panic was recovered."},
-					})
 				span.End()
 				close(attemptChan)
 			}
@@ -220,20 +216,17 @@ func (e *AlertEngine) processJob(attemptID int, attemptChan chan int, cancelChan
 
 		e.evalHandler.Eval(evalContext)
 
-		span.SetAttributes("alertId", evalContext.Rule.ID, attribute.Key("alertId").Int64(evalContext.Rule.ID))
+		span.SetAttributes(
-		span.SetAttributes("dashboardId", evalContext.Rule.DashboardID, attribute.Key("dashboardId").Int64(evalContext.Rule.DashboardID))
+			attribute.Int64("alertId", evalContext.Rule.ID),
-		span.SetAttributes("firing", evalContext.Firing, attribute.Key("firing").Bool(evalContext.Firing))
+			attribute.Int64("dashboardId", evalContext.Rule.DashboardID),
-		span.SetAttributes("nodatapoints", evalContext.NoDataFound, attribute.Key("nodatapoints").Bool(evalContext.NoDataFound))
+			attribute.Bool("firing", evalContext.Firing),
-		span.SetAttributes("attemptID", attemptID, attribute.Key("attemptID").Int(attemptID))
+			attribute.Bool("nodatapoints", evalContext.NoDataFound),
+			attribute.Int("attemptID", attemptID),
+		)
 
 		if evalContext.Error != nil {
+			span.SetStatus(codes.Error, "alerting execution attempt failed")
 			span.RecordError(evalContext.Error)
-			span.AddEvents(
-				[]string{"error", "message"},
-				[]tracing.EventValue{
-					{Str: fmt.Sprintf("%v", evalContext.Error)},
-					{Str: "alerting execution attempt failed"},
-				})
 
 			if attemptID < setting.AlertingMaxAttempts {
 				span.End()

pkg/services/authn/authnimpl/service.go

@@ -8,6 +8,7 @@ import (
 
 	"github.com/prometheus/client_golang/prometheus"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/network"
@@ -262,9 +263,10 @@ func (s *Service) RegisterPostAuthHook(hook authn.PostAuthHookFn, priority uint)
 }
 
 func (s *Service) Login(ctx context.Context, client string, r *authn.Request) (identity *authn.Identity, err error) {
-	ctx, span := s.tracer.Start(ctx, "authn.Login")
+	ctx, span := s.tracer.Start(ctx, "authn.Login", trace.WithAttributes(
+		attribute.String(attributeKeyClient, client),
+	))
 	defer span.End()
-	span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))
 
 	defer func() {
 		for _, hook := range s.postLoginHooks.items {
@@ -316,9 +318,10 @@ func (s *Service) RegisterPostLoginHook(hook authn.PostLoginHookFn, priority uin
 }
 
 func (s *Service) RedirectURL(ctx context.Context, client string, r *authn.Request) (*authn.Redirect, error) {
-	ctx, span := s.tracer.Start(ctx, "authn.RedirectURL")
+	ctx, span := s.tracer.Start(ctx, "authn.RedirectURL", trace.WithAttributes(
+		attribute.String(attributeKeyClient, client),
+	))
 	defer span.End()
-	span.SetAttributes(attributeKeyClient, client, attribute.Key(attributeKeyClient).String(client))
 
 	c, ok := s.clients[client]
 	if !ok {

pkg/services/cleanup/cleanup.go

@@ -138,7 +138,7 @@ func (srv *CleanUpService) cleanUpTmpFiles(ctx context.Context) {
 
 	for _, f := range folders {
 		ctx, span := srv.tracer.Start(ctx, "delete stale files in temporary directory")
-		span.SetAttributes("directory", f, attribute.Key("directory").String(f))
+		span.SetAttributes(attribute.String("directory", f))
 		srv.cleanUpTmpFolder(ctx, f)
 		span.End()
 	}

pkg/services/contexthandler/contexthandler.go

@@ -18,6 +18,8 @@ import (
 	"github.com/grafana/grafana/pkg/services/user"
 	"github.com/grafana/grafana/pkg/setting"
 	"github.com/grafana/grafana/pkg/web"
+	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 )
 
 func ProvideService(cfg *setting.Cfg, tracer tracing.Tracer, features *featuremgmt.FeatureManager, authnService authn.Service,
@@ -125,13 +127,11 @@ func (h *ContextHandler) Middleware(next http.Handler) http.Handler {
 		}
 
 		reqContext.Logger = reqContext.Logger.New("userId", reqContext.UserID, "orgId", reqContext.OrgID, "uname", reqContext.Login)
-		span.AddEvents(
+		span.AddEvent("user", trace.WithAttributes(
-			[]string{"uname", "orgId", "userId"},
+			attribute.String("uname", reqContext.Login),
-			[]tracing.EventValue{
+			attribute.Int64("orgId", reqContext.OrgID),
-				{Str: reqContext.Login},
+			attribute.Int64("userId", reqContext.UserID),
-				{Num: reqContext.OrgID},
+		))
-				{Num: reqContext.UserID}},
-		)
 
 		next.ServeHTTP(w, r)
 	})

pkg/services/live/pipeline/pipeline.go

@@ -14,7 +14,7 @@ import (
 	"go.opentelemetry.io/otel/exporters/jaeger"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
-	semconv "go.opentelemetry.io/otel/semconv/v1.4.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/services/auth/identity"

pkg/services/ngalert/schedule/schedule.go

@@ -9,6 +9,8 @@ import (
 
 	"github.com/benbjohnson/clock"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 	"golang.org/x/sync/errgroup"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -372,7 +374,7 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 		notify(states)
 	}
 
-	evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span tracing.Span) {
+	evaluate := func(ctx context.Context, f fingerprint, attempt int64, e *evaluation, span trace.Span) {
 		logger := logger.New("version", e.rule.Version, "fingerprint", f, "attempt", attempt, "now", e.scheduledAt).FromContext(ctx)
 		start := sch.clock.Now()
 
@@ -406,21 +408,13 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 					}
 				}
 			}
+			span.SetStatus(codes.Error, "rule evaluation failed")
 			span.RecordError(err)
-			span.AddEvents(
-				[]string{"error", "message"},
-				[]tracing.EventValue{
-					{Str: fmt.Sprintf("%v", err)},
-					{Str: "rule evaluation failed"},
-				})
 		} else {
 			logger.Debug("Alert rule evaluated", "results", results, "duration", dur)
-			span.AddEvents(
+			span.AddEvent("rule evaluated", trace.WithAttributes(
-				[]string{"message", "results"},
+				attribute.Int64("results", int64(len(results))),
-				[]tracing.EventValue{
+			))
-					{Str: "rule evaluated"},
-					{Num: int64(len(results))},
-				})
 		}
 		if ctx.Err() != nil { // check if the context is not cancelled. The evaluation can be a long-running task.
 			logger.Debug("Skip updating the state because the context has been cancelled")
@@ -438,13 +432,10 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 
 		start = sch.clock.Now()
 		alerts := state.FromStateTransitionToPostableAlerts(processedStates, sch.stateManager, sch.appURL)
-		span.AddEvents(
+		span.AddEvent("results processed", trace.WithAttributes(
-			[]string{"message", "state_transitions", "alerts_to_send"},
+			attribute.Int64("state_transitions", int64(len(processedStates))),
-			[]tracing.EventValue{
+			attribute.Int64("alerts_to_send", int64(len(alerts.PostableAlerts))),
-				{Str: "results processed"},
+		))
-				{Num: int64(len(processedStates))},
-				{Num: int64(len(alerts.PostableAlerts))},
-			})
 		if len(alerts.PostableAlerts) > 0 {
 			sch.alertsSender.Send(key, alerts)
 		}
@@ -517,16 +508,17 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 						logger.Debug("Skip rule evaluation because it is paused")
 						return nil
 					}
-					tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution")
-					defer span.End()
 
-					span.SetAttributes("rule_uid", ctx.rule.UID, attribute.String("rule_uid", ctx.rule.UID))
-					span.SetAttributes("org_id", ctx.rule.OrgID, attribute.Int64("org_id", ctx.rule.OrgID))
-					span.SetAttributes("rule_version", ctx.rule.Version, attribute.Int64("rule_version", ctx.rule.Version))
 					fpStr := currentFingerprint.String()
-					span.SetAttributes("rule_fingerprint", fpStr, attribute.String("rule_fingerprint", fpStr))
 					utcTick := ctx.scheduledAt.UTC().Format(time.RFC3339Nano)
-					span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
+					tracingCtx, span := sch.tracer.Start(grafanaCtx, "alert rule execution", trace.WithAttributes(
+						attribute.String("rule_uid", ctx.rule.UID),
+						attribute.Int64("org_id", ctx.rule.OrgID),
+						attribute.Int64("rule_version", ctx.rule.Version),
+						attribute.String("rule_fingerprint", fpStr),
+						attribute.String("tick", utcTick),
+					))
+					defer span.End()
 
 					evaluate(tracingCtx, f, attempt, ctx, span)
 					return nil

pkg/services/ngalert/state/historian/annotation.go

@@ -11,10 +11,10 @@ import (
 
 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/annotations"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
@@ -73,7 +73,7 @@ func (h *AnnotationBackend) Record(ctx context.Context, rule history_model.RuleM
 	writeCtx := context.Background()
 	writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
 	writeCtx = history_model.WithRuleData(writeCtx, rule)
-	writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
+	writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))
 
 	go func(ctx context.Context) {
 		defer cancel()

pkg/services/ngalert/state/historian/loki.go

@@ -11,10 +11,10 @@ import (
 	"github.com/benbjohnson/clock"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"github.com/weaveworks/common/http/client"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/ngalert/eval"
 	"github.com/grafana/grafana/pkg/services/ngalert/metrics"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
@@ -89,7 +89,7 @@ func (h *RemoteLokiBackend) Record(ctx context.Context, rule history_model.RuleM
 	writeCtx := context.Background()
 	writeCtx, cancel := context.WithTimeout(writeCtx, StateHistoryWriteTimeout)
 	writeCtx = history_model.WithRuleData(writeCtx, rule)
-	writeCtx = tracing.ContextWithSpan(writeCtx, tracing.SpanFromContext(ctx))
+	writeCtx = trace.ContextWithSpan(writeCtx, trace.SpanFromContext(ctx))
 
 	go func(ctx context.Context) {
 		defer cancel()

pkg/services/ngalert/state/manager.go

@@ -9,6 +9,7 @@ import (
 	"github.com/grafana/dskit/concurrency"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -251,42 +252,33 @@ func (st *Manager) ResetStateByRuleUID(ctx context.Context, rule *ngModels.Alert
 // ProcessEvalResults updates the current states that belong to a rule with the evaluation results.
 // if extraLabels is not empty, those labels will be added to every state. The extraLabels take precedence over rule labels and result labels
 func (st *Manager) ProcessEvalResults(ctx context.Context, evaluatedAt time.Time, alertRule *ngModels.AlertRule, results eval.Results, extraLabels data.Labels) []StateTransition {
-	tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation")
-	defer span.End()
-	span.SetAttributes("rule_uid", alertRule.UID, attribute.String("rule_uid", alertRule.UID))
-	span.SetAttributes("org_id", alertRule.OrgID, attribute.Int64("org_id", alertRule.OrgID))
-	span.SetAttributes("rule_version", alertRule.Version, attribute.Int64("rule_version", alertRule.Version))
 	utcTick := evaluatedAt.UTC().Format(time.RFC3339Nano)
-	span.SetAttributes("tick", utcTick, attribute.String("tick", utcTick))
+	tracingCtx, span := st.tracer.Start(ctx, "alert rule state calculation", trace.WithAttributes(
-	span.SetAttributes("results", len(results), attribute.Int("tick", len(results)))
+		attribute.String("rule_uid", alertRule.UID),
+		attribute.Int64("org_id", alertRule.OrgID),
+		attribute.Int64("rule_version", alertRule.Version),
+		attribute.String("tick", utcTick),
+		attribute.Int("results", len(results))))
+	defer span.End()
 
 	logger := st.log.FromContext(tracingCtx)
 	logger.Debug("State manager processing evaluation results", "resultCount", len(results))
 	states := st.setNextStateForRule(tracingCtx, alertRule, results, extraLabels, logger)
-
+	span.AddEvent("results processed", trace.WithAttributes(
-	span.AddEvents([]string{"message", "state_transitions"},
+		attribute.Int64("state_transitions", int64(len(states))),
-		[]tracing.EventValue{
+	))
-			{Str: "results processed"},
-			{Num: int64(len(states))},
-		})
 
 	staleStates := st.deleteStaleStatesFromCache(ctx, logger, evaluatedAt, alertRule)
 	st.deleteAlertStates(tracingCtx, logger, staleStates)
 
 	if len(staleStates) > 0 {
-		span.AddEvents([]string{"message", "state_transitions"},
+		span.AddEvent("deleted stale states", trace.WithAttributes(
-			[]tracing.EventValue{
+			attribute.Int64("state_transitions", int64(len(staleStates))),
-				{Str: "deleted stale states"},
+		))
-				{Num: int64(len(staleStates))},
-			})
 	}
 
 	st.saveAlertStates(tracingCtx, logger, states...)
-
+	span.AddEvent("updated database")
-	span.AddEvents([]string{"message"},
-		[]tracing.EventValue{
-			{Str: "updated database"},
-		})
 
 	allChanges := append(states, staleStates...)
 	if st.historian != nil {

pkg/services/pluginsintegration/clientmiddleware/tracing_middleware.go

@@ -8,6 +8,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/plugins"
@@ -32,10 +33,10 @@ type TracingMiddleware struct {
 
 // setSpanAttributeFromHTTPHeader takes a ReqContext and a span, and adds the specified HTTP header as a span attribute
 // (string value), if the header is present.
-func setSpanAttributeFromHTTPHeader(headers http.Header, span tracing.Span, attributeName, headerName string) {
+func setSpanAttributeFromHTTPHeader(headers http.Header, span trace.Span, attributeName, headerName string) {
 	// Set the attribute as string
 	if v := headers.Get(headerName); v != "" {
-		span.SetAttributes(attributeName, v, attribute.Key(attributeName).String(v))
+		span.SetAttributes(attribute.String(attributeName, v))
 	}
 }
 
@@ -46,23 +47,24 @@ func (m *TracingMiddleware) traceWrap(
 	ctx context.Context, pluginContext backend.PluginContext, opName string,
 ) (context.Context, func(error)) {
 	// Start span
-	ctx, span := m.tracer.Start(ctx, "PluginClient."+opName)
+	ctx, span := m.tracer.Start(ctx, "PluginClient."+opName, trace.WithAttributes(
+		// Attach some plugin context information to span
+		attribute.String("plugin_id", pluginContext.PluginID),
+		attribute.Int64("org_id", pluginContext.OrgID),
+	))
 
-	// Attach some plugin context information to span
-	span.SetAttributes("plugin_id", pluginContext.PluginID, attribute.String("plugin_id", pluginContext.PluginID))
-	span.SetAttributes("org_id", pluginContext.OrgID, attribute.Int64("org_id", pluginContext.OrgID))
 	if settings := pluginContext.DataSourceInstanceSettings; settings != nil {
-		span.SetAttributes("datasource_name", settings.Name, attribute.Key("datasource_name").String(settings.Name))
+		span.SetAttributes(attribute.String("datasource_name", settings.Name))
-		span.SetAttributes("datasource_uid", settings.UID, attribute.Key("datasource_uid").String(settings.UID))
+		span.SetAttributes(attribute.String("datasource_uid", settings.UID))
 	}
 	if u := pluginContext.User; u != nil {
-		span.SetAttributes("user", u.Login, attribute.String("user", u.Login))
+		span.SetAttributes(attribute.String("user", u.Login))
 	}
 
 	// Additional attributes from http headers
 	if reqCtx := contexthandler.FromContext(ctx); reqCtx != nil && reqCtx.Req != nil && len(reqCtx.Req.Header) > 0 {
 		if v, err := strconv.Atoi(reqCtx.Req.Header.Get(query.HeaderPanelID)); err == nil {
-			span.SetAttributes("panel_id", v, attribute.Key("panel_id").Int(v))
+			span.SetAttributes(attribute.Int("panel_id", v))
 		}
 		setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "query_group_id", query.HeaderQueryGroupID)
 		setSpanAttributeFromHTTPHeader(reqCtx.Req.Header, span, "dashboard_uid", query.HeaderDashboardUID)

pkg/services/searchV2/index.go

@@ -15,6 +15,7 @@ import (
 
 	"github.com/blugelabs/bluge"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/db"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -466,8 +467,9 @@ func (i *searchIndex) reportSizeOfIndexDiskBackup(orgID int64) {
 }
 
 func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, error) {
-	spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex")
+	spanCtx, span := i.tracer.Start(ctx, "searchV2 buildOrgIndex", trace.WithAttributes(
-	span.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
+		attribute.Int64("org_id", orgID),
+	))
 
 	started := time.Now()
 	ctx, cancel := context.WithTimeout(spanCtx, time.Minute)
@@ -489,9 +491,10 @@ func (i *searchIndex) buildOrgIndex(ctx context.Context, orgID int64) (int, erro
 
 	dashboardExtender := i.extender.GetDashboardExtender(orgID)
 
-	_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index")
+	_, initOrgIndexSpan := i.tracer.Start(ctx, "searchV2 buildOrgIndex init org index", trace.WithAttributes(
-	initOrgIndexSpan.SetAttributes("org_id", orgID, attribute.Key("org_id").Int64(orgID))
+		attribute.Int64("org_id", orgID),
-	initOrgIndexSpan.SetAttributes("dashboardCount", len(dashboards), attribute.Key("dashboardCount").Int(len(dashboards)))
+		attribute.Int("dashboardCount", len(dashboards)),
+	))
 
 	index, err := initOrgIndex(dashboards, i.logger, dashboardExtender)
 
@@ -837,10 +840,11 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
 			default:
 			}
 
-			dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery")
+			dashboardQueryCtx, dashboardQuerySpan := l.tracer.Start(ctx, "sqlDashboardLoader dashboardQuery", trace.WithAttributes(
-			dashboardQuerySpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
+				attribute.Int64("orgID", orgID),
-			dashboardQuerySpan.SetAttributes("dashboardUID", dashboardUID, attribute.Key("dashboardUID").String(dashboardUID))
+				attribute.String("dashboardUID", dashboardUID),
-			dashboardQuerySpan.SetAttributes("lastID", lastID, attribute.Key("lastID").Int64(lastID))
+				attribute.Int64("lastID", lastID),
+			))
 
 			rows := make([]*dashboardQueryResult, 0)
 			err := l.sql.WithDbSession(dashboardQueryCtx, func(sess *db.Session) error {
@@ -887,9 +891,9 @@ func (l sqlDashboardLoader) loadAllDashboards(ctx context.Context, limit int, or
 }
 
 func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) {
-	ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards")
+	ctx, span := l.tracer.Start(ctx, "sqlDashboardLoader LoadDashboards", trace.WithAttributes(
-	span.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
+		attribute.Int64("orgID", orgID),
-
+	))
 	defer span.End()
 
 	var dashboards []dashboard
@@ -901,8 +905,9 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 		dashboards = make([]dashboard, 0, limit)
 	}
 
-	loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup")
+	loadDatasourceCtx, loadDatasourceSpan := l.tracer.Start(ctx, "sqlDashboardLoader LoadDatasourceLookup", trace.WithAttributes(
-	loadDatasourceSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
+		attribute.Int64("orgID", orgID),
+	))
 
 	// key will allow name or uid
 	lookup, err := kdash.LoadDatasourceLookup(loadDatasourceCtx, orgID, l.sql)
@@ -930,9 +935,10 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
 
 		rows := res.dashboards
 
-		_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard")
+		_, readDashboardSpan := l.tracer.Start(ctx, "sqlDashboardLoader readDashboard", trace.WithAttributes(
-		readDashboardSpan.SetAttributes("orgID", orgID, attribute.Key("orgID").Int64(orgID))
+			attribute.Int64("orgID", orgID),
-		readDashboardSpan.SetAttributes("dashboardCount", len(rows), attribute.Key("dashboardCount").Int(len(rows)))
+			attribute.Int("dashboardCount", len(rows)),
+		))
 
 		reader := kdash.NewStaticDashboardSummaryBuilder(lookup, false)
 

pkg/services/sqlstore/database_wrapper.go

@@ -13,6 +13,7 @@ import (
 	"github.com/lib/pq"
 	"github.com/mattn/go-sqlite3"
 	"github.com/prometheus/client_golang/prometheus"
+	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/trace"
 	"xorm.io/core"
 
@@ -100,10 +101,11 @@ func (h *databaseQueryWrapper) instrument(ctx context.Context, status string, qu
 	_, span := h.tracer.Start(ctx, "database query", trace.WithTimestamp(begin))
 	defer span.End()
 
-	span.AddEvents([]string{"query", "status"}, []tracing.EventValue{{Str: query}, {Str: status}})
+	span.AddEvent("query", trace.WithAttributes(attribute.String("query", query)))
+	span.AddEvent("status", trace.WithAttributes(attribute.String("status", status)))
 
 	if err != nil {
-		span.AddEvents([]string{"error"}, []tracing.EventValue{{Str: err.Error()}})
+		span.RecordError(err)
 	}
 
 	ctxLogger := h.log.FromContext(ctx)

pkg/services/sqlstore/session.go

@@ -9,6 +9,7 @@ import (
 
 	"github.com/mattn/go-sqlite3"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"xorm.io/xorm"
 
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -37,7 +38,7 @@ func (sess *DBSession) PublishAfterCommit(msg any) {
 	sess.events = append(sess.events, msg)
 }
 
-func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, tracing.Span, error) {
+func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTran bool, tracer tracing.Tracer) (*DBSession, bool, trace.Span, error) {
 	value := ctx.Value(ContextSessionKey{})
 	var sess *DBSession
 	sess, ok := value.(*DBSession)
@@ -50,7 +51,7 @@ func startSessionOrUseExisting(ctx context.Context, engine *xorm.Engine, beginTr
 	}
 
 	tctx, span := tracer.Start(ctx, "open session")
-	span.SetAttributes("transaction", beginTran, attribute.Key("transaction").Bool(beginTran))
+	span.SetAttributes(attribute.Bool("transaction", beginTran))
 
 	newSess := &DBSession{Session: engine.NewSession(), transactionOpen: beginTran}
 

pkg/tsdb/azuremonitor/loganalytics/azure-log-analytics-datasource.go

@@ -19,6 +19,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 	"k8s.io/utils/strings/slices"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -283,13 +284,13 @@ func (e *AzureLogAnalyticsDatasource) executeQuery(ctx context.Context, query *A
 		return nil, err
 	}
 
-	ctx, span := tracer.Start(ctx, "azure log analytics query")
+	ctx, span := tracer.Start(ctx, "azure log analytics query", trace.WithAttributes(
-	span.SetAttributes("target", query.Query, attribute.Key("target").String(query.Query))
+		attribute.String("target", query.Query),
-	span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
+		attribute.Int64("datasource_id", dsInfo.DatasourceID),
-	span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
+		attribute.Int64("org_id", dsInfo.OrgID),
-
+	))
 	defer span.End()
 
 	tracer.Inject(ctx, req.Header, span)
@@ -559,11 +560,11 @@ func getCorrelationWorkspaces(ctx context.Context, baseResource string, resource
 		req.URL.RawQuery = values.Encode()
 		req.Method = "GET"
 
-		ctx, span := tracer.Start(ctx, "azure traces correlation request")
+		ctx, span := tracer.Start(ctx, "azure traces correlation request", trace.WithAttributes(
-		span.SetAttributes("target", req.URL, attribute.Key("target").String(req.URL.String()))
+			attribute.String("target", req.URL.String()),
-		span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
+			attribute.Int64("datasource_id", dsInfo.DatasourceID),
-		span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
+			attribute.Int64("org_id", dsInfo.OrgID),
-
+		))
 		defer span.End()
 
 		tracer.Inject(ctx, req.Header, span)

pkg/tsdb/azuremonitor/metrics/azuremonitor-datasource.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/services/featuremgmt"
@@ -252,12 +253,13 @@ func (e *AzureMonitorDatasource) retrieveSubscriptionDetails(cli *http.Client, c
 	values.Add("api-version", "2022-12-01")
 	req.URL.RawQuery = values.Encode()
 
-	ctx, span := tracer.Start(ctx, "azuremonitor query")
+	ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
-	span.SetAttributes("subscription", subscriptionId, attribute.Key("subscription").String(subscriptionId))
+		attribute.String("subscription", subscriptionId),
-	span.SetAttributes("datasource_id", dsId, attribute.Key("datasource_id").Int64(dsId))
+		attribute.Int64("datasource_id", dsId),
-	span.SetAttributes("org_id", orgId, attribute.Key("org_id").Int64(orgId))
+		attribute.Int64("org_id", orgId),
-
+	))
 	defer span.End()
+
 	tracer.Inject(ctx, req.Header, span)
 	res, err := cli.Do(req)
 	if err != nil {
@@ -302,14 +304,15 @@ func (e *AzureMonitorDatasource) executeQuery(ctx context.Context, query *types.
 		req.Body = io.NopCloser(strings.NewReader(fmt.Sprintf(`{"filter": "%s"}`, query.BodyFilter)))
 	}
 
-	ctx, span := tracer.Start(ctx, "azuremonitor query")
+	ctx, span := tracer.Start(ctx, "azuremonitor query", trace.WithAttributes(
-	span.SetAttributes("target", query.Target, attribute.Key("target").String(query.Target))
+		attribute.String("target", query.Target),
-	span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
+		attribute.Int64("datasource_id", dsInfo.DatasourceID),
-	span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
+		attribute.Int64("org_id", dsInfo.OrgID),
-
+	))
 	defer span.End()
+
 	tracer.Inject(ctx, req.Header, span)
 
 	res, err := cli.Do(req)

pkg/tsdb/azuremonitor/resourcegraph/azure-resource-graph-datasource.go

@@ -14,6 +14,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
 	"github.com/grafana/grafana/pkg/tsdb/azuremonitor/kinds/dataquery"
@@ -150,13 +151,13 @@ func (e *AzureResourceGraphDatasource) executeQuery(ctx context.Context, query *
 	req.URL.Path = path.Join(req.URL.Path, argQueryProviderName)
 	req.URL.RawQuery = params.Encode()
 
-	ctx, span := tracer.Start(ctx, "azure resource graph query")
+	ctx, span := tracer.Start(ctx, "azure resource graph query", trace.WithAttributes(
-	span.SetAttributes("interpolated_query", query.InterpolatedQuery, attribute.Key("interpolated_query").String(query.InterpolatedQuery))
+		attribute.String("interpolated_query", query.InterpolatedQuery),
-	span.SetAttributes("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond), attribute.Key("from").Int64(query.TimeRange.From.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("from", query.TimeRange.From.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond), attribute.Key("until").Int64(query.TimeRange.To.UnixNano()/int64(time.Millisecond)))
+		attribute.Int64("until", query.TimeRange.To.UnixNano()/int64(time.Millisecond)),
-	span.SetAttributes("datasource_id", dsInfo.DatasourceID, attribute.Key("datasource_id").Int64(dsInfo.DatasourceID))
+		attribute.Int64("datasource_id", dsInfo.DatasourceID),
-	span.SetAttributes("org_id", dsInfo.OrgID, attribute.Key("org_id").Int64(dsInfo.OrgID))
+		attribute.Int64("org_id", dsInfo.OrgID),
-
+	))
 	defer span.End()
 
 	tracer.Inject(ctx, req.Header, span)

pkg/tsdb/cloud-monitoring/utils.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -124,13 +125,14 @@ func doRequestWithPagination(ctx context.Context, logger log.Logger, r *http.Req
 	return d, nil
 }
 
-func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) tracing.Span {
+func traceReq(ctx context.Context, tracer tracing.Tracer, req *backend.QueryDataRequest, dsInfo datasourceInfo, r *http.Request, target string) trace.Span {
-	ctx, span := tracer.Start(ctx, "cloudMonitoring query")
+	ctx, span := tracer.Start(ctx, "cloudMonitoring query", trace.WithAttributes(
-	span.SetAttributes("target", target, attribute.Key("target").String(target))
+		attribute.String("target", target),
-	span.SetAttributes("from", req.Queries[0].TimeRange.From, attribute.Key("from").String(req.Queries[0].TimeRange.From.String()))
+		attribute.String("from", req.Queries[0].TimeRange.From.String()),
-	span.SetAttributes("until", req.Queries[0].TimeRange.To, attribute.Key("until").String(req.Queries[0].TimeRange.To.String()))
+		attribute.String("until", req.Queries[0].TimeRange.To.String()),
-	span.SetAttributes("datasource_id", dsInfo.id, attribute.Key("datasource_id").Int64(dsInfo.id))
+		attribute.Int64("datasource_id", dsInfo.id),
-	span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
+		attribute.Int64("org_id", req.PluginContext.OrgID),
+	))
 	tracer.Inject(ctx, r.Header, span)
 	return span
 }

pkg/tsdb/elasticsearch/client/client.go

@@ -16,6 +16,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -171,9 +172,10 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
 	var err error
 	multiRequests := c.createMultiSearchRequests(r.Requests)
 	queryParams := c.getMultiSearchQueryParameters()
-	_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch")
+	_, span := c.tracer.Start(c.ctx, "datasource.elasticsearch.queryData.executeMultisearch", trace.WithAttributes(
-	span.SetAttributes("queryParams", queryParams, attribute.Key("queryParams").String(queryParams))
+		attribute.String("queryParams", queryParams),
-	span.SetAttributes("url", c.ds.URL, attribute.Key("url").String(c.ds.URL))
+		attribute.String("url", c.ds.URL),
+	))
 	defer func() {
 		if err != nil {
 			span.RecordError(err)

pkg/tsdb/elasticsearch/response_parser.go

@@ -15,6 +15,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/components/simplejson"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -52,13 +53,15 @@ func parseResponse(ctx context.Context, responses []*es.SearchResponse, targets
 	if responses == nil {
 		return &result, nil
 	}
-	ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse")
+	ctx, span := tracer.Start(ctx, "datasource.elastic.parseResponse", trace.WithAttributes(
-	span.SetAttributes("responseLength", len(responses), attribute.Key("responseLength").Int(len(responses)))
+		attribute.Int("responseLength", len(responses)),
+	))
 	defer span.End()
 
 	for i, res := range responses {
-		_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response")
+		_, resSpan := tracer.Start(ctx, "datasource.elastic.parseResponse.response", trace.WithAttributes(
-		resSpan.SetAttributes("queryMetricType", targets[i].Metrics[0].Type, attribute.Key("queryMetricType").String(targets[i].Metrics[0].Type))
+			attribute.String("queryMetricType", targets[i].Metrics[0].Type),
+		))
 		start := time.Now()
 		target := targets[i]
 

pkg/tsdb/graphite/graphite.go

@@ -143,17 +143,18 @@ func (s *Service) QueryData(ctx context.Context, req *backend.QueryDataRequest)
 	defer span.End()
 
 	targetStr := strings.Join(formData["target"], ",")
-	span.SetAttributes("target", targetStr, attribute.Key("target").String(targetStr))
+	span.SetAttributes(
-	span.SetAttributes("from", from, attribute.Key("from").String(from))
+		attribute.String("target", targetStr),
-	span.SetAttributes("until", until, attribute.Key("until").String(until))
+		attribute.String("from", from),
-	span.SetAttributes("datasource_id", dsInfo.Id, attribute.Key("datasource_id").Int64(dsInfo.Id))
+		attribute.String("until", until),
-	span.SetAttributes("org_id", req.PluginContext.OrgID, attribute.Key("org_id").Int64(req.PluginContext.OrgID))
+		attribute.Int64("datasource_id", dsInfo.Id),
-
+		attribute.Int64("org_id", req.PluginContext.OrgID),
+	)
 	s.tracer.Inject(ctx, graphiteReq.Header, span)
 
 	res, err := dsInfo.HTTPClient.Do(graphiteReq)
 	if res != nil {
-		span.SetAttributes("graphite.response.code", res.StatusCode, attribute.Key("graphite.response.code").Int(res.StatusCode))
+		span.SetAttributes(attribute.Int("graphite.response.code", res.StatusCode))
 	}
 	if err != nil {
 		span.RecordError(err)

pkg/tsdb/loki/api.go

@@ -17,6 +17,7 @@ import (
 	jsoniter "github.com/json-iterator/go"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/log"
 	"github.com/grafana/grafana/pkg/infra/tracing"
@@ -197,8 +198,9 @@ func (api *LokiAPI) DataQuery(ctx context.Context, query lokiQuery, responseOpts
 	}
 
 	start = time.Now()
-	_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse")
+	_, span := api.tracer.Start(ctx, "datasource.loki.parseResponse", trace.WithAttributes(
-	span.SetAttributes("metricDataplane", responseOpts.metricDataplane, attribute.Key("metricDataplane").Bool(responseOpts.metricDataplane))
+		attribute.Bool("metricDataplane", responseOpts.metricDataplane),
+	))
 	defer span.End()
 
 	iter := jsoniter.Parse(jsoniter.ConfigDefault, resp.Body, 1024)

pkg/tsdb/loki/loki.go

@@ -17,6 +17,7 @@ import (
 	"github.com/grafana/grafana-plugin-sdk-go/data"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/httpclient"
 	"github.com/grafana/grafana/pkg/infra/log"
@@ -134,8 +135,9 @@ func callResource(ctx context.Context, req *backend.CallResourceRequest, sender
 	}
 	lokiURL := fmt.Sprintf("/loki/api/v1/%s", url)
 
-	ctx, span := tracer.Start(ctx, "datasource.loki.CallResource")
+	ctx, span := tracer.Start(ctx, "datasource.loki.CallResource", trace.WithAttributes(
-	span.SetAttributes("url", lokiURL, attribute.Key("url").String(lokiURL))
+		attribute.String("url", lokiURL),
+	))
 	defer span.End()
 
 	api := newLokiAPI(dsInfo.HTTPClient, dsInfo.URL, plog, tracer)
@@ -192,11 +194,12 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
 
 	plog.Info("Prepared request to Loki", "duration", time.Since(start), "queriesLength", len(queries), "stage", stagePrepareRequest, "runInParallel", runInParallel)
 
-	ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries")
+	ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries", trace.WithAttributes(
-	span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
+		attribute.Bool("runInParallel", runInParallel),
-	span.SetAttributes("queriesLength", len(queries), attribute.Key("queriesLength").Int((len(queries))))
+		attribute.Int("queriesLength", len(queries)),
+	))
 	if req.GetHTTPHeader("X-Query-Group-Id") != "" {
-		span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
+		span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
 	}
 	defer span.End()
 	start = time.Now()
@@ -224,13 +227,14 @@ func queryData(ctx context.Context, req *backend.QueryDataRequest, dsInfo *datas
 }
 
 func executeQuery(ctx context.Context, query *lokiQuery, req *backend.QueryDataRequest, runInParallel bool, api *LokiAPI, responseOpts ResponseOpts, tracer tracing.Tracer, plog log.Logger) backend.DataResponse {
-	ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery")
+	ctx, span := tracer.Start(ctx, "datasource.loki.queryData.runQueries.runQuery", trace.WithAttributes(
-	span.SetAttributes("runInParallel", runInParallel, attribute.Key("runInParallel").Bool(runInParallel))
+		attribute.Bool("runInParallel", runInParallel),
-	span.SetAttributes("expr", query.Expr, attribute.Key("expr").String(query.Expr))
+		attribute.String("expr", query.Expr),
-	span.SetAttributes("start_unixnano", query.Start, attribute.Key("start_unixnano").Int64(query.Start.UnixNano()))
+		attribute.Int64("start_unixnano", query.Start.UnixNano()),
-	span.SetAttributes("stop_unixnano", query.End, attribute.Key("stop_unixnano").Int64(query.End.UnixNano()))
+		attribute.Int64("stop_unixnano", query.End.UnixNano()),
+	))
 	if req.GetHTTPHeader("X-Query-Group-Id") != "" {
-		span.SetAttributes("query_group_id", req.GetHTTPHeader("X-Query-Group-Id"), attribute.Key("query_group_id").String(req.GetHTTPHeader("X-Query-Group-Id")))
+		span.SetAttributes(attribute.String("query_group_id", req.GetHTTPHeader("X-Query-Group-Id")))
 	}
 
 	defer span.End()

pkg/tsdb/prometheus/querydata/request.go

@@ -212,9 +212,9 @@ func (s *QueryData) exemplarQuery(ctx context.Context, c *client.Client, q *mode
 }
 
 func (s *QueryData) trace(ctx context.Context, q *models.Query) (context.Context, func()) {
-	return utils.StartTrace(ctx, s.tracer, "datasource.prometheus", []utils.Attribute{
+	return utils.StartTrace(ctx, s.tracer, "datasource.prometheus",
-		{Key: "expr", Value: q.Expr, Kv: attribute.Key("expr").String(q.Expr)},
+		attribute.String("expr", q.Expr),
-		{Key: "start_unixnano", Value: q.Start, Kv: attribute.Key("start_unixnano").Int64(q.Start.UnixNano())},
+		attribute.Int64("start_unixnano", q.Start.UnixNano()),
-		{Key: "stop_unixnano", Value: q.End, Kv: attribute.Key("stop_unixnano").Int64(q.End.UnixNano())},
+		attribute.Int64("stop_unixnano", q.End.UnixNano()),
-	})
+	)
 }

pkg/tsdb/prometheus/querydata/response.go

@@ -25,7 +25,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
 		}
 	}()
 
-	ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse", []utils.Attribute{})
+	ctx, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.parseResponse")
 	defer endSpan()
 
 	iter := jsoniter.Parse(jsoniter.ConfigDefault, res.Body, 1024)
@@ -54,7 +54,7 @@ func (s *QueryData) parseResponse(ctx context.Context, q *models.Query, res *htt
 }
 
 func (s *QueryData) processExemplars(ctx context.Context, q *models.Query, dr backend.DataResponse) backend.DataResponse {
-	_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars", []utils.Attribute{})
+	_, endSpan := utils.StartTrace(ctx, s.tracer, "datasource.prometheus.processExemplars")
 	defer endSpan()
 	sampler := s.exemplarSampler()
 	labelTracker := exemplar.NewLabelTracker()

pkg/tsdb/prometheus/utils/utils.go

@@ -7,6 +7,7 @@ import (
 
 	"github.com/grafana/grafana-plugin-sdk-go/backend"
 	"go.opentelemetry.io/otel/attribute"
+	"go.opentelemetry.io/otel/trace"
 
 	"github.com/grafana/grafana/pkg/infra/tracing"
 )
@@ -22,21 +23,12 @@ func GetJsonData(settings backend.DataSourceInstanceSettings) (map[string]any, e
 	return jsonData, nil
 }
 
-type Attribute struct {
-	Key   string
-	Value any
-	Kv    attribute.KeyValue
-}
-
 // StartTrace setups a trace but does not panic if tracer is nil which helps with testing
-func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes []Attribute) (context.Context, func()) {
+func StartTrace(ctx context.Context, tracer tracing.Tracer, name string, attributes ...attribute.KeyValue) (context.Context, func()) {
 	if tracer == nil {
 		return ctx, func() {}
 	}
-	ctx, span := tracer.Start(ctx, name)
+	ctx, span := tracer.Start(ctx, name, trace.WithAttributes(attributes...))
-	for _, attr := range attributes {
-		span.SetAttributes(attr.Key, attr.Value, attr.Kv)
-	}
 	return ctx, func() {
 		span.End()
 	}

pkg/tsdb/tempo/trace_transform.go

@@ -10,7 +10,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/ptrace"
 	"go.opentelemetry.io/otel/attribute"
-	semconv "go.opentelemetry.io/otel/semconv/v1.15.0"
+	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 )
 
 type KeyValue struct {

pkg/util/proxyutil/reverse_proxy_test.go

@@ -180,7 +180,8 @@ func TestReverseProxy(t *testing.T) {
 			{status: 599, expectedSource: requestmeta.StatusSourceDownstream},
 		}
 
-		for _, tc := range testCases {
+		for _, testCase := range testCases {
+			tc := testCase
 			t.Run(fmt.Sprintf("status %d => source %s ", tc.status, tc.expectedSource), func(t *testing.T) {
 				upstream := newUpstreamServer(t, http.HandlerFunc(func(w http.ResponseWriter, req *http.Request) {
 					w.WriteHeader(tc.status)