Add OTLP exporter for OpenTelemetry (#47987)

* Add OTLP exporter for OpenTelemtry

* Fix lint

* Refactore parse settings

* Add configuration for propagation + fix tests

* Fix tests and lint

* Fix alerting tests

* Add coments to config

* Add propagation to custom.ini

conf/defaults.ini

@@ -975,7 +975,16 @@ disable_shared_zipkin_spans = false
 
 [tracing.opentelemetry.jaeger]
 # jaeger destination (ex http://localhost:14268/api/traces)
-address =
+address = 
+# Propagation specifies the text map propagation format: w3c, jaeger
+propagation = 
+
+# This is a configuration for OTLP exporter with GRPC protocol
+[tracing.opentelemetry.otlp]
+# otlp destination (ex localhost:4317) 
+address = 
+# Propagation specifies the text map propagation format: w3c, jaeger
+propagation = 
 
 #################################### External Image Storage ##############
 [external_image_storage]

conf/sample.ini

@@ -958,6 +958,15 @@
 [tracing.opentelemetry.jaeger]
 # jaeger destination (ex http://localhost:14268/api/traces)
 ; address = http://localhost:14268/api/traces
+# Propagation specifies the text map propagation format: w3c, jaeger
+; propagation = jaeger
+
+# This is a configuration for OTLP exporter with GRPC protocol
+[tracing.opentelemetry.otlp]
+# otlp destination (ex localhost:4317) 
+; address = localhost:4317
+# Propagation specifies the text map propagation format: w3c, jaeger
+; propagation = w3c
 
 #################################### External image storage ##########################
 [external_image_storage]

go.mod

@@ -99,7 +99,7 @@ require (
 	go.opentelemetry.io/collector/model v0.31.0
 	go.opentelemetry.io/otel v1.6.3
 	go.opentelemetry.io/otel/exporters/jaeger v1.0.0
-	go.opentelemetry.io/otel/sdk v1.3.0
+	go.opentelemetry.io/otel/sdk v1.6.3
 	go.opentelemetry.io/otel/trace v1.6.3
 	golang.org/x/crypto v0.0.0-20220331220935-ae2d96664a29
 	golang.org/x/exp v0.0.0-20210220032938-85be41e4509f
@@ -172,7 +172,7 @@ require (
 	github.com/gogo/status v1.1.0 // indirect
 	github.com/golang-jwt/jwt/v4 v4.4.1 // indirect
 	github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe // indirect
-	github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b // indirect
+	github.com/golang/glog v1.0.0 // indirect
 	github.com/golang/groupcache v0.0.0-20210331224755-41bb18bfe9da // indirect
 	github.com/golang/protobuf v1.5.2 // indirect
 	github.com/gomodule/redigo v2.0.0+incompatible // indirect
@@ -249,6 +249,10 @@ require (
 	github.com/golang-migrate/migrate/v4 v4.7.0
 	github.com/grafana/dskit v0.0.0-20211011144203-3a88ec0b675f
 	github.com/grafana/thema v0.0.0-20220413232647-fc54c169b508
+	go.etcd.io/etcd v3.3.25+incompatible
+	go.opentelemetry.io/contrib/propagators/jaeger v1.6.0
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3
+	go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3
 	gocloud.dev v0.25.0
 )
 
@@ -272,12 +276,14 @@ require (
 	github.com/Microsoft/go-winio v0.5.2 // indirect
 	github.com/chromedp/cdproto v0.0.0-20220208224320-6efb837e6bc2 // indirect
 	github.com/containerd/containerd v1.6.2 // indirect
+	github.com/coreos/go-semver v0.3.0 // indirect
 	github.com/elazarl/goproxy v0.0.0-20220115173737-adb46da277ac // indirect
 	github.com/getkin/kin-openapi v0.94.0 // indirect
 	github.com/ghodss/yaml v1.0.1-0.20190212211648-25d852aebe32 // indirect
 	github.com/go-logr/logr v1.2.3 // indirect
 	github.com/go-logr/stdr v1.2.2 // indirect
 	github.com/golang-jwt/jwt v3.2.2+incompatible // indirect
+	github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 // indirect
 	github.com/imdario/mergo v0.3.12 // indirect
 	github.com/klauspost/compress v1.15.1 // indirect
 	github.com/kylelemons/godebug v1.1.0 // indirect
@@ -285,6 +291,8 @@ require (
 	github.com/pierrec/lz4/v4 v4.1.8 // indirect
 	github.com/segmentio/asm v1.1.1 // indirect
 	github.com/yudai/pp v2.0.1+incompatible // indirect
+	go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 // indirect
+	go.opentelemetry.io/proto/otlp v0.15.0 // indirect
 	k8s.io/api v0.22.5 // indirect
 	k8s.io/apimachinery v0.22.5 // indirect
 	k8s.io/klog/v2 v2.30.0 // indirect

go.sum

@@ -680,6 +680,7 @@ github.com/coreos/go-iptables v0.6.0/go.mod h1:Qe8Bv2Xik5FyTXwgIbLAnv2sWSBmvWdFE
 github.com/coreos/go-oidc v2.1.0+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-oidc v2.2.1+incompatible/go.mod h1:CgnwVTmzoESiwO9qyAFEMiHoZ1nMCKZlZ9V6mm3/LKc=
 github.com/coreos/go-semver v0.2.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
+github.com/coreos/go-semver v0.3.0 h1:wkHLiw0WNATZnSG7epLsujiMCgPAc9xhjJ4tgnAxmfM=
 github.com/coreos/go-semver v0.3.0/go.mod h1:nnelYz7RCh+5ahJtPPxZlU+153eP4D4r3EedlOD2RNk=
 github.com/coreos/go-systemd v0.0.0-20161114122254-48702e0da86b/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
 github.com/coreos/go-systemd v0.0.0-20180511133405-39ca1b05acc7/go.mod h1:F5haX7vjVVG0kc13fIWeqUViNPyEJxv/OmvnBo0Yme4=
@@ -1216,8 +1217,9 @@ github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe h1:lXe2qZdvpiX5WZ
 github.com/golang-sql/civil v0.0.0-20190719163853-cb61b32ac6fe/go.mod h1:8vg3r2VgvsThLBIFL93Qb5yWzgyZWhEmBwUJWevAkK0=
 github.com/golang/freetype v0.0.0-20170609003504-e2365dfdc4a0/go.mod h1:E/TSTwGwJL78qG/PmXZO1EjYhfJinVAhrmmHX6Z8B9k=
 github.com/golang/geo v0.0.0-20190916061304-5b978397cfec/go.mod h1:QZ0nwyI2jOfgRAoBvP+ab5aRr7c9x7lhGEJrKvBwjWI=
-github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b h1:VKtxabqXZkF25pY9ekfRL6a582T4P37/31XEstQ5p58=
 github.com/golang/glog v0.0.0-20160126235308-23def4e6c14b/go.mod h1:SBH7ygxi8pfUlaOkMMuAQtPIUF8ecWP5IEl/CR7VP2Q=
+github.com/golang/glog v1.0.0 h1:nfP3RFugxnNRyKgeWd4oI1nYvXpxrx8ck8ZrcizshdQ=
+github.com/golang/glog v1.0.0/go.mod h1:EWib/APOK0SL3dFbYqvxE3UYd8E6s1ouQ7iEp/0LWV4=
 github.com/golang/groupcache v0.0.0-20160516000752-02826c3e7903/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190129154638-5b532d6fd5ef/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
 github.com/golang/groupcache v0.0.0-20190702054246-869f871628b6/go.mod h1:cIg4eruTrX1D+g88fzRXU5OdNfaM+9IcxsU14FzY7Hc=
@@ -1455,7 +1457,10 @@ github.com/grpc-ecosystem/grpc-gateway v1.12.1/go.mod h1:8XEsbTttt/W+VvjtQhLACqC
 github.com/grpc-ecosystem/grpc-gateway v1.14.4/go.mod h1:6CwZWGDSPRJidgKAtJVvND6soZe6fT7iteq8wDPdhb0=
 github.com/grpc-ecosystem/grpc-gateway v1.14.6/go.mod h1:zdiPV4Yse/1gnckTHtghG4GkDEdKCRJduHpTxT3/jcw=
 github.com/grpc-ecosystem/grpc-gateway v1.15.0/go.mod h1:vO11I9oWA+KsxmfFQPhLnnIb1VDE24M+pdxZFiuZcA8=
+github.com/grpc-ecosystem/grpc-gateway v1.16.0 h1:gmcG1KaJ57LophUzW0Hy8NmPhnMZb4M0+kPpLofRdBo=
 github.com/grpc-ecosystem/grpc-gateway v1.16.0/go.mod h1:BDjrQk3hbvj6Nolgz8mAMFbcEtjT1g+wF4CSlocrBnw=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0 h1:BZHcxBETFHIdVyhyEfOvn/RdU/QGdLI4y34qQGjGWO0=
+github.com/grpc-ecosystem/grpc-gateway/v2 v2.7.0/go.mod h1:hgWBS7lorOAVIJEQMi4ZsPv9hVvWI6+ch50m39Pf2Ks=
 github.com/grpc-ecosystem/grpc-opentracing v0.0.0-20180507213350-8e809c8a8645/go.mod h1:6iZfnjpejD4L/4DwD7NryNaJyCQdzwWwH2MWhCA90Kw=
 github.com/hailocab/go-hostpool v0.0.0-20160125115350-e80d13ce29ed/go.mod h1:tMWxXQ9wFIaZeTI9F+hmhFiGpFmhOHzyShyFUhRm0H4=
 github.com/hanwen/go-fuse v1.0.0/go.mod h1:unqXarDXqzAk0rt98O2tVndEPIpUgLD9+rwFisZH3Ok=
@@ -2721,6 +2726,7 @@ go.etcd.io/etcd v0.0.0-20190709142735-eb7dd97135a5/go.mod h1:N0RPWo9FXJYZQI4BTkD
 go.etcd.io/etcd v0.0.0-20191023171146-3cf2f69b5738/go.mod h1:dnLIgRNXwCJa5e+c6mIZCrds/GIG4ncV9HhK5PX7jPg=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200520232829-54ba9589114f/go.mod h1:skWido08r9w6Lq/w70DO5XYIKMu4QFu1+4VsqLQuJy8=
 go.etcd.io/etcd v0.5.0-alpha.5.0.20200910180754-dd1b699fc489/go.mod h1:yVHk9ub3CSBatqGNg7GRmsnfLWtoW60w4eDYfh7vHDg=
+go.etcd.io/etcd v3.3.25+incompatible h1:V1RzkZJj9LqsJRy+TUBgpWSbZXITLB819lstuTFoZOY=
 go.etcd.io/etcd v3.3.25+incompatible/go.mod h1:yaeTdrJi5lOmYerz05bd8+V7KubZs8YSFZfzsF9A6aI=
 go.etcd.io/etcd/api/v3 v3.5.0-alpha.0/go.mod h1:mPcW6aZJukV6Aa81LSKpBjQXTWlXB5r74ymPoSWa3Sw=
 go.etcd.io/etcd/api/v3 v3.5.0/go.mod h1:cbVKeC6lCfl7j/8jBhAK6aIYO9XOjdptoxU/nLQcPvs=
@@ -2771,26 +2777,37 @@ go.opentelemetry.io/collector v0.31.0/go.mod h1:A9vKmEa2MI/vJXNUoRinq9w25ZMmxWJL
 go.opentelemetry.io/collector/model v0.31.0 h1:IgMOkSBd/n/gV4EQQ1nJ+/ylddOlqTfMGjku91yC0d8=
 go.opentelemetry.io/collector/model v0.31.0/go.mod h1:PcHNnM+RUl0uD8VkSn93PO78N7kQYhfqpI/eki57pl4=
 go.opentelemetry.io/contrib v0.20.0/go.mod h1:G/EtFaa6qaN7+LxqfIAT3GiZa7Wv5DTBUzl5H4LY0Kc=
+go.opentelemetry.io/contrib v0.21.0 h1:RMJ6GlUVzLYp/zmItxTTdAmr1gnpO/HHMFmvjAhvJQM=
 go.opentelemetry.io/contrib v0.21.0/go.mod h1:EH4yDYeNoaTqn/8yCWQmfNB78VHfGX2Jt2bvnvzBlGM=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.20.0/go.mod h1:oVGt1LRbBOBq1A5BQLlUg9UaU/54aiHw8cgjV3aWZ/E=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.21.0/go.mod h1:Vm5u/mtkj1OMhtao0v+BGo2LUoLCgHYXvRmj0jWITlE=
 go.opentelemetry.io/contrib/instrumentation/google.golang.org/grpc/otelgrpc v0.28.0/go.mod h1:vEhqr0m4eTc+DWxfsXoXue2GBgV2uUwVznkGIHW/e5w=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.20.0/go.mod h1:2AboqHi0CiIZU0qwhtUfCYD1GeUzvvIXWNkhDt7ZMG4=
 go.opentelemetry.io/contrib/instrumentation/net/http/otelhttp v0.21.0/go.mod h1:JQAtechjxLEL81EjmbRwxBq/XEzGaHcsPuDHAx54hg4=
+go.opentelemetry.io/contrib/propagators/jaeger v1.6.0 h1:tCc+sWgHVeOMp4zmUxHHTaoA5vQlGO089zfg97d+BvU=
+go.opentelemetry.io/contrib/propagators/jaeger v1.6.0/go.mod h1:cqu1XdBYBXqXHxZLJdK00G9rT5Hda7Fa938I8LVYz/Y=
 go.opentelemetry.io/contrib/zpages v0.0.0-20210722161726-7668016acb73/go.mod h1:NAkejuYm41lpyL43Fu1XdnCOYxN5NVV80/MJ03JQ/X8=
 go.opentelemetry.io/otel v0.11.0/go.mod h1:G8UCk+KooF2HLkgo8RHX9epABH/aRGYET7gQOqBVdB0=
 go.opentelemetry.io/otel v0.20.0/go.mod h1:Y3ugLH2oa81t5QO+Lty+zXf8zC9L26ax4Nzoxm/dooo=
 go.opentelemetry.io/otel v1.0.0-RC1/go.mod h1:x9tRa9HK4hSSq7jf2TKbqFbtt58/TGk0f9XiEYISI1I=
 go.opentelemetry.io/otel v1.0.0/go.mod h1:AjRVh9A5/5DE7S+mZtTR6t8vpKKryam+0lREnfmS4cg=
 go.opentelemetry.io/otel v1.3.0/go.mod h1:PWIKzi6JCp7sM0k9yZ43VX+T345uNbAkDKwHVjb2PTs=
+go.opentelemetry.io/otel v1.6.1/go.mod h1:blzUabWHkX6LJewxvadmzafgh/wnvBSDBdOuwkAtrWQ=
 go.opentelemetry.io/otel v1.6.3 h1:FLOfo8f9JzFVFVyU+MSRJc2HdEAXQgm7pIv2uFKRSZE=
 go.opentelemetry.io/otel v1.6.3/go.mod h1:7BgNga5fNlF/iZjG06hM3yofffp0ofKCDwSXx1GC4dI=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0 h1:cLhx8llHw02h5JTqGqaRbYn+QVKHmrzD9vEbKnSPk5U=
 go.opentelemetry.io/otel/exporters/jaeger v1.0.0/go.mod h1:q10N1AolE1JjqKrFJK2tYw0iZpmX+HBaXBtuCzRnBGQ=
+go.opentelemetry.io/otel/exporters/otlp v0.20.0 h1:PTNgq9MRmQqqJY0REVbZFvwkYOA85vbdQU/nVfxDyqg=
 go.opentelemetry.io/otel/exporters/otlp v0.20.0/go.mod h1:YIieizyaN77rtLJra0buKiNBOm9XQfkPEKBeuhoMwAM=
 go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.3.0/go.mod h1:VpP4/RMn8bv8gNo9uK7/IMY4mtWLELsS+JIP0inH0h4=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3 h1:nAmg1WgsUXoXf46dJG9eS/AzOcvkCTK4xJSUYpWyHYg=
+go.opentelemetry.io/otel/exporters/otlp/internal/retry v1.6.3/go.mod h1:NEu79Xo32iVb+0gVNV8PMd7GoWqnyDXRlj04yFjqz40=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.3.0/go.mod h1:hO1KLR7jcKaDDKDkvI9dP/FIhpmna5lkqPUQdEjFAM8=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3 h1:4/UjHWMVVc5VwX/KAtqJOHErKigMCH8NexChMuanb/o=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace v1.6.3/go.mod h1:UJmXdiVVBaZ63umRUTwJuCMAV//GCMvDiQwn703/GoY=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.3.0/go.mod h1:keUU7UfnwWTWpJ+FWnyqmogPa82nuU5VUANFq49hlMY=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3 h1:leYDq5psbM3K4QNcZ2juCj30LjUnvxjuYQj1mkGjXFM=
+go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc v1.6.3/go.mod h1:ycItY/esVj8c0dKgYTOztTERXtPzcfDU/0o8EdwCjoA=
 go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracehttp v1.3.0/go.mod h1:QNX1aly8ehqqX1LEa6YniTU7VY9I6R3X/oPxhGdTceE=
 go.opentelemetry.io/otel/internal/metric v0.21.0/go.mod h1:iOfAaY2YycsXfYD4kaRSbLx2LKmfpKObWBEv9QK5zFo=
 go.opentelemetry.io/otel/metric v0.20.0/go.mod h1:598I5tYlH1vzBjn+BTuhzTCSb/9debfNp6R3s7Pr1eU=
@@ -2800,18 +2817,22 @@ go.opentelemetry.io/otel/oteltest v1.0.0-RC1/go.mod h1:+eoIG0gdEOaPNftuy1YScLr1G
 go.opentelemetry.io/otel/sdk v0.20.0/go.mod h1:g/IcepuwNsoiX5Byy2nNV0ySUF1em498m7hBWC279Yc=
 go.opentelemetry.io/otel/sdk v1.0.0-RC1/go.mod h1:kj6yPn7Pgt5ByRuwesbaWcRLA+V7BSDg3Hf8xRvsvf8=
 go.opentelemetry.io/otel/sdk v1.0.0/go.mod h1:PCrDHlSy5x1kjezSdL37PhbFUMjrsLRshJ2zCzeXwbM=
-go.opentelemetry.io/otel/sdk v1.3.0 h1:3278edCoH89MEJ0Ky8WQXVmDQv3FX4ZJ3Pp+9fJreAI=
 go.opentelemetry.io/otel/sdk v1.3.0/go.mod h1:rIo4suHNhQwBIPg9axF8V9CA72Wz2mKF1teNrup8yzs=
+go.opentelemetry.io/otel/sdk v1.6.3 h1:prSHYdwCQOX5DrsEzxowH3nLhoAzEBdZhvrR79scfLs=
+go.opentelemetry.io/otel/sdk v1.6.3/go.mod h1:A4iWF7HTXa+GWL/AaqESz28VuSBIcZ+0CV+IzJ5NMiQ=
 go.opentelemetry.io/otel/sdk/export/metric v0.20.0/go.mod h1:h7RBNMsDJ5pmI1zExLi+bJK+Dr8NQCh0qGhm1KDnNlE=
 go.opentelemetry.io/otel/sdk/metric v0.20.0/go.mod h1:knxiS8Xd4E/N+ZqKmUPf3gTTZ4/0TjTXukfxjzSTpHE=
 go.opentelemetry.io/otel/trace v0.20.0/go.mod h1:6GjCW8zgDjwGHGa6GkyeB8+/5vjT16gUEi0Nf1iBdgw=
 go.opentelemetry.io/otel/trace v1.0.0-RC1/go.mod h1:86UHmyHWFEtWjfWPSbu0+d0Pf9Q6e1U+3ViBOc+NXAg=
 go.opentelemetry.io/otel/trace v1.0.0/go.mod h1:PXTWqayeFUlJV1YDNhsJYB184+IvAH814St6o6ajzIs=
 go.opentelemetry.io/otel/trace v1.3.0/go.mod h1:c/VDhno8888bvQYmbYLqe41/Ldmr/KKunbvWM4/fEjk=
+go.opentelemetry.io/otel/trace v1.6.1/go.mod h1:RkFRM1m0puWIq10oxImnGEduNBzxiN7TXluRBtE+5j0=
 go.opentelemetry.io/otel/trace v1.6.3 h1:IqN4L+5b0mPNjdXIiZ90Ni4Bl5BRkDQywePLWemd9bc=
 go.opentelemetry.io/otel/trace v1.6.3/go.mod h1:GNJQusJlUgZl9/TQBPKU/Y/ty+0iVB5fjhKeJGZPGFs=
 go.opentelemetry.io/proto/otlp v0.7.0/go.mod h1:PqfVotwruBrMGOCsRd/89rSnXhoiJIqeYNgFYFoEGnI=
 go.opentelemetry.io/proto/otlp v0.11.0/go.mod h1:QpEjXPrNQzrFDZgoTo49dgHR9RYRSrg3NAKnUGl9YpQ=
+go.opentelemetry.io/proto/otlp v0.15.0 h1:h0bKrvdrT/9sBwEJ6iWUqT/N/xPcS66bL4u3isneJ6w=
+go.opentelemetry.io/proto/otlp v0.15.0/go.mod h1:H7XAot3MsfNsj7EXtrA2q5xSNQ10UqI405h3+duxN4U=
 go.starlark.net v0.0.0-20200901195727-6e684ef5eeee/go.mod h1:f0znQkUKRrkk36XxWbGjMqQM8wGv/xHBVE2qc3B5oFU=
 go.uber.org/atomic v1.3.2/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=
 go.uber.org/atomic v1.4.0/go.mod h1:gD2HeocX3+yG+ygLZcrzQJaqmWj9AIm7n08wl/qW/PE=

pkg/infra/tracing/opentelemetry_tracing.go

@@ -6,11 +6,16 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/infra/log/level"
 	"github.com/grafana/grafana/pkg/setting"
+	"go.etcd.io/etcd/version"
+	jaegerpropagator "go.opentelemetry.io/contrib/propagators/jaeger"
 	"go.opentelemetry.io/otel"
 	"go.opentelemetry.io/otel/attribute"
 	"go.opentelemetry.io/otel/codes"
 	"go.opentelemetry.io/otel/exporters/jaeger"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace"
+	"go.opentelemetry.io/otel/exporters/otlp/otlptrace/otlptracegrpc"
 	"go.opentelemetry.io/otel/propagation"
 	"go.opentelemetry.io/otel/sdk/resource"
 	tracesdk "go.opentelemetry.io/otel/sdk/trace"
@@ -18,6 +23,13 @@ import (
 	trace "go.opentelemetry.io/otel/trace"
 )
 
+const (
+	jaegerExporter   string = "jaeger"
+	otlpExporter     string = "otlp"
+	jaegerPropagator string = "jaeger"
+	w3cPropagator    string = "w3c"
+)
+
 type Tracer interface {
 	Run(context.Context) error
 	Start(ctx context.Context, spanName string, opts ...trace.SpanStartOption) (context.Context, Span)
@@ -34,9 +46,10 @@ type Span interface {
 }
 
 type Opentelemetry struct {
-	enabled bool
+	enabled     string
-	address string
+	address     string
-	log     log.Logger
+	propagation string
+	log         log.Logger
 
 	tracerProvider *tracesdk.TracerProvider
 	tracer         trace.Tracer
@@ -53,6 +66,12 @@ type EventValue struct {
 	Num int64
 }
 
+type otelErrHandler func(err error)
+
+func (o otelErrHandler) Handle(err error) {
+	o(err)
+}
+
 func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
 	section, err := ots.Cfg.Raw.GetSection("tracing.opentelemetry.jaeger")
 	if err != nil {
@@ -61,13 +80,26 @@ func (ots *Opentelemetry) parseSettingsOpentelemetry() error {
 
 	ots.address = section.Key("address").MustString("")
 	if ots.address != "" {
-		ots.enabled = true
+		ots.enabled = jaegerExporter
+		return nil
+	}
+	ots.propagation = section.Key("propagation").MustString("")
+
+	section, err = ots.Cfg.Raw.GetSection("tracing.opentelemetry.otlp")
+	if err != nil {
+		return err
 	}
 
+	ots.address = section.Key("address").MustString("")
+	if ots.address != "" {
+		ots.enabled = otlpExporter
+	}
+	ots.propagation = section.Key("propagation").MustString("")
+
 	return nil
 }
 
-func (ots *Opentelemetry) initTracerProvider() (*tracesdk.TracerProvider, error) {
+func (ots *Opentelemetry) initJaegerTracerProvider() (*tracesdk.TracerProvider, error) {
 	// Create the Jaeger exporter
 	exp, err := jaeger.New(jaeger.WithCollectorEndpoint(jaeger.WithEndpoint(ots.address)))
 	if err != nil {
@@ -86,18 +118,69 @@ func (ots *Opentelemetry) initTracerProvider() (*tracesdk.TracerProvider, error)
 	return tp, nil
 }
 
-func (ots *Opentelemetry) initOpentelemetryTracer() error {
+func (ots *Opentelemetry) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
-	tp, err := ots.initTracerProvider()
+	client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.address), otlptracegrpc.WithInsecure())
+	exp, err := otlptrace.New(context.Background(), client)
 	if err != nil {
-		return err
+		return nil, err
+	}
+
+	res, err := resource.New(
+		context.Background(),
+		resource.WithAttributes(
+			semconv.ServiceNameKey.String("grafana"),
+			semconv.ServiceVersionKey.String(version.Version),
+		),
+		resource.WithProcessRuntimeDescription(),
+		resource.WithTelemetrySDK(),
+	)
+	if err != nil {
+		return nil, err
 	}
+
+	tp := tracesdk.NewTracerProvider(
+		tracesdk.WithBatcher(exp),
+		tracesdk.WithSampler(tracesdk.ParentBased(
+			tracesdk.AlwaysSample(),
+		)),
+		tracesdk.WithResource(res),
+	)
+	return tp, nil
+}
+
+func (ots *Opentelemetry) initOpentelemetryTracer() error {
+	var tp *tracesdk.TracerProvider
+	var err error
+	switch ots.enabled {
+	case jaegerExporter:
+		tp, err = ots.initJaegerTracerProvider()
+		if err != nil {
+			return err
+		}
+	case otlpExporter:
+		tp, err = ots.initOTLPTracerProvider()
+		if err != nil {
+			return err
+		}
+	default:
+		ots.log.Error("invalid trace exporter")
+	}
+
 	// Register our TracerProvider as the global so any imported
 	// instrumentation in the future will default to using it
 	// only if tracing is enabled
-	if ots.enabled {
+	if ots.enabled != "" {
 		otel.SetTracerProvider(tp)
 	}
 
+	switch ots.propagation {
+	case w3cPropagator:
+		otel.SetTextMapPropagator(propagation.TraceContext{})
+	case jaegerPropagator:
+		otel.SetTextMapPropagator(jaegerpropagator.Jaeger{})
+	default:
+		otel.SetTextMapPropagator(propagation.TraceContext{})
+	}
 	ots.tracerProvider = tp
 	ots.tracer = otel.GetTracerProvider().Tracer("component-main")
 
@@ -105,6 +188,12 @@ func (ots *Opentelemetry) initOpentelemetryTracer() error {
 }
 
 func (ots *Opentelemetry) Run(ctx context.Context) error {
+	otel.SetErrorHandler(otelErrHandler(func(err error) {
+		err = level.Error(ots.log).Log("msg", "OpenTelemetry handler returned an error", "err", err)
+		if err != nil {
+			ots.log.Error("OpenTelemetry log returning error", err)
+		}
+	}))
 	<-ctx.Done()
 
 	ots.log.Info("Closing tracing")

pkg/infra/tracing/test_helper.go

@@ -1,7 +1,9 @@
 package tracing
 
 func InitializeTracerForTest() (Tracer, error) {
-	ots := &Opentelemetry{}
+	ots := &Opentelemetry{
+		enabled: "jaeger",
+	}
 	err := ots.initOpentelemetryTracer()
 	if err != nil {
 		return ots, err
@@ -10,7 +12,9 @@ func InitializeTracerForTest() (Tracer, error) {
 }
 
 func InitializeForBus() Tracer {
-	ots := &Opentelemetry{}
+	ots := &Opentelemetry{
+		enabled: "jaeger",
+	}
 	_ = ots.initOpentelemetryTracer()
 	return ots
 }

pkg/infra/tracing/tracing.go

@@ -28,29 +28,37 @@ const (
 )
 
 func ProvideService(cfg *setting.Cfg) (Tracer, error) {
+	ts, ots, err := parseSettings(cfg)
+	if err != nil {
+		return nil, err
+	}
+
+	if ts.enabled {
+		return ts, ts.initJaegerGlobalTracer()
+	}
+
+	return ots, ots.initOpentelemetryTracer()
+}
+
+func parseSettings(cfg *setting.Cfg) (*Opentracing, *Opentelemetry, error) {
 	ts := &Opentracing{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
-
+	err := ts.parseSettings()
-	if err := ts.parseSettings(); err != nil {
+	if err != nil {
-		return nil, err
+		return ts, nil, err
 	}
-
 	if ts.enabled {
-		return ts, ts.initGlobalTracer()
+		return ts, nil, nil
 	}
 
 	ots := &Opentelemetry{
 		Cfg: cfg,
 		log: log.New("tracing"),
 	}
-
+	err = ots.parseSettingsOpentelemetry()
-	if err := ots.parseSettingsOpentelemetry(); err != nil {
+	return ts, ots, err
-		return nil, err
-	}
-
-	return ots, ots.initOpentelemetryTracer()
 }
 
 type traceKey struct{}
@@ -136,7 +144,7 @@ func (ts *Opentracing) initJaegerCfg() (jaegercfg.Configuration, error) {
 	return cfg, nil
 }
 
-func (ts *Opentracing) initGlobalTracer() error {
+func (ts *Opentracing) initJaegerGlobalTracer() error {
 	cfg, err := ts.initJaegerCfg()
 	if err != nil {
 		return err

pkg/tests/api/alerting/api_alertmanager_test.go

@@ -27,6 +27,11 @@ import (
 	"github.com/grafana/grafana/pkg/tests/testinfra"
 )
 
+type Response struct {
+	Message string `json:"message"`
+	TraceID string `json:"traceID"`
+}
+
 func TestAMConfigAccess(t *testing.T) {
 	_, err := tracing.InitializeTracerForTest()
 	require.NoError(t, err)
@@ -880,11 +885,11 @@ func TestAlertRuleCRUD(t *testing.T) {
 	// Now, let's try to create some invalid alert rules.
 	{
 		testCases := []struct {
-			desc             string
+			desc            string
-			rulegroup        string
+			rulegroup       string
-			interval         model.Duration
+			interval        model.Duration
-			rule             apimodels.PostableExtendedRuleNode
+			rule            apimodels.PostableExtendedRuleNode
-			expectedResponse string
+			expectedMessage string
 		}{
 			{
 				desc:      "alert rule without queries and expressions",
@@ -900,7 +905,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						Data:  []ngmodels.AlertQuery{},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: invalid alert rule: no queries or expressions are found", "traceID":"00000000000000000000000000000000"}`,
+				expectedMessage: "invalid rule specification at index [0]: invalid alert rule: no queries or expressions are found",
 			},
 			{
 				desc:      "alert rule with empty title",
@@ -930,7 +935,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title cannot be empty", "traceID":"00000000000000000000000000000000"}`,
+				expectedMessage: "invalid rule specification at index [0]: alert rule title cannot be empty",
 			},
 			{
 				desc:      "alert rule with too long name",
@@ -960,7 +965,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: alert rule title is too long. Max length is 190", "traceID":"00000000000000000000000000000000"}`,
+				expectedMessage: "invalid rule specification at index [0]: alert rule title is too long. Max length is 190",
 			},
 			{
 				desc:      "alert rule with too long rulegroup",
@@ -990,7 +995,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "rule group name is too long. Max length is 190", "traceID":"00000000000000000000000000000000"}`,
+				expectedMessage: "rule group name is too long. Max length is 190",
 			},
 			{
 				desc:      "alert rule with invalid interval",
@@ -1021,8 +1026,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "rule evaluation interval (1 second) should be positive ` +
+				expectedMessage: "rule evaluation interval (1 second) should be positive number that is multiple of the base interval of 10 seconds",
-					`number that is multiple of the base interval of 10 seconds", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with unknown datasource",
@@ -1052,8 +1056,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring:` +
+				expectedMessage: "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: invalid query A: data source not found: unknown",
-					` invalid query A: data source not found: unknown", "traceID":"00000000000000000000000000000000"}`,
 			},
 			{
 				desc:      "alert rule with invalid condition",
@@ -1083,8 +1086,7 @@ func TestAlertRuleCRUD(t *testing.T) {
 						},
 					},
 				},
-				expectedResponse: `{"message": "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: ` +
+				expectedMessage: "invalid rule specification at index [0]: failed to validate condition of alert rule AlwaysFiring: condition B not found in any query or expression: it should be one of: [A]",
-					`condition B not found in any query or expression: it should be one of: [A]", "traceID":"00000000000000000000000000000000"}`,
 			},
 		}
 
@@ -1113,8 +1115,14 @@ func TestAlertRuleCRUD(t *testing.T) {
 				b, err := ioutil.ReadAll(resp.Body)
 				require.NoError(t, err)
 
+				res := &Response{}
+				err = json.Unmarshal(b, &res)
+				require.NoError(t, err)
+
+				assert.Equal(t, res.Message, tc.expectedMessage)
+				assert.NotEmpty(t, res.TraceID)
+
 				assert.Equal(t, resp.StatusCode, http.StatusBadRequest)
-				require.JSONEq(t, tc.expectedResponse, string(b))
 			})
 		}
 	}
@@ -2266,6 +2274,7 @@ func TestEval(t *testing.T) {
 		payload            string
 		expectedStatusCode int
 		expectedResponse   string
+		expectedMessage    string
 	}{
 		{
 			desc: "alerting condition",
@@ -2414,8 +2423,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse: `{"message": "invalid condition: condition B not found in any query or expression: it should be one of: [A]",` +
+			expectedMessage:    "invalid condition: condition B not found in any query or expression: it should be one of: [A]",
-				`"traceID": "00000000000000000000000000000000"}`,
 		},
 		{
 			desc: "unknown query datasource",
@@ -2440,7 +2448,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse:   `{"message": "invalid condition: invalid query A: data source not found: unknown", "traceID": "00000000000000000000000000000000"}`,
+			expectedMessage:    "invalid condition: invalid query A: data source not found: unknown",
 		},
 	}
 
@@ -2457,9 +2465,18 @@ func TestEval(t *testing.T) {
 			})
 			b, err := ioutil.ReadAll(resp.Body)
 			require.NoError(t, err)
+			res := Response{}
+			err = json.Unmarshal(b, &res)
+			require.NoError(t, err)
 
 			assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
-			require.JSONEq(t, tc.expectedResponse, string(b))
+			if tc.expectedResponse != "" {
+				require.JSONEq(t, tc.expectedResponse, string(b))
+			}
+			if tc.expectedMessage != "" {
+				assert.Equal(t, tc.expectedMessage, res.Message)
+				assert.NotEmpty(t, res.TraceID)
+			}
 		})
 	}
 
@@ -2469,6 +2486,7 @@ func TestEval(t *testing.T) {
 		payload            string
 		expectedStatusCode int
 		expectedResponse   string
+		expectedMessage    string
 	}{
 		{
 			desc: "alerting condition",
@@ -2596,8 +2614,7 @@ func TestEval(t *testing.T) {
 			}
 			`,
 			expectedStatusCode: http.StatusBadRequest,
-			expectedResponse: `{"message": "invalid queries or expressions: invalid query A: data source not found: unknown",` +
+			expectedMessage:    "invalid queries or expressions: invalid query A: data source not found: unknown",
-				`"traceID": "00000000000000000000000000000000"}`,
 		},
 	}
 
@@ -2614,9 +2631,19 @@ func TestEval(t *testing.T) {
 			})
 			b, err := ioutil.ReadAll(resp.Body)
 			require.NoError(t, err)
+			res := Response{}
+			err = json.Unmarshal(b, &res)
+			require.NoError(t, err)
 
 			assert.Equal(t, tc.expectedStatusCode, resp.StatusCode)
-			require.JSONEq(t, tc.expectedResponse, string(b))
+			if tc.expectedResponse != "" {
+				require.JSONEq(t, tc.expectedResponse, string(b))
+			}
+
+			if tc.expectedMessage != "" {
+				require.Equal(t, tc.expectedMessage, res.Message)
+				require.NotEmpty(t, res.TraceID)
+			}
 		})
 	}
 }

pkg/tests/api/alerting/api_notification_channel_test.go

@@ -62,7 +62,10 @@ func TestTestReceivers(t *testing.T) {
 
 		b, err := ioutil.ReadAll(resp.Body)
 		require.NoError(t, err)
-		require.JSONEq(t, `{"traceID":"00000000000000000000000000000000"}`, string(b))
+		res := Response{}
+		err = json.Unmarshal(b, &res)
+		require.NoError(t, err)
+		require.NotEmpty(t, res.TraceID)
 	})
 
 	t.Run("assert working receiver returns OK", func(t *testing.T) {