Tracing: chore make insecure option configurable (#99236)

* Tracing: chore: make insecure option configurable

In case of insecure set to false, default system cert pool will be used to create otlpgrpc client.

* test: add tracing/insecure config checks

This is to verify whether insecure configuration is properly parsed.

* chore: update configura Grafana docs with new setting

* chore: also update comment in the ini files

---------

Co-authored-by: Matheus Macabu <macabu.matheus@gmail.com>

conf/defaults.ini

@@ -1738,6 +1738,9 @@ propagation =
 address =
 # Propagation specifies the text map propagation format: w3c, jaeger
 propagation =
+# Toggles the insecure communication setting, defaults to `true`.
+# When set to `false`, the OTLP client will use TLS credentials with the default system cert pool for communication.
+insecure =
 
 #################################### External Image Storage ##############
 [external_image_storage]

conf/sample.ini

@@ -1691,6 +1691,9 @@ default_datasource_uid =
 ; address = localhost:4317
 # Propagation specifies the text map propagation format: w3c, jaeger
 ; propagation = w3c
+# Toggles the insecure communication setting, defaults to `true`.
+# When set to `false`, the OTLP client will use TLS credentials with the default system cert pool for communication.
+; insecure = false
 
 #################################### External image storage ##########################
 [external_image_storage]

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -2228,6 +2228,11 @@ The values `jaeger` and `w3c` are supported.
 Add a comma (`,`) between values to specify multiple formats (for example, `"jaeger,w3c"`).
 The default value is `w3c`.
 
+#### `insecure`
+
+Toggles the insecure communication setting, defaults to `true`.
+When set to `false`, the OTLP client will use TLS credentials with the default system cert pool for communication.
+
 <hr>
 
 ### `[external_image_storage]`

pkg/infra/tracing/tracing.go

@@ -25,6 +25,7 @@ import (
 	semconv "go.opentelemetry.io/otel/semconv/v1.17.0"
 	trace "go.opentelemetry.io/otel/trace"
 	"go.opentelemetry.io/otel/trace/noop"
+	"google.golang.org/grpc/credentials"
 
 	"github.com/go-kit/log/level"
 
@@ -167,7 +168,14 @@ func (ots *TracingService) initJaegerTracerProvider() (*tracesdk.TracerProvider,
 }
 
 func (ots *TracingService) initOTLPTracerProvider() (*tracesdk.TracerProvider, error) {
-	client := otlptracegrpc.NewClient(otlptracegrpc.WithEndpoint(ots.cfg.Address), otlptracegrpc.WithInsecure())
+	opts := []otlptracegrpc.Option{otlptracegrpc.WithEndpoint(ots.cfg.Address)}
+	if ots.cfg.Insecure {
+		opts = append(opts, otlptracegrpc.WithInsecure())
+	} else {
+		opts = append(opts, otlptracegrpc.WithTLSCredentials(credentials.NewTLS(nil)))
+	}
+
+	client := otlptracegrpc.NewClient(opts...)
 	exp, err := otlptrace.New(context.Background(), client)
 	if err != nil {
 		return nil, err

pkg/infra/tracing/tracing_config.go

@@ -24,6 +24,7 @@ type TracingConfig struct {
 	ServiceVersion string
 
 	ProfilingIntegration bool
+	Insecure             bool
 }
 
 func ProvideTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) {
@@ -123,6 +124,7 @@ func ParseTracingConfig(cfg *setting.Cfg) (*TracingConfig, error) {
 		tc.enabled = otlpExporter
 	}
 	tc.Propagation = section.Key("propagation").MustString("")
+	tc.Insecure = section.Key("insecure").MustBool(true)
 	return tc, nil
 }
 

pkg/infra/tracing/tracing_config_test.go

@@ -61,6 +61,7 @@ func TestTracingConfig(t *testing.T) {
 		Env                map[string]string
 		ExpectedExporter   string
 		ExpectedAddress    string
+		ExpectedInsecure   bool
 		ExpectedPropagator string
 		ExpectedAttrs      []attribute.KeyValue
 
@@ -72,6 +73,7 @@ func TestTracingConfig(t *testing.T) {
 			Name:             "default config uses noop exporter",
 			Cfg:              "",
 			ExpectedExporter: noopExporter,
+			ExpectedInsecure: true,
 			ExpectedAttrs:    []attribute.KeyValue{},
 		},
 		{
@@ -81,6 +83,7 @@ func TestTracingConfig(t *testing.T) {
 			custom_attributes = key1:value1,key2:value2
 			`,
 			ExpectedExporter: noopExporter,
+			ExpectedInsecure: true,
 			ExpectedAttrs:    []attribute.KeyValue{attribute.String("key1", "value1"), attribute.String("key2", "value2")},
 		},
 		{
@@ -101,6 +104,19 @@ func TestTracingConfig(t *testing.T) {
 			`,
 			ExpectedExporter: otlpExporter,
 			ExpectedAddress:  "otlp.example.com:4317",
+			ExpectedInsecure: true,
+			ExpectedAttrs:    []attribute.KeyValue{},
+		},
+		{
+			Name: "OTLP insecure is parsed",
+			Cfg: `
+			[tracing.opentelemetry.otlp]
+			address = otlp.example.com:4317
+			insecure = false
+			`,
+			ExpectedExporter: otlpExporter,
+			ExpectedAddress:  "otlp.example.com:4317",
+			ExpectedInsecure: false,
 			ExpectedAttrs:    []attribute.KeyValue{},
 		},
 		{
@@ -154,6 +170,7 @@ func TestTracingConfig(t *testing.T) {
 			`,
 			ExpectedExporter:          otlpExporter,
 			ExpectedAddress:           "otlp.example.com:4317",
+			ExpectedInsecure:          true,
 			ExpectedAttrs:             []attribute.KeyValue{},
 			ExpectedSampler:           "remote",
 			ExpectedSamplerParam:      0.5,
@@ -179,6 +196,7 @@ func TestTracingConfig(t *testing.T) {
 			assert.Equal(t, test.ExpectedAddress, tracingConfig.Address)
 			assert.Equal(t, test.ExpectedPropagator, tracingConfig.Propagation)
 			assert.Equal(t, test.ExpectedAttrs, tracingConfig.CustomAttribs)
+			assert.Equal(t, test.ExpectedInsecure, tracingConfig.Insecure)
 
 			if test.ExpectedSampler != "" {
 				assert.Equal(t, test.ExpectedSampler, tracingConfig.Sampler)