Add support for promoting all OTel resource attributes (#16426)

Add support for promoting all OTel resource attributes via `promote_all_resource_attributes`,
except for those ignored using 'ignore_resource_attributes'.

---------

Signed-off-by: Antonio Jimenez <antonjim@thousandEyes.com>
Signed-off-by: Antonio Jimenez <123171955+antonjim-te@users.noreply.github.com>

config/config.go

@@ -1555,7 +1555,9 @@ var (
 
 // OTLPConfig is the configuration for writing to the OTLP endpoint.
 type OTLPConfig struct {
+	PromoteAllResourceAttributes      bool                      `yaml:"promote_all_resource_attributes,omitempty"`
 	PromoteResourceAttributes         []string                  `yaml:"promote_resource_attributes,omitempty"`
+	IgnoreResourceAttributes          []string                  `yaml:"ignore_resource_attributes,omitempty"`
 	TranslationStrategy               translationStrategyOption `yaml:"translation_strategy,omitempty"`
 	KeepIdentifyingResourceAttributes bool                      `yaml:"keep_identifying_resource_attributes,omitempty"`
 	ConvertHistogramsToNHCB           bool                      `yaml:"convert_histograms_to_nhcb,omitempty"`
@@ -1569,21 +1571,41 @@ func (c *OTLPConfig) UnmarshalYAML(unmarshal func(interface{}) error) error {
 		return err
 	}
 
+	if c.PromoteAllResourceAttributes {
+		if len(c.PromoteResourceAttributes) > 0 {
+			return errors.New("'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously")
+		}
+		if err := sanitizeAttributes(c.IgnoreResourceAttributes, "ignored"); err != nil {
+			return fmt.Errorf("invalid 'ignore_resource_attributes': %w", err)
+		}
+	} else {
+		if len(c.IgnoreResourceAttributes) > 0 {
+			return errors.New("'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true")
+		}
+		if err := sanitizeAttributes(c.PromoteResourceAttributes, "promoted"); err != nil {
+			return fmt.Errorf("invalid 'promote_resource_attributes': %w", err)
+		}
+	}
+
+	return nil
+}
+
+func sanitizeAttributes(attributes []string, adjective string) error {
 	seen := map[string]struct{}{}
 	var err error
-	for i, attr := range c.PromoteResourceAttributes {
+	for i, attr := range attributes {
 		attr = strings.TrimSpace(attr)
 		if attr == "" {
-			err = errors.Join(err, errors.New("empty promoted OTel resource attribute"))
+			err = errors.Join(err, fmt.Errorf("empty %s OTel resource attribute", adjective))
 			continue
 		}
 		if _, exists := seen[attr]; exists {
-			err = errors.Join(err, fmt.Errorf("duplicated promoted OTel resource attribute %q", attr))
+			err = errors.Join(err, fmt.Errorf("duplicated %s OTel resource attribute %q", adjective, attr))
 			continue
 		}
 
 		seen[attr] = struct{}{}
-		c.PromoteResourceAttributes[i] = attr
+		attributes[i] = attr
 	}
 	return err
 }

config/config_test.go

@@ -1661,8 +1661,8 @@ func TestRemoteWriteRetryOnRateLimit(t *testing.T) {
 }
 
 func TestOTLPSanitizeResourceAttributes(t *testing.T) {
-	t.Run("good config", func(t *testing.T) {
-		want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.good.yml"), false, promslog.NewNopLogger())
+	t.Run("good config - default resource attributes", func(t *testing.T) {
+		want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_default_resource_attributes.good.yml"), false, promslog.NewNopLogger())
 		require.NoError(t, err)
 
 		out, err := yaml.Marshal(want)
@@ -1670,14 +1670,74 @@ func TestOTLPSanitizeResourceAttributes(t *testing.T) {
 		var got Config
 		require.NoError(t, yaml.UnmarshalStrict(out, &got))
 
+		require.False(t, got.OTLPConfig.PromoteAllResourceAttributes)
+		require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes)
+		require.Empty(t, got.OTLPConfig.PromoteResourceAttributes)
+	})
+
+	t.Run("good config - promote resource attributes", func(t *testing.T) {
+		want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.good.yml"), false, promslog.NewNopLogger())
+		require.NoError(t, err)
+
+		out, err := yaml.Marshal(want)
+		require.NoError(t, err)
+		var got Config
+		require.NoError(t, yaml.UnmarshalStrict(out, &got))
+
+		require.False(t, got.OTLPConfig.PromoteAllResourceAttributes)
+		require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes)
 		require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.PromoteResourceAttributes)
 	})
 
-	t.Run("bad config", func(t *testing.T) {
-		_, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes.bad.yml"), false, promslog.NewNopLogger())
+	t.Run("bad config - promote resource attributes", func(t *testing.T) {
+		_, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_promote_resource_attributes.bad.yml"), false, promslog.NewNopLogger())
+		require.ErrorContains(t, err, `invalid 'promote_resource_attributes'`)
 		require.ErrorContains(t, err, `duplicated promoted OTel resource attribute "k8s.job.name"`)
 		require.ErrorContains(t, err, `empty promoted OTel resource attribute`)
 	})
+
+	t.Run("good config - promote all resource attributes", func(t *testing.T) {
+		want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_resource_attributes_promote_all.good.yml"), false, promslog.NewNopLogger())
+		require.NoError(t, err)
+
+		out, err := yaml.Marshal(want)
+		require.NoError(t, err)
+		var got Config
+		require.NoError(t, yaml.UnmarshalStrict(out, &got))
+		require.True(t, got.OTLPConfig.PromoteAllResourceAttributes)
+		require.Empty(t, got.OTLPConfig.PromoteResourceAttributes)
+		require.Empty(t, got.OTLPConfig.IgnoreResourceAttributes)
+	})
+
+	t.Run("good config - ignore resource attributes", func(t *testing.T) {
+		want, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.good.yml"), false, promslog.NewNopLogger())
+		require.NoError(t, err)
+
+		out, err := yaml.Marshal(want)
+		require.NoError(t, err)
+		var got Config
+		require.NoError(t, yaml.UnmarshalStrict(out, &got))
+		require.True(t, got.OTLPConfig.PromoteAllResourceAttributes)
+		require.Empty(t, got.OTLPConfig.PromoteResourceAttributes)
+		require.Equal(t, []string{"k8s.cluster.name", "k8s.job.name", "k8s.namespace.name"}, got.OTLPConfig.IgnoreResourceAttributes)
+	})
+
+	t.Run("bad config - ignore resource attributes", func(t *testing.T) {
+		_, err := LoadFile(filepath.Join("testdata", "otlp_sanitize_ignore_resource_attributes.bad.yml"), false, promslog.NewNopLogger())
+		require.ErrorContains(t, err, `invalid 'ignore_resource_attributes'`)
+		require.ErrorContains(t, err, `duplicated ignored OTel resource attribute "k8s.job.name"`)
+		require.ErrorContains(t, err, `empty ignored OTel resource attribute`)
+	})
+
+	t.Run("bad config - conflict between promote all and promote specific resource attributes", func(t *testing.T) {
+		_, err := LoadFile(filepath.Join("testdata", "otlp_promote_all_resource_attributes.bad.yml"), false, promslog.NewNopLogger())
+		require.ErrorContains(t, err, `'promote_all_resource_attributes' and 'promote_resource_attributes' cannot be configured simultaneously`)
+	})
+
+	t.Run("bad config - configuring ignoring of resource attributes without also enabling promotion of all resource attributes", func(t *testing.T) {
+		_, err := LoadFile(filepath.Join("testdata", "otlp_ignore_resource_attributes_without_promote_all.bad.yml"), false, promslog.NewNopLogger())
+		require.ErrorContains(t, err, `'ignore_resource_attributes' cannot be configured unless 'promote_all_resource_attributes' is true`)
+	})
 }
 
 func TestOTLPAllowServiceNameInTargetInfo(t *testing.T) {

config/testdata/otlp_ignore_resource_attributes_without_promote_all.bad.yml

@@ -0,0 +1,2 @@
+otlp:
+  ignore_resource_attributes: ["k8s.job.name"]

config/testdata/otlp_promote_all_resource_attributes.bad.yml

@@ -0,0 +1,3 @@
+otlp:
+  promote_all_resource_attributes: true
+  promote_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name"]

config/testdata/otlp_sanitize_default_resource_attributes.good.yml

@@ -0,0 +1 @@
+otlp:

config/testdata/otlp_sanitize_ignore_resource_attributes.bad.yml

@@ -0,0 +1,3 @@
+otlp:
+  promote_all_resource_attributes: true
+  ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name", "k8s.job.name", ""]

config/testdata/otlp_sanitize_ignore_resource_attributes.good.yml

@@ -0,0 +1,3 @@
+otlp:
+  promote_all_resource_attributes: true
+  ignore_resource_attributes: ["k8s.cluster.name", " k8s.job.name ", "k8s.namespace.name"]

config/testdata/otlp_sanitize_resource_attributes_promote_all.good.yml

@@ -0,0 +1,2 @@
+otlp:
+  promote_all_resource_attributes: true

docs/configuration/configuration.md

@@ -183,7 +183,15 @@ remote_write:
 # Settings related to the OTLP receiver feature.
 # See https://prometheus.io/docs/guides/opentelemetry/ for best practices.
 otlp:
+  # Promote specific list of resource attributes to labels.
+  # It cannot be configured simultaneously with 'promote_all_resource_attributes: true'.
   [ promote_resource_attributes: [<string>, ...] | default = [ ] ]
+  # Promoting all resource attributes to labels, except for the ones configured with 'ignore_resource_attributes'.
+  # Be aware that changes in attributes received by the OTLP endpoint may result in time series churn and lead to high memory usage by the Prometheus server.
+  # It cannot be set to 'true' simultaneously with 'promote_resource_attributes'.
+  [ promote_all_resource_attributes: <boolean> | default = false ]
+  # Which resource attributes to ignore, can only be set when 'promote_all_resource_attributes' is true.
+  [ ignore_resource_attributes: [<string>, ...] | default = [] ]
   # Configures translation of OTLP metrics when received through the OTLP metrics
   # endpoint. Available values:
   # - "UnderscoreEscapingWithSuffixes" refers to commonly agreed normalization used

storage/remote/otlptranslator/prometheusremotewrite/helper.go

@@ -122,13 +122,7 @@ func createAttributes(resource pcommon.Resource, attributes pcommon.Map, setting
 	serviceName, haveServiceName := resourceAttrs.Get(conventions.AttributeServiceName)
 	instance, haveInstanceID := resourceAttrs.Get(conventions.AttributeServiceInstanceID)
 
-	promotedAttrs := make([]prompb.Label, 0, len(settings.PromoteResourceAttributes))
-	for _, name := range settings.PromoteResourceAttributes {
-		if value, exists := resourceAttrs.Get(name); exists {
-			promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()})
-		}
-	}
-	sort.Stable(ByLabelName(promotedAttrs))
+	promotedAttrs := settings.PromoteResourceAttributes.promotedAttributes(resourceAttrs)
 
 	// Calculate the maximum possible number of labels we could return so we can preallocate l
 	maxLabelCount := attributes.Len() + len(settings.ExternalLabels) + len(promotedAttrs) + len(extras)/2

storage/remote/otlptranslator/prometheusremotewrite/helper_test.go

@@ -26,6 +26,7 @@ import (
 	"go.opentelemetry.io/collector/pdata/pcommon"
 	"go.opentelemetry.io/collector/pdata/pmetric"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 )
 
@@ -51,10 +52,12 @@ func TestCreateAttributes(t *testing.T) {
 	attrs.PutStr("metric-attr-other", "metric value other")
 
 	testCases := []struct {
-		name                      string
-		promoteResourceAttributes []string
-		ignoreAttrs               []string
-		expectedLabels            []prompb.Label
+		name                         string
+		promoteAllResourceAttributes bool
+		promoteResourceAttributes    []string
+		ignoreResourceAttributes     []string
+		ignoreAttrs                  []string
+		expectedLabels               []prompb.Label
 	}{
 		{
 			name:                      "Successful conversion without resource attribute promotion",
@@ -195,11 +198,90 @@ func TestCreateAttributes(t *testing.T) {
 				},
 			},
 		},
+		{
+			name:                         "Successful conversion promoting all resource attributes",
+			promoteAllResourceAttributes: true,
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_metric",
+				},
+				{
+					Name:  "instance",
+					Value: "service ID",
+				},
+				{
+					Name:  "job",
+					Value: "service name",
+				},
+				{
+					Name:  "existent_attr",
+					Value: "resource value",
+				},
+				{
+					Name:  "metric_attr",
+					Value: "metric value",
+				},
+				{
+					Name:  "metric_attr_other",
+					Value: "metric value other",
+				},
+				{
+					Name:  "service_name",
+					Value: "service name",
+				},
+				{
+					Name:  "service_instance_id",
+					Value: "service ID",
+				},
+			},
+		},
+		{
+			name:                         "Successful conversion promoting all resource attributes, ignoring 'service.instance.id'",
+			promoteAllResourceAttributes: true,
+			ignoreResourceAttributes: []string{
+				"service.instance.id",
+			},
+			expectedLabels: []prompb.Label{
+				{
+					Name:  "__name__",
+					Value: "test_metric",
+				},
+				{
+					Name:  "instance",
+					Value: "service ID",
+				},
+				{
+					Name:  "job",
+					Value: "service name",
+				},
+				{
+					Name:  "existent_attr",
+					Value: "resource value",
+				},
+				{
+					Name:  "metric_attr",
+					Value: "metric value",
+				},
+				{
+					Name:  "metric_attr_other",
+					Value: "metric value other",
+				},
+				{
+					Name:  "service_name",
+					Value: "service name",
+				},
+			},
+		},
 	}
 	for _, tc := range testCases {
 		t.Run(tc.name, func(t *testing.T) {
 			settings := Settings{
-				PromoteResourceAttributes: tc.promoteResourceAttributes,
+				PromoteResourceAttributes: NewPromoteResourceAttributes(config.OTLPConfig{
+					PromoteAllResourceAttributes: tc.promoteAllResourceAttributes,
+					PromoteResourceAttributes:    tc.promoteResourceAttributes,
+					IgnoreResourceAttributes:     tc.ignoreResourceAttributes,
+				}),
 			}
 			lbls := createAttributes(resource, attrs, settings, tc.ignoreAttrs, false, model.MetricNameLabel, "test_metric")
 

storage/remote/otlptranslator/prometheusremotewrite/metrics_to_prw.go

@@ -27,10 +27,16 @@ import (
 	"go.opentelemetry.io/collector/pdata/pmetric"
 	"go.uber.org/multierr"
 
+	"github.com/prometheus/prometheus/config"
 	"github.com/prometheus/prometheus/prompb"
 	"github.com/prometheus/prometheus/util/annotations"
 )
 
+type PromoteResourceAttributes struct {
+	promoteAll bool
+	attrs      map[string]struct{}
+}
+
 type Settings struct {
 	Namespace                         string
 	ExternalLabels                    map[string]string
@@ -38,7 +44,7 @@ type Settings struct {
 	ExportCreatedMetric               bool
 	AddMetricSuffixes                 bool
 	AllowUTF8                         bool
-	PromoteResourceAttributes         []string
+	PromoteResourceAttributes         *PromoteResourceAttributes
 	KeepIdentifyingResourceAttributes bool
 	ConvertHistogramsToNHCB           bool
 	AllowDeltaTemporality             bool
@@ -272,3 +278,46 @@ func (c *PrometheusConverter) addSample(sample *prompb.Sample, lbls []prompb.Lab
 	ts.Samples = append(ts.Samples, *sample)
 	return ts
 }
+
+func NewPromoteResourceAttributes(otlpCfg config.OTLPConfig) *PromoteResourceAttributes {
+	attrs := otlpCfg.PromoteResourceAttributes
+	if otlpCfg.PromoteAllResourceAttributes {
+		attrs = otlpCfg.IgnoreResourceAttributes
+	}
+	attrsMap := make(map[string]struct{}, len(attrs))
+	for _, s := range attrs {
+		attrsMap[s] = struct{}{}
+	}
+	return &PromoteResourceAttributes{
+		promoteAll: otlpCfg.PromoteAllResourceAttributes,
+		attrs:      attrsMap,
+	}
+}
+
+// promotedAttributes returns labels for promoted resourceAttributes.
+func (s *PromoteResourceAttributes) promotedAttributes(resourceAttributes pcommon.Map) []prompb.Label {
+	if s == nil {
+		return nil
+	}
+
+	var promotedAttrs []prompb.Label
+	if s.promoteAll {
+		promotedAttrs = make([]prompb.Label, 0, resourceAttributes.Len())
+		resourceAttributes.Range(func(name string, value pcommon.Value) bool {
+			if _, exists := s.attrs[name]; !exists {
+				promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()})
+			}
+			return true
+		})
+	} else {
+		promotedAttrs = make([]prompb.Label, 0, len(s.attrs))
+		resourceAttributes.Range(func(name string, value pcommon.Value) bool {
+			if _, exists := s.attrs[name]; exists {
+				promotedAttrs = append(promotedAttrs, prompb.Label{Name: name, Value: value.AsString()})
+			}
+			return true
+		})
+	}
+	sort.Stable(ByLabelName(promotedAttrs))
+	return promotedAttrs
+}

storage/remote/write_handler.go

@@ -592,7 +592,7 @@ func (rw *rwExporter) ConsumeMetrics(ctx context.Context, md pmetric.Metrics) er
 	annots, err := converter.FromMetrics(ctx, md, otlptranslator.Settings{
 		AddMetricSuffixes:                 otlpCfg.TranslationStrategy != config.NoTranslation,
 		AllowUTF8:                         otlpCfg.TranslationStrategy != config.UnderscoreEscapingWithSuffixes,
-		PromoteResourceAttributes:         otlpCfg.PromoteResourceAttributes,
+		PromoteResourceAttributes:         otlptranslator.NewPromoteResourceAttributes(otlpCfg),
 		KeepIdentifyingResourceAttributes: otlpCfg.KeepIdentifyingResourceAttributes,
 		ConvertHistogramsToNHCB:           otlpCfg.ConvertHistogramsToNHCB,
 		AllowDeltaTemporality:             rw.allowDeltaTemporality,