Add an option to use the external labels as selectors for the remote read endpoint (#10254)

* An option to ignore external_labels

Signed-off-by: DrAuYueng <ouyang1204@gmail.com>

config/config.go

@@ -203,8 +203,9 @@ var (
 
 	// DefaultRemoteReadConfig is the default remote read configuration.
 	DefaultRemoteReadConfig = RemoteReadConfig{
-		RemoteTimeout:    model.Duration(1 * time.Minute),
-		HTTPClientConfig: config.DefaultHTTPClientConfig,
+		RemoteTimeout:        model.Duration(1 * time.Minute),
+		HTTPClientConfig:     config.DefaultHTTPClientConfig,
+		FilterExternalLabels: true,
 	}
 
 	// DefaultStorageConfig is the default TSDB/Exemplar storage configuration.
@@ -854,6 +855,9 @@ type RemoteReadConfig struct {
 	// RequiredMatchers is an optional list of equality matchers which have to
 	// be present in a selector to query the remote read endpoint.
 	RequiredMatchers model.LabelSet `yaml:"required_matchers,omitempty"`
+
+	// Whether to use the external labels as selectors for the remote read endpoint.
+	FilterExternalLabels bool `yaml:"filter_external_labels,omitempty"`
 }
 
 // SetDirectory joins any relative file paths with dir.

config/config_test.go

@@ -130,11 +130,12 @@ var expectedConf = &Config{
 
 	RemoteReadConfigs: []*RemoteReadConfig{
 		{
-			URL:              mustParseURL("http://remote1/read"),
-			RemoteTimeout:    model.Duration(1 * time.Minute),
-			ReadRecent:       true,
-			Name:             "default",
-			HTTPClientConfig: config.DefaultHTTPClientConfig,
+			URL:                  mustParseURL("http://remote1/read"),
+			RemoteTimeout:        model.Duration(1 * time.Minute),
+			ReadRecent:           true,
+			Name:                 "default",
+			HTTPClientConfig:     config.DefaultHTTPClientConfig,
+			FilterExternalLabels: true,
 		},
 		{
 			URL:              mustParseURL("http://remote3/read"),
@@ -149,6 +150,7 @@ var expectedConf = &Config{
 				},
 				FollowRedirects: true,
 			},
+			FilterExternalLabels: true,
 		},
 	},
 

docs/configuration/configuration.md

@@ -2868,6 +2868,9 @@ tls_config:
 
 # Configure whether HTTP requests follow HTTP 3xx redirects.
 [ follow_redirects: <boolean> | default = true ]
+
+# Whether to use the external labels as selectors for the remote read endpoint.
+[ filter_external_labels: <boolean> | default = true ]
 ```
 
 There is a list of

storage/remote/storage.go

@@ -118,9 +118,13 @@ func (s *Storage) ApplyConfig(conf *config.Config) error {
 			return err
 		}
 
+		externalLabels := conf.GlobalConfig.ExternalLabels
+		if !rrConf.FilterExternalLabels {
+			externalLabels = make(labels.Labels, 0)
+		}
 		queryables = append(queryables, NewSampleAndChunkQueryableClient(
 			c,
-			conf.GlobalConfig.ExternalLabels,
+			externalLabels,
 			labelsToEqualityMatchers(rrConf.RequiredMatchers),
 			rrConf.ReadRecent,
 			s.localStartTimeCallback,

storage/remote/storage_test.go

@@ -21,6 +21,7 @@ import (
 	"github.com/stretchr/testify/require"
 
 	"github.com/prometheus/prometheus/config"
+	"github.com/prometheus/prometheus/model/labels"
 )
 
 func TestStorageLifecycle(t *testing.T) {
@@ -80,3 +81,55 @@ func TestUpdateRemoteReadConfigs(t *testing.T) {
 	err := s.Close()
 	require.NoError(t, err)
 }
+
+func TestFilterExternalLabels(t *testing.T) {
+	dir := t.TempDir()
+
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+
+	conf := &config.Config{
+		GlobalConfig: config.GlobalConfig{
+			ExternalLabels: labels.Labels{labels.Label{Name: "foo", Value: "bar"}},
+		},
+	}
+	require.NoError(t, s.ApplyConfig(conf))
+	require.Equal(t, 0, len(s.queryables))
+
+	conf.RemoteReadConfigs = []*config.RemoteReadConfig{
+		&config.DefaultRemoteReadConfig,
+	}
+
+	require.NoError(t, s.ApplyConfig(conf))
+	require.Equal(t, 1, len(s.queryables))
+	require.Equal(t, 1, len(s.queryables[0].(*sampleAndChunkQueryableClient).externalLabels))
+
+	err := s.Close()
+	require.NoError(t, err)
+}
+
+func TestIgnoreExternalLabels(t *testing.T) {
+	dir := t.TempDir()
+
+	s := NewStorage(nil, nil, nil, dir, defaultFlushDeadline, nil)
+
+	conf := &config.Config{
+		GlobalConfig: config.GlobalConfig{
+			ExternalLabels: labels.Labels{labels.Label{Name: "foo", Value: "bar"}},
+		},
+	}
+	require.NoError(t, s.ApplyConfig(conf))
+	require.Equal(t, 0, len(s.queryables))
+
+	conf.RemoteReadConfigs = []*config.RemoteReadConfig{
+		&config.DefaultRemoteReadConfig,
+	}
+
+	conf.RemoteReadConfigs[0].FilterExternalLabels = false
+
+	require.NoError(t, s.ApplyConfig(conf))
+	require.Equal(t, 1, len(s.queryables))
+	require.Equal(t, 0, len(s.queryables[0].(*sampleAndChunkQueryableClient).externalLabels))
+
+	err := s.Close()
+	require.NoError(t, err)
+}