Ruler: Disable x-scope-orgid header append in remote write (#11819)

Co-authored-by: Michel Hollands <42814411+MichelHollands@users.noreply.github.com>
Co-authored-by: Danny Kopping <dannykopping@gmail.com>

CHANGELOG.md

@@ -6,6 +6,7 @@
 
 ##### Enhancements
 
+* [11819](https://github.com/grafana/loki/pull/11819) **jburnham**: Ruler: Add the ability to disable the `X-Scope-OrgId` tenant identification header in remote write requests.
 * [11633](https://github.com/grafana/loki/pull/11633) **cyriltovena**: Add profiling integrations to tracing instrumentation.
 * [11571](https://github.com/grafana/loki/pull/11571) **MichelHollands**: Add a metrics.go log line for requests from querier to ingester
 * [11477](https://github.com/grafana/loki/pull/11477) **MichelHollands**: support GET for /ingester/shutdown

docs/sources/configure/_index.md

@@ -1274,6 +1274,10 @@ remote_write:
   # CLI flag: -ruler.remote-write.config-refresh-period
   [config_refresh_period: <duration> | default = 10s]
 
+  # Add X-Scope-OrgID header in remote write requests.
+  # CLI flag: -ruler.remote-write.add-org-id-header
+  [add_org_id_header: <boolean> | default = true]
+
 # Configuration for rule evaluation.
 evaluation:
   # The evaluation mode for the ruler. Can be either 'local' or 'remote'. If set

pkg/ruler/config.go

@@ -56,6 +56,7 @@ type RemoteWriteConfig struct {
 	Clients             map[string]config.RemoteWriteConfig `yaml:"clients,omitempty" doc:"description=Configure remote write clients. A map with remote client id as key."`
 	Enabled             bool                                `yaml:"enabled"`
 	ConfigRefreshPeriod time.Duration                       `yaml:"config_refresh_period"`
+	AddOrgIDHeader      bool                                `yaml:"add_org_id_header" doc:"description=Add X-Scope-OrgID header in remote write requests."`
 }
 
 func (c *RemoteWriteConfig) Validate() error {
@@ -108,6 +109,7 @@ func (c *RemoteWriteConfig) Clone() (*RemoteWriteConfig, error) {
 
 // RegisterFlags adds the flags required to config this to the given FlagSet.
 func (c *RemoteWriteConfig) RegisterFlags(f *flag.FlagSet) {
+	f.BoolVar(&c.AddOrgIDHeader, "ruler.remote-write.add-org-id-header", true, "Add X-Scope-OrgID header in remote write requests.")
 	f.BoolVar(&c.Enabled, "ruler.remote-write.enabled", false, "Enable remote-write functionality.")
 	f.DurationVar(&c.ConfigRefreshPeriod, "ruler.remote-write.config-refresh-period", 10*time.Second, "Minimum period to wait between refreshing remote-write reconfigurations. This should be greater than or equivalent to -limits.per-user-override-period.")
 

pkg/ruler/registry.go

@@ -212,8 +212,10 @@ func (r *walRegistry) getTenantConfig(tenant string) (instance.Config, error) {
 				}
 			}
 
-			// always inject the X-Scope-OrgId header for multi-tenant metrics backends
-			clt.Headers[user.OrgIDHeaderName] = tenant
+			if rwCfg.AddOrgIDHeader {
+				// inject the X-Scope-OrgId header for multi-tenant metrics backends
+				clt.Headers[user.OrgIDHeaderName] = tenant
+			}
 
 			rwCfg.Clients[id] = clt
 

pkg/ruler/registry_test.go

@@ -47,6 +47,7 @@ const remote2 = "remote-2"
 var remoteURL, _ = url.Parse("http://remote-write")
 var backCompatCfg = Config{
 	RemoteWrite: RemoteWriteConfig{
+		AddOrgIDHeader: true,
 		Client: &config.RemoteWriteConfig{
 			URL: &promConfig.URL{URL: remoteURL},
 			QueueConfig: config.QueueConfig{
@@ -105,6 +106,7 @@ var backCompatCfg = Config{
 var remoteURL2, _ = url.Parse("http://remote-write2")
 var cfg = Config{
 	RemoteWrite: RemoteWriteConfig{
+		AddOrgIDHeader: true,
 		Clients: map[string]config.RemoteWriteConfig{
 			remote1: {
 				URL: &promConfig.URL{URL: remoteURL},
@@ -751,6 +753,43 @@ func TestTenantRemoteWriteHeadersNoOverride(t *testing.T) {
 	assert.ElementsMatch(t, actual, expected, "Headers do not match")
 }
 
+func TestTenantRemoteWriteHeadersNoOrgIDHeader(t *testing.T) {
+	backCompatCfg.RemoteWrite.AddOrgIDHeader = false
+	reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat())
+
+	tenantCfg, err := reg.getTenantConfig(enabledRWTenant)
+	require.NoError(t, err)
+
+	assert.Len(t, tenantCfg.RemoteWrite[0].Headers, 1)
+	// ensure that X-Scope-OrgId header is missing
+	assert.Equal(t, tenantCfg.RemoteWrite[0].Headers[user.OrgIDHeaderName], "")
+	// the original header must be present
+	assert.Equal(t, tenantCfg.RemoteWrite[0].Headers["Base"], "value")
+
+	cfg.RemoteWrite.AddOrgIDHeader = false
+	reg = setupRegistry(t, cfg, newFakeLimits())
+
+	tenantCfg, err = reg.getTenantConfig(enabledRWTenant)
+	require.NoError(t, err)
+
+	// Ensure that overrides take plus and that X-Scope-OrgID header is still missing
+	expected := []map[string]string{
+		{
+			"Base": "value",
+		},
+		{
+			"Base": "value2",
+		},
+	}
+
+	actual := []map[string]string{}
+	for _, rw := range tenantCfg.RemoteWrite {
+		actual = append(actual, rw.Headers)
+	}
+
+	assert.ElementsMatch(t, actual, expected, "Headers do not match")
+}
+
 func TestRelabelConfigOverrides(t *testing.T) {
 	reg := setupRegistry(t, backCompatCfg, newFakeLimitsBackwardCompat())