Alerting: Fix route validation in provisioning service (#107550)

* introduce ExtraConfigsCrypto
* delete getLastConfiguration and move all code into method Get
* update legacy store to encrypt ExtraConfigs same way we do in MultiOrgAlertmanager
* update legacy store init
* delete PersistConfig
* remove unnecessary unmarshalling
* add better error

* add tests for legacy store

* add tests that cover extra config validation

* add integration test for conflicting routes

---------

Signed-off-by: Yuri Tseretyan <yuriy.tseretyan@grafana.com>

pkg/services/ngalert/api/api_provisioning_test.go

@@ -2034,7 +2034,7 @@ func createProvisioningSrvSut(t *testing.T) ProvisioningSrv {
 func createProvisioningSrvSutFromEnv(t *testing.T, env *testEnvironment) ProvisioningSrv {
 	t.Helper()
 	tracer := tracing.InitializeTracerForTest()
-	configStore := legacy_storage.NewAlertmanagerConfigStore(env.configs)
+	configStore := legacy_storage.NewAlertmanagerConfigStore(env.configs, notifier.NewExtraConfigsCrypto(env.secrets))
 	receiverSvc := notifier.NewReceiverService(
 		ac.NewReceiverAccess[*models.Receiver](env.ac, true),
 		configStore,

pkg/services/ngalert/ngalert.go

@@ -412,7 +412,7 @@ func (ng *AlertNG) init() error {
 	ng.stateManager = stateManager
 	ng.schedule = scheduler
 
-	configStore := legacy_storage.NewAlertmanagerConfigStore(ng.store)
+	configStore := legacy_storage.NewAlertmanagerConfigStore(ng.store, notifier.NewExtraConfigsCrypto(ng.SecretsService))
 	receiverService := notifier.NewReceiverService(
 		ac.NewReceiverAccess[*models.Receiver](ng.accesscontrol, false),
 		configStore,

pkg/services/ngalert/notifier/crypto.go

@@ -28,16 +28,16 @@ type Crypto interface {
 
 // alertmanagerCrypto implements decryption of Alertmanager configuration and encryption of arbitrary payloads based on Grafana's encryptions.
 type alertmanagerCrypto struct {
-	secrets secrets.Service
+	*ExtraConfigsCrypto
 	configs configurationStore
 	log     log.Logger
 }
 
 func NewCrypto(secrets secrets.Service, configs configurationStore, log log.Logger) Crypto {
 	return &alertmanagerCrypto{
-		secrets: secrets,
-		configs: configs,
-		log:     log,
+		ExtraConfigsCrypto: NewExtraConfigsCrypto(secrets),
+		configs:            configs,
+		log:                log,
 	}
 }
 
@@ -241,7 +241,17 @@ func (c *alertmanagerCrypto) Decrypt(ctx context.Context, payload []byte) ([]byt
 	return c.secrets.Decrypt(ctx, payload)
 }
 
-func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+type ExtraConfigsCrypto struct {
+	secrets secretService
+}
+
+func NewExtraConfigsCrypto(secrets secretService) *ExtraConfigsCrypto {
+	return &ExtraConfigsCrypto{
+		secrets: secrets,
+	}
+}
+
+func (c *ExtraConfigsCrypto) EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
 	for i := range config.ExtraConfigs {
 		encryptedValue, err := c.secrets.Encrypt(ctx, []byte(config.ExtraConfigs[i].AlertmanagerConfig), secrets.WithoutScope())
 		if err != nil {
@@ -254,7 +264,7 @@ func (c *alertmanagerCrypto) EncryptExtraConfigs(ctx context.Context, config *de
 	return nil
 }
 
-func (c *alertmanagerCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+func (c *ExtraConfigsCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
 	for i := range config.ExtraConfigs {
 		// Check if the config is encrypted by trying to base64 decode it
 		encryptedValue, err := base64.StdEncoding.DecodeString(config.ExtraConfigs[i].AlertmanagerConfig)

pkg/services/ngalert/notifier/legacy_storage/config.go

@@ -9,6 +9,11 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 )
 
+type crypto interface {
+	EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
+	DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error
+}
+
 type amConfigStore interface {
 	GetLatestAlertmanagerConfiguration(ctx context.Context, orgID int64) (*models.AlertConfiguration, error)
 	UpdateAlertmanagerConfiguration(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error
@@ -31,9 +36,17 @@ type ConfigRevision struct {
 	ConcurrencyToken string
 	Version          string
 }
+type alertmanagerConfigStoreImpl struct {
+	store  amConfigStore
+	crypto crypto
+}
 
-func getLastConfiguration(ctx context.Context, orgID int64, store amConfigStore) (*ConfigRevision, error) {
-	alertManagerConfig, err := store.GetLatestAlertmanagerConfiguration(ctx, orgID)
+func NewAlertmanagerConfigStore(store amConfigStore, crypto crypto) *alertmanagerConfigStoreImpl {
+	return &alertmanagerConfigStoreImpl{store: store, crypto: crypto}
+}
+
+func (a alertmanagerConfigStoreImpl) Get(ctx context.Context, orgID int64) (*ConfigRevision, error) {
+	alertManagerConfig, err := a.store.GetLatestAlertmanagerConfiguration(ctx, orgID)
 	if err != nil {
 		return nil, err
 	}
@@ -48,6 +61,11 @@ func getLastConfiguration(ctx context.Context, orgID int64, store amConfigStore)
 		return nil, err
 	}
 
+	err = a.crypto.DecryptExtraConfigs(ctx, cfg)
+	if err != nil {
+		return nil, fmt.Errorf("failed to decrypt extra configurations: %w", err)
+	}
+
 	return &ConfigRevision{
 		Config:           cfg,
 		ConcurrencyToken: concurrencyToken,
@@ -55,38 +73,21 @@ func getLastConfiguration(ctx context.Context, orgID int64, store amConfigStore)
 	}, nil
 }
 
-type alertmanagerConfigStoreImpl struct {
-	store amConfigStore
-}
-
-func NewAlertmanagerConfigStore(store amConfigStore) *alertmanagerConfigStoreImpl {
-	return &alertmanagerConfigStoreImpl{store: store}
-}
-
-func (a alertmanagerConfigStoreImpl) Get(ctx context.Context, orgID int64) (*ConfigRevision, error) {
-	return getLastConfiguration(ctx, orgID, a.store)
-}
-
 func (a alertmanagerConfigStoreImpl) Save(ctx context.Context, revision *ConfigRevision, orgID int64) error {
+	err := a.crypto.EncryptExtraConfigs(ctx, revision.Config)
+	if err != nil {
+		return fmt.Errorf("failed to encrypt extra configurations: %w", err)
+	}
+
 	serialized, err := SerializeAlertmanagerConfig(*revision.Config)
 	if err != nil {
 		return err
 	}
-	cmd := models.SaveAlertmanagerConfigurationCmd{
+	return a.store.UpdateAlertmanagerConfiguration(ctx, &models.SaveAlertmanagerConfigurationCmd{
 		AlertmanagerConfiguration: string(serialized),
 		ConfigurationVersion:      revision.Version,
 		FetchedConfigurationHash:  revision.ConcurrencyToken,
 		Default:                   false,
 		OrgID:                     orgID,
-	}
-	return a.PersistConfig(ctx, &cmd)
-}
-
-// PersistConfig validates to config before eventually persisting it if no error occurs
-func (a alertmanagerConfigStoreImpl) PersistConfig(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error {
-	cfg := &definitions.PostableUserConfig{}
-	if err := json.Unmarshal([]byte(cmd.AlertmanagerConfiguration), cfg); err != nil {
-		return fmt.Errorf("change would result in an invalid configuration state: %w", err)
-	}
-	return a.store.UpdateAlertmanagerConfiguration(ctx, cmd)
+	})
 }

pkg/services/ngalert/notifier/legacy_storage/config_test.go

@@ -22,8 +22,9 @@ func TestAlertmanagerConfigStoreGet(t *testing.T) {
 	orgID := int64(1)
 
 	t.Run("should read the latest config for giving organization", func(t *testing.T) {
+		cryptoMock := newFakeCrypto()
 		storeMock := &MockAMConfigStore{}
-		store := &alertmanagerConfigStoreImpl{store: storeMock}
+		store := NewAlertmanagerConfigStore(storeMock, cryptoMock)
 
 		expected := models.AlertConfiguration{
 			ID:                        1,
@@ -48,12 +49,18 @@ func TestAlertmanagerConfigStoreGet(t *testing.T) {
 		require.Equal(t, expectedCfg, *revision.Config)
 
 		storeMock.AssertCalled(t, "GetLatestAlertmanagerConfiguration", mock.Anything, orgID)
+
+		t.Run("should decrypt extra configs ", func(t *testing.T) {
+			require.Len(t, cryptoMock.Calls, 1)
+			require.Equal(t, "DecryptExtraConfigs", cryptoMock.Calls[0].Method)
+			require.Equal(t, &expectedCfg, cryptoMock.Calls[0].Args[1])
+		})
 	})
 
 	t.Run("propagate errors", func(t *testing.T) {
 		t.Run("when underlying store fails", func(t *testing.T) {
 			storeMock := &MockAMConfigStore{}
-			store := &alertmanagerConfigStoreImpl{store: storeMock}
+			store := NewAlertmanagerConfigStore(storeMock, newFakeCrypto())
 			expectedErr := errors.New("test=err")
 			storeMock.EXPECT().GetLatestAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(nil, expectedErr)
 
@@ -63,7 +70,7 @@ func TestAlertmanagerConfigStoreGet(t *testing.T) {
 
 		t.Run("return ErrNoAlertmanagerConfiguration config does not exist", func(t *testing.T) {
 			storeMock := &MockAMConfigStore{}
-			store := &alertmanagerConfigStoreImpl{store: storeMock}
+			store := NewAlertmanagerConfigStore(storeMock, newFakeCrypto())
 			storeMock.EXPECT().GetLatestAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(nil, nil)
 
 			_, err := store.Get(context.Background(), orgID)
@@ -72,7 +79,7 @@ func TestAlertmanagerConfigStoreGet(t *testing.T) {
 
 		t.Run("when config cannot be unmarshalled", func(t *testing.T) {
 			storeMock := &MockAMConfigStore{}
-			store := &alertmanagerConfigStoreImpl{store: storeMock}
+			store := NewAlertmanagerConfigStore(storeMock, newFakeCrypto())
 			storeMock.EXPECT().GetLatestAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(&models.AlertConfiguration{
 				AlertmanagerConfiguration: "invalid-json",
 			}, nil)
@@ -80,6 +87,23 @@ func TestAlertmanagerConfigStoreGet(t *testing.T) {
 			_, err := store.Get(context.Background(), orgID)
 			require.Truef(t, ErrBadAlertmanagerConfiguration.Base.Is(err), "expected ErrBadAlertmanagerConfiguration but got %s", err.Error())
 		})
+
+		t.Run("when decrypting extra configs fails", func(t *testing.T) {
+			cryptoMock := newFakeCrypto()
+			storeMock := &MockAMConfigStore{}
+			store := NewAlertmanagerConfigStore(storeMock, cryptoMock)
+
+			expectedErr := errors.New("test-err")
+			cryptoMock.DecryptExtraConfigsFunc = func(ctx context.Context, config *definitions.PostableUserConfig) error {
+				return expectedErr
+			}
+			storeMock.EXPECT().GetLatestAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(&models.AlertConfiguration{
+				AlertmanagerConfiguration: defaultConfig,
+			}, nil)
+
+			_, err := store.Get(context.Background(), orgID)
+			require.ErrorIs(t, err, expectedErr)
+		})
 	})
 }
 
@@ -98,8 +122,9 @@ func TestAlertmanagerConfigStoreSave(t *testing.T) {
 	}
 
 	t.Run("should save the config to store", func(t *testing.T) {
+		cryptoMock := newFakeCrypto()
 		storeMock := &MockAMConfigStore{}
-		store := &alertmanagerConfigStoreImpl{store: storeMock}
+		store := NewAlertmanagerConfigStore(storeMock, cryptoMock)
 
 		storeMock.EXPECT().UpdateAlertmanagerConfiguration(mock.Anything, mock.Anything).RunAndReturn(func(ctx context.Context, cmd *models.SaveAlertmanagerConfigurationCmd) error {
 			assert.Equal(t, string(expectedCfg), cmd.AlertmanagerConfiguration)
@@ -114,17 +139,39 @@ func TestAlertmanagerConfigStoreSave(t *testing.T) {
 		require.NoError(t, err)
 
 		storeMock.AssertCalled(t, "UpdateAlertmanagerConfiguration", mock.Anything, mock.Anything)
+
+		t.Run("should encrypt extra configs ", func(t *testing.T) {
+			require.Len(t, cryptoMock.Calls, 1)
+			require.Equal(t, "EncryptExtraConfigs", cryptoMock.Calls[0].Method)
+			require.Equal(t, &cfg, cryptoMock.Calls[0].Args[1])
+		})
 	})
 
-	t.Run("propagates errors when underlying storage returns error", func(t *testing.T) {
-		storeMock := &MockAMConfigStore{}
-		store := &alertmanagerConfigStoreImpl{store: storeMock}
+	t.Run("propagates errors", func(t *testing.T) {
+		t.Run("when underlying storage returns error", func(t *testing.T) {
+			storeMock := &MockAMConfigStore{}
+			store := NewAlertmanagerConfigStore(storeMock, newFakeCrypto())
 
-		expectedErr := errors.New("test-err")
-		storeMock.EXPECT().UpdateAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(expectedErr)
+			expectedErr := errors.New("test-err")
+			storeMock.EXPECT().UpdateAlertmanagerConfiguration(mock.Anything, mock.Anything).Return(expectedErr)
 
-		err := store.Save(context.Background(), &revision, orgID)
+			err := store.Save(context.Background(), &revision, orgID)
+
+			require.ErrorIs(t, err, expectedErr)
+		})
+
+		t.Run("when encrypting extra configs fails", func(t *testing.T) {
+			cryptoMock := newFakeCrypto()
+			storeMock := &MockAMConfigStore{}
+			store := NewAlertmanagerConfigStore(storeMock, cryptoMock)
 
-		require.ErrorIs(t, err, expectedErr)
+			expectedErr := errors.New("test-err")
+			cryptoMock.EncryptExtraConfigsFunc = func(ctx context.Context, config *definitions.PostableUserConfig) error {
+				return expectedErr
+			}
+
+			err := store.Save(context.Background(), &revision, orgID)
+			require.ErrorIs(t, err, expectedErr)
+		})
 	})
 }

pkg/services/ngalert/notifier/legacy_storage/testing.go

@@ -5,6 +5,7 @@ import (
 
 	"github.com/stretchr/testify/mock"
 
+	"github.com/grafana/grafana/pkg/services/ngalert/api/tooling/definitions"
 	"github.com/grafana/grafana/pkg/services/ngalert/models"
 )
 
@@ -59,3 +60,39 @@ func (a *AlertmanagerConfigStoreFake) Save(ctx context.Context, revision *Config
 	}
 	return nil
 }
+
+type fakeCrypto struct {
+	Calls                   []methodCall
+	EncryptExtraConfigsFunc func(context.Context, *definitions.PostableUserConfig) error
+	DecryptExtraConfigsFunc func(context.Context, *definitions.PostableUserConfig) error
+}
+
+func newFakeCrypto() *fakeCrypto {
+	return &fakeCrypto{
+		Calls: []methodCall{},
+	}
+}
+
+func (f *fakeCrypto) EncryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+	f.Calls = append(f.Calls, methodCall{
+		Method: "EncryptExtraConfigs",
+		Args:   []interface{}{ctx, config},
+	})
+
+	if f.EncryptExtraConfigsFunc != nil {
+		return f.EncryptExtraConfigsFunc(ctx, config)
+	}
+	return nil
+}
+
+func (f *fakeCrypto) DecryptExtraConfigs(ctx context.Context, config *definitions.PostableUserConfig) error {
+	f.Calls = append(f.Calls, methodCall{
+		Method: "DecryptExtraConfigs",
+		Args:   []interface{}{ctx, config},
+	})
+
+	if f.DecryptExtraConfigsFunc != nil {
+		return f.DecryptExtraConfigsFunc(ctx, config)
+	}
+	return nil
+}

pkg/services/ngalert/notifier/receiver_svc_test.go

@@ -275,7 +275,7 @@ func TestReceiverService_Delete(t *testing.T) {
 			deleteUID:        baseReceiver.UID,
 			callerProvenance: definitions.Provenance(models.ProvenanceFile),
 			existing:         util.Pointer(models.CopyReceiverWith(baseReceiver, models.ReceiverMuts.WithProvenance(models.ProvenanceAPI))),
-			//expectedErr:      validation.MakeErrProvenanceChangeNotAllowed(models.ProvenanceAPI, models.ProvenanceFile),
+			// expectedErr:      validation.MakeErrProvenanceChangeNotAllowed(models.ProvenanceAPI, models.ProvenanceFile),
 		},
 		{
 			name:        "delete receiver with optimistic version mismatch fails",
@@ -673,7 +673,7 @@ func TestReceiverService_Update(t *testing.T) {
 			user:     writer,
 			receiver: models.CopyReceiverWith(baseReceiver, models.ReceiverMuts.WithProvenance(models.ProvenanceAPI)),
 			existing: util.Pointer(models.CopyReceiverWith(baseReceiver, models.ReceiverMuts.WithProvenance(models.ProvenanceFile))),
-			//expectedErr: validation.MakeErrProvenanceChangeNotAllowed(models.ProvenanceFile, models.ProvenanceAPI),
+			// expectedErr: validation.MakeErrProvenanceChangeNotAllowed(models.ProvenanceFile, models.ProvenanceAPI),
 			expectedUpdate: models.CopyReceiverWith(baseReceiver,
 				models.ReceiverMuts.WithProvenance(models.ProvenanceAPI),
 				rm.Encrypted(models.Base64Enrypt)),
@@ -1529,7 +1529,7 @@ func createReceiverServiceSut(t *testing.T, encryptSvc secretService) *ReceiverS
 
 	return NewReceiverService(
 		ac.NewReceiverAccess[*models.Receiver](acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), false),
-		legacy_storage.NewAlertmanagerConfigStore(store),
+		legacy_storage.NewAlertmanagerConfigStore(store, NewExtraConfigsCrypto(encryptSvc)),
 		provisioningStore,
 		&fakeAlertRuleNotificationStore{},
 		encryptSvc,

pkg/services/ngalert/provisioning/contactpoints_test.go

@@ -493,7 +493,7 @@ func createContactPointServiceSutWithConfigStore(t *testing.T, secretService sec
 
 	receiverService := notifier.NewReceiverService(
 		ac.NewReceiverAccess[*models.Receiver](acimpl.ProvideAccessControl(featuremgmt.WithFeatures()), true),
-		legacy_storage.NewAlertmanagerConfigStore(configStore),
+		legacy_storage.NewAlertmanagerConfigStore(configStore, notifier.NewExtraConfigsCrypto(secretService)),
 		provisioningStore,
 		&fakeAlertRuleNotificationStore{},
 		secretService,
@@ -504,7 +504,7 @@ func createContactPointServiceSutWithConfigStore(t *testing.T, secretService sec
 	)
 
 	return NewContactPointService(
-		legacy_storage.NewAlertmanagerConfigStore(configStore),
+		legacy_storage.NewAlertmanagerConfigStore(configStore, notifier.NewExtraConfigsCrypto(secretService)),
 		secretService,
 		provisioningStore,
 		xact,

pkg/services/ngalert/provisioning/errors.go

@@ -37,6 +37,10 @@ var (
 		"Invalid format of the submitted route.",
 		errutil.WithPublic("Invalid format of the submitted route: {{.Public.Error}}. Correct the payload and try again."),
 	)
+
+	ErrRouteConflictingMatchers = errutil.BadRequest("alerting.notifications.routes.conflictingMatchers").MustTemplate("Routing tree conflicts with the external configuration",
+		errutil.WithPublic("Cannot add\\update route: matchers conflict with an external routing tree merging matchers {{ .Public.Matchers }}, making the added\\updated route unreachable."),
+	)
 )
 
 // MakeErrTimeIntervalInvalid creates an error with the ErrTimeIntervalInvalid template
@@ -109,6 +113,14 @@ func MakeErrRouteInvalidFormat(err error) error {
 	})
 }
 
+func MakeErrRouteConflictingMatchers(matchers string) error {
+	return ErrRouteConflictingMatchers.Build(errutil.TemplateData{
+		Public: map[string]any{
+			"Matchers": matchers,
+		},
+	})
+}
+
 func MakeErrContactPointUidExists(uid, name string) error {
 	return ErrContactPointUidExists.Build(errutil.TemplateData{
 		Public: map[string]any{

pkg/services/ngalert/provisioning/notification_policies.go

@@ -3,12 +3,14 @@ package provisioning
 import (
 	"context"
 	"encoding/binary"
+	"errors"
 	"fmt"
 	"hash"
 	"hash/fnv"
 	"slices"
 	"unsafe"
 
+	"github.com/grafana/alerting/definition"
 	"github.com/prometheus/common/model"
 	"golang.org/x/exp/maps"
 
@@ -113,7 +115,11 @@ func (nps *NotificationPolicyService) UpdatePolicyTree(ctx context.Context, orgI
 
 	_, err = revision.Config.GetMergedAlertmanagerConfig()
 	if err != nil {
-		return definitions.Route{}, "", fmt.Errorf("new routing tree is not compatible with extra configuration: %w", err)
+		if errors.Is(err, definition.ErrSubtreeMatchersConflict) {
+			// TODO temporarily get the conflicting matchers
+			return definitions.Route{}, "", MakeErrRouteConflictingMatchers(fmt.Sprintf("%s", revision.Config.ExtraConfigs[0].MergeMatchers))
+		}
+		nps.log.Warn("Unable to validate the combined routing tree because of an error during merging. This could be a sign of broken external configuration. Skipping", "error", err)
 	}
 
 	err = nps.xact.InTransaction(ctx, func(ctx context.Context) error {

pkg/services/ngalert/provisioning/notification_policies_test.go

@@ -189,6 +189,93 @@ func TestUpdatePolicyTree(t *testing.T) {
 		assert.Equal(t, orgID, prov.Calls[0].Arguments[2].(int64))
 	})
 
+	t.Run("ErrRouteConflictingMatchers if the new route has conflicting matchers ", func(t *testing.T) {
+		rev := getDefaultConfigRevision()
+		rev.Config.ExtraConfigs = append(rev.Config.ExtraConfigs, definitions.ExtraConfiguration{
+			Identifier: "test",
+			MergeMatchers: config.Matchers{
+				{
+					Type:  labels.MatchEqual,
+					Name:  "imported",
+					Value: "true",
+				},
+			},
+			AlertmanagerConfig: `{"route":{"receiver":"mimir-receiver"},"receivers":[{"name":"mimir-receiver"}]}`,
+		})
+
+		route := definitions.Route{
+			Receiver: rev.Config.AlertmanagerConfig.Receivers[0].Name,
+			Routes: []*definitions.Route{
+				{
+					ObjectMatchers: definitions.ObjectMatchers{
+						{
+							Type:  labels.MatchEqual,
+							Name:  "imported",
+							Value: "true",
+						},
+						{
+							Type:  labels.MatchEqual,
+							Name:  "label",
+							Value: "value",
+						},
+					},
+				},
+			},
+		}
+
+		sut, store, _ := createNotificationPolicyServiceSut()
+		store.GetFn = func(ctx context.Context, orgID int64) (*legacy_storage.ConfigRevision, error) {
+			return &rev, nil
+		}
+
+		_, _, err := sut.UpdatePolicyTree(context.Background(), orgID, route, models.ProvenanceAPI, defaultVersion)
+		require.ErrorIs(t, err, ErrRouteConflictingMatchers)
+	})
+
+	t.Run("should ignore extra config validation if it is invalid", func(t *testing.T) {
+		extra := definitions.ExtraConfiguration{
+			MergeMatchers: config.Matchers{
+				{
+					Type:  labels.MatchEqual,
+					Name:  "imported",
+					Value: "true",
+				},
+			},
+		}
+		rev := getDefaultConfigRevision()
+		rev.Config.ExtraConfigs = append(rev.Config.ExtraConfigs, extra)
+
+		route := definitions.Route{
+			Receiver: rev.Config.AlertmanagerConfig.Receivers[0].Name,
+			Routes: []*definitions.Route{
+				{
+					ObjectMatchers: definitions.ObjectMatchers{
+						{
+							Type:  labels.MatchEqual,
+							Name:  "imported",
+							Value: "true",
+						},
+						{
+							Type:  labels.MatchEqual,
+							Name:  "label",
+							Value: "value",
+						},
+					},
+				},
+			},
+		}
+
+		sut, store, _ := createNotificationPolicyServiceSut()
+		store.GetFn = func(ctx context.Context, orgID int64) (*legacy_storage.ConfigRevision, error) {
+			return &rev, nil
+		}
+
+		result, version, err := sut.UpdatePolicyTree(context.Background(), orgID, route, models.ProvenanceAPI, defaultVersion)
+		require.NoError(t, err)
+		assert.Equal(t, route, result)
+		assert.Equal(t, calculateRouteFingerprint(route), version)
+	})
+
 	t.Run("updates Route and sets provenance in transaction if route is valid and version matches", func(t *testing.T) {
 		sut, store, prov := createNotificationPolicyServiceSut()
 		store.GetFn = func(ctx context.Context, orgID int64) (*legacy_storage.ConfigRevision, error) {

pkg/services/provisioning/provisioning.go

@@ -294,7 +294,7 @@ func (ps *ProvisioningServiceImpl) ProvisionAlerting(ctx context.Context) error
 		ps.alertingStore,
 		ps.alertingStore,
 		ps.folderService,
-		//ps.dashboardService,
+		// ps.dashboardService,
 		ps.quotaService,
 		ps.SQLStore,
 		int64(ps.Cfg.UnifiedAlerting.DefaultRuleEvaluationInterval.Seconds()),
@@ -304,7 +304,7 @@ func (ps *ProvisioningServiceImpl) ProvisionAlerting(ctx context.Context) error
 		notifier.NewCachedNotificationSettingsValidationService(ps.alertingStore),
 		alertingauthz.NewRuleService(ps.ac),
 	)
-	configStore := legacy_storage.NewAlertmanagerConfigStore(ps.alertingStore)
+	configStore := legacy_storage.NewAlertmanagerConfigStore(ps.alertingStore, notifier.NewExtraConfigsCrypto(ps.secretService))
 	receiverSvc := notifier.NewReceiverService(
 		alertingauthz.NewReceiverAccess[*ngmodels.Receiver](ps.ac, true),
 		configStore,

pkg/tests/apis/alerting/notifications/routingtree/routing_tree_test.go

@@ -663,3 +663,66 @@ func TestIntegrationDataConsistency(t *testing.T) {
 		assert.ElementsMatch(t, expected, tree.Spec.Routes[0].Matchers)
 	})
 }
+
+func TestIntegrationExtraConfigsConflicts(t *testing.T) {
+	if testing.Short() {
+		t.Skip("skipping integration test")
+	}
+
+	ctx := context.Background()
+	helper := apis.NewK8sTestHelper(t, testinfra.GrafanaOpts{
+		EnableFeatureToggles: []string{"alertingImportAlertmanagerAPI"},
+	})
+
+	cliCfg := helper.Org1.Admin.NewRestConfig()
+	legacyCli := alerting.NewAlertingLegacyAPIClient(helper.GetEnv().Server.HTTPServer.Listener.Addr().String(), cliCfg.Username, cliCfg.Password)
+
+	client := common.NewRoutingTreeClient(t, helper.Org1.Admin)
+
+	// Now upload a new extra config
+	testAlertmanagerConfigYAML := `
+route:
+  receiver: default
+
+receivers:
+- name: default
+  webhook_configs:
+  - url: 'http://localhost/webhook'
+`
+
+	headers := map[string]string{
+		"Content-Type":                         "application/yaml",
+		"X-Grafana-Alerting-Config-Identifier": "external-system",
+		"X-Grafana-Alerting-Merge-Matchers":    "imported=true",
+	}
+
+	// Post the configuration to Grafana
+	response := legacyCli.ConvertPrometheusPostAlertmanagerConfig(t, definitions.AlertmanagerUserConfig{
+		AlertmanagerConfig: testAlertmanagerConfigYAML,
+	}, headers)
+	require.Equal(t, "success", response.Status)
+
+	current, err := client.Get(ctx, v0alpha1.UserDefinedRoutingTreeName, v1.GetOptions{})
+	require.NoError(t, err)
+	updated := current.Copy().(*v0alpha1.RoutingTree)
+	updated.Spec.Routes = append(updated.Spec.Routes, v0alpha1.RoutingTreeRoute{
+		Matchers: []v0alpha1.RoutingTreeMatcher{
+			{
+				Label: "imported",
+				Type:  v0alpha1.RoutingTreeMatcherTypeEqual,
+				Value: "true",
+			},
+		},
+	})
+
+	_, err = client.Update(ctx, updated, v1.UpdateOptions{})
+	require.Error(t, err)
+	require.Truef(t, errors.IsBadRequest(err), "Should get BadRequest error but got: %s", err)
+
+	// Now delete extra config
+	legacyCli.ConvertPrometheusDeleteAlertmanagerConfig(t, headers)
+
+	// and try again
+	_, err = client.Update(ctx, updated, v1.UpdateOptions{})
+	require.NoError(t, err)
+}