Alerting: Add config disabled_labels to disable reserved labels (#51832)

* Alerting: Add config disabled_labels to disable reserved labels

[unified_alerting.reserved_labels]
disabled_labels

* Replace IsGrafanaFolderDisabled with more generic IsReservedLabelDisabled

* Simplify SchedulerCfg by including UnifiedAlertingSettings

conf/defaults.ini

@@ -877,6 +877,11 @@ max_concurrent_screenshots = 5
 # screenshots will be persisted to disk for up to temp_data_lifetime.
 upload_external_image_storage = false
 
+[unified_alerting.reserved_labels]
+# Comma-separated list of reserved labels added by the Grafana Alerting engine that should be disabled.
+# For example: `disabled_labels=grafana_folder`
+disabled_labels =
+
 #################################### Alerting ############################
 [alerting]
 # Enable the legacy alerting sub-system and interface. If Unified Alerting is already enabled and you try to go back to legacy alerting, all data that is part of Unified Alerting will be deleted. When this configuration section and flag are not defined, the state is defined at runtime. See the documentation for more details.

conf/sample.ini

@@ -847,6 +847,11 @@
 # The interval string is a possibly signed sequence of decimal numbers, followed by a unit suffix (ms, s, m, h, d), e.g. 30s or 1m.
 ;min_interval = 10s
 
+[unified_alerting.reserved_labels]
+# Comma-separated list of reserved labels added by the Grafana Alerting engine that should be disabled.
+# For example: `disabled_labels=grafana_folder`
+;disabled_labels =
+
 #################################### Alerting ############################
 [alerting]
 # Disable legacy alerting engine & UI features

docs/sources/alerting/fundamentals/annotation-label/how-to-use-labels.md

@@ -26,6 +26,7 @@ This topic explains why labels are a fundamental component of alerting.
 # Grafana reserved labels
 
 > **Note:** Labels prefixed with `grafana_` are reserved by Grafana for special use. If a manually configured label is added beginning with `grafana_` it may be overwritten in case of collision.
+> To stop the Grafana Alerting engine from adding a reserved label, you can disable it via the `disabled_labels` option in [unified_alerting.reserved_labels]({{< relref "../../../setup-grafana/configure-grafana/#unified_alertingreserved_labels" >}}) configuration.
 
 Grafana reserved labels can be used in the same way as manually configured labels. The current list of available reserved labels are:
 

docs/sources/setup-grafana/configure-grafana/_index.md

@@ -1305,6 +1305,18 @@ Uploads screenshots to the local Grafana server or remote storage such as Azure,
 
 <hr>
 
+## [unified_alerting.reserved_labels]
+
+For more information about Grafana Reserved Labels, refer to [Labels in Grafana Alerting]({{< relref "../../alerting/fundamentals/annotation-label/how-to-use-labels/#grafana-reserved-labels" >}}).
+
+### disabled_labels
+
+Comma-separated list of reserved labels added by the Grafana Alerting engine that should be disabled.
+
+For example: `disabled_labels=grafana_folder`
+
+<hr>
+
 ## [alerting]
 
 For more information about the legacy dashboard alerting feature in Grafana, refer to [Alerts overview]({{< relref "../../alerting/" >}}).

pkg/services/ngalert/ngalert.go

@@ -129,20 +129,16 @@ func (ng *AlertNG) init() error {
 	}
 
 	schedCfg := schedule.SchedulerCfg{
-		C:                       clock.New(),
-		BaseInterval:            ng.Cfg.UnifiedAlerting.BaseInterval,
-		Logger:                  ng.Log,
-		MaxAttempts:             ng.Cfg.UnifiedAlerting.MaxAttempts,
-		Evaluator:               eval.NewEvaluator(ng.Cfg, ng.Log, ng.DataSourceCache, ng.SecretsService, ng.ExpressionService),
-		InstanceStore:           store,
-		RuleStore:               store,
-		AdminConfigStore:        store,
-		OrgStore:                store,
-		MultiOrgNotifier:        ng.MultiOrgAlertmanager,
-		Metrics:                 ng.Metrics.GetSchedulerMetrics(),
-		AdminConfigPollInterval: ng.Cfg.UnifiedAlerting.AdminConfigPollInterval,
-		DisabledOrgs:            ng.Cfg.UnifiedAlerting.DisabledOrgs,
-		MinRuleInterval:         ng.Cfg.UnifiedAlerting.MinInterval,
+		Cfg:              ng.Cfg.UnifiedAlerting,
+		C:                clock.New(),
+		Logger:           ng.Log,
+		Evaluator:        eval.NewEvaluator(ng.Cfg, ng.Log, ng.DataSourceCache, ng.SecretsService, ng.ExpressionService),
+		InstanceStore:    store,
+		RuleStore:        store,
+		AdminConfigStore: store,
+		OrgStore:         store,
+		MultiOrgNotifier: ng.MultiOrgAlertmanager,
+		Metrics:          ng.Metrics.GetSchedulerMetrics(),
 	}
 
 	appUrl, err := url.Parse(ng.Cfg.AppURL)

pkg/services/ngalert/schedule/schedule.go

@@ -21,6 +21,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/sender"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/services/ngalert/store"
+	"github.com/grafana/grafana/pkg/setting"
 
 	"github.com/benbjohnson/clock"
 	"golang.org/x/sync/errgroup"
@@ -89,7 +90,8 @@ type schedule struct {
 
 	stateManager *state.Manager
 
-	appURL *url.URL
+	appURL               *url.URL
+	disableGrafanaFolder bool
 
 	multiOrgNotifier *notifier.MultiOrgAlertmanager
 	metrics          *metrics.Scheduler
@@ -115,33 +117,29 @@ type schedule struct {
 
 // SchedulerCfg is the scheduler configuration.
 type SchedulerCfg struct {
-	C                       clock.Clock
-	BaseInterval            time.Duration
-	Logger                  log.Logger
-	EvalAppliedFunc         func(ngmodels.AlertRuleKey, time.Time)
-	MaxAttempts             int64
-	StopAppliedFunc         func(ngmodels.AlertRuleKey)
-	Evaluator               eval.Evaluator
-	RuleStore               store.RuleStore
-	OrgStore                store.OrgStore
-	InstanceStore           store.InstanceStore
-	AdminConfigStore        store.AdminConfigurationStore
-	MultiOrgNotifier        *notifier.MultiOrgAlertmanager
-	Metrics                 *metrics.Scheduler
-	AdminConfigPollInterval time.Duration
-	DisabledOrgs            map[int64]struct{}
-	MinRuleInterval         time.Duration
+	Cfg              setting.UnifiedAlertingSettings
+	C                clock.Clock
+	Logger           log.Logger
+	EvalAppliedFunc  func(ngmodels.AlertRuleKey, time.Time)
+	StopAppliedFunc  func(ngmodels.AlertRuleKey)
+	Evaluator        eval.Evaluator
+	RuleStore        store.RuleStore
+	OrgStore         store.OrgStore
+	InstanceStore    store.InstanceStore
+	AdminConfigStore store.AdminConfigurationStore
+	MultiOrgNotifier *notifier.MultiOrgAlertmanager
+	Metrics          *metrics.Scheduler
 }
 
 // NewScheduler returns a new schedule.
 func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager, bus bus.Bus) *schedule {
-	ticker := alerting.NewTicker(cfg.C, cfg.BaseInterval, cfg.Metrics.Ticker)
+	ticker := alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
 
 	sch := schedule{
 		registry:                alertRuleInfoRegistry{alertRuleInfo: make(map[ngmodels.AlertRuleKey]*alertRuleInfo)},
-		maxAttempts:             cfg.MaxAttempts,
+		maxAttempts:             cfg.Cfg.MaxAttempts,
 		clock:                   cfg.C,
-		baseInterval:            cfg.BaseInterval,
+		baseInterval:            cfg.Cfg.BaseInterval,
 		log:                     cfg.Logger,
 		ticker:                  ticker,
 		evalAppliedFunc:         cfg.EvalAppliedFunc,
@@ -154,13 +152,14 @@ func NewScheduler(cfg SchedulerCfg, appURL *url.URL, stateManager *state.Manager
 		multiOrgNotifier:        cfg.MultiOrgNotifier,
 		metrics:                 cfg.Metrics,
 		appURL:                  appURL,
+		disableGrafanaFolder:    cfg.Cfg.ReservedLabels.IsReservedLabelDisabled(ngmodels.FolderTitleLabel),
 		stateManager:            stateManager,
 		sendAlertsTo:            map[int64]ngmodels.AlertmanagersChoice{},
 		senders:                 map[int64]*sender.Sender{},
 		sendersCfgHash:          map[int64]string{},
-		adminConfigPollInterval: cfg.AdminConfigPollInterval,
-		disabledOrgs:            cfg.DisabledOrgs,
-		minRuleInterval:         cfg.MinRuleInterval,
+		adminConfigPollInterval: cfg.Cfg.AdminConfigPollInterval,
+		disabledOrgs:            cfg.Cfg.DisabledOrgs,
+		minRuleInterval:         cfg.Cfg.MinInterval,
 		schedulableAlertRules:   schedulableAlertRulesRegistry{rules: make(map[ngmodels.AlertRuleKey]*ngmodels.SchedulableAlertRule)},
 		bus:                     bus,
 	}
@@ -600,18 +599,20 @@ func (sch *schedule) ruleRoutine(grafanaCtx context.Context, key ngmodels.AlertR
 			OrgId:   key.OrgID,
 		}
 
-		folder, err := sch.ruleStore.GetNamespaceByUID(ctx, q.Result.NamespaceUID, q.Result.OrgID, user)
-		if err != nil {
-			logger.Error("failed to fetch alert rule namespace", "err", err)
-			return nil, err
-		}
+		if !sch.disableGrafanaFolder {
+			folder, err := sch.ruleStore.GetNamespaceByUID(ctx, q.Result.NamespaceUID, q.Result.OrgID, user)
+			if err != nil {
+				logger.Error("failed to fetch alert rule namespace", "err", err)
+				return nil, err
+			}
 
-		if q.Result.Labels == nil {
-			q.Result.Labels = make(map[string]string)
-		} else if val, ok := q.Result.Labels[ngmodels.FolderTitleLabel]; ok {
-			logger.Warn("alert rule contains protected label, value will be overwritten", "label", ngmodels.FolderTitleLabel, "value", val)
+			if q.Result.Labels == nil {
+				q.Result.Labels = make(map[string]string)
+			} else if val, ok := q.Result.Labels[ngmodels.FolderTitleLabel]; ok {
+				logger.Warn("alert rule contains protected label, value will be overwritten", "label", ngmodels.FolderTitleLabel, "value", val)
+			}
+			q.Result.Labels[ngmodels.FolderTitleLabel] = folder.Title
 		}
-		q.Result.Labels[ngmodels.FolderTitleLabel] = folder.Title
 
 		return q.Result, nil
 	}
@@ -740,15 +741,18 @@ func (sch *schedule) saveAlertStates(ctx context.Context, states []*state.State)
 
 // folderUpdateHandler listens for folder update events and updates all rules in the given folder.
 func (sch *schedule) folderUpdateHandler(ctx context.Context, evt *events.FolderUpdated) error {
+	if sch.disableGrafanaFolder {
+		return nil
+	}
 	return sch.UpdateAlertRulesByNamespaceUID(ctx, evt.OrgID, evt.UID)
 }
 
 // overrideCfg is only used on tests.
 func (sch *schedule) overrideCfg(cfg SchedulerCfg) {
 	sch.clock = cfg.C
-	sch.baseInterval = cfg.BaseInterval
+	sch.baseInterval = cfg.Cfg.BaseInterval
 	sch.ticker.Stop()
-	sch.ticker = alerting.NewTicker(cfg.C, cfg.BaseInterval, cfg.Metrics.Ticker)
+	sch.ticker = alerting.NewTicker(cfg.C, cfg.Cfg.BaseInterval, cfg.Metrics.Ticker)
 	sch.evalAppliedFunc = cfg.EvalAppliedFunc
 	sch.stopAppliedFunc = cfg.StopAppliedFunc
 }

pkg/services/ngalert/schedule/schedule_test.go

@@ -27,6 +27,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/ngalert/schedule"
 	"github.com/grafana/grafana/pkg/services/ngalert/state"
 	"github.com/grafana/grafana/pkg/services/ngalert/tests"
+	"github.com/grafana/grafana/pkg/setting"
 )
 
 var testMetrics = metrics.NewNGAlert(prometheus.NewPedanticRegistry())
@@ -98,16 +99,19 @@ func TestWarmStateCache(t *testing.T) {
 	}
 	_ = dbstore.SaveAlertInstance(ctx, saveCmd2)
 
-	schedCfg := schedule.SchedulerCfg{
-		C:            clock.NewMock(),
-		BaseInterval: time.Second,
-		Logger:       log.New("ngalert cache warming test"),
-
-		RuleStore:               dbstore,
-		InstanceStore:           dbstore,
-		Metrics:                 testMetrics.GetSchedulerMetrics(),
+	cfg := setting.UnifiedAlertingSettings{
+		BaseInterval:            time.Second,
 		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
 	}
+
+	schedCfg := schedule.SchedulerCfg{
+		Cfg:           cfg,
+		C:             clock.NewMock(),
+		Logger:        log.New("ngalert cache warming test"),
+		RuleStore:     dbstore,
+		InstanceStore: dbstore,
+		Metrics:       testMetrics.GetSchedulerMetrics(),
+	}
 	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
 	st.Warm(ctx)
 
@@ -140,25 +144,28 @@ func TestAlertingTicker(t *testing.T) {
 	stopAppliedCh := make(chan models.AlertRuleKey, len(alerts))
 
 	mockedClock := clock.NewMock()
-	baseInterval := time.Second
+
+	cfg := setting.UnifiedAlertingSettings{
+		BaseInterval:            time.Second,
+		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
+		DisabledOrgs: map[int64]struct{}{
+			disabledOrgID: {},
+		},
+	}
 
 	schedCfg := schedule.SchedulerCfg{
-		C:            mockedClock,
-		BaseInterval: baseInterval,
+		Cfg: cfg,
+		C:   mockedClock,
 		EvalAppliedFunc: func(alertDefKey models.AlertRuleKey, now time.Time) {
 			evalAppliedCh <- evalAppliedInfo{alertDefKey: alertDefKey, now: now}
 		},
 		StopAppliedFunc: func(alertDefKey models.AlertRuleKey) {
 			stopAppliedCh <- alertDefKey
 		},
-		RuleStore:               dbstore,
-		InstanceStore:           dbstore,
-		Logger:                  log.New("ngalert schedule test"),
-		Metrics:                 testMetrics.GetSchedulerMetrics(),
-		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
-		DisabledOrgs: map[int64]struct{}{
-			disabledOrgID: {},
-		},
+		RuleStore:     dbstore,
+		InstanceStore: dbstore,
+		Logger:        log.New("ngalert schedule test"),
+		Metrics:       testMetrics.GetSchedulerMetrics(),
 	}
 	st := state.NewManager(schedCfg.Logger, testMetrics.GetStateMetrics(), nil, dbstore, dbstore, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
 	appUrl := &url.URL{

pkg/services/ngalert/schedule/schedule_unit_test.go

@@ -945,19 +945,23 @@ func setupScheduler(t *testing.T, rs store.RuleStore, is store.InstanceStore, ac
 	moa, err := notifier.NewMultiOrgAlertmanager(&setting.Cfg{}, &notifier.FakeConfigStore{}, &notifier.FakeOrgStore{}, &notifier.FakeKVStore{}, provisioning.NewFakeProvisioningStore(), decryptFn, m.GetMultiOrgAlertmanagerMetrics(), nil, log.New("testlogger"), secretsService)
 	require.NoError(t, err)
 
-	schedCfg := SchedulerCfg{
-		C:                       mockedClock,
+	cfg := setting.UnifiedAlertingSettings{
 		BaseInterval:            time.Second,
 		MaxAttempts:             1,
-		Evaluator:               eval.NewEvaluator(&setting.Cfg{ExpressionsEnabled: true}, logger, nil, secretsService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil)),
-		RuleStore:               rs,
-		InstanceStore:           is,
-		AdminConfigStore:        acs,
-		MultiOrgNotifier:        moa,
-		Logger:                  logger,
-		Metrics:                 m.GetSchedulerMetrics(),
 		AdminConfigPollInterval: 10 * time.Minute, // do not poll in unit tests.
 	}
+
+	schedCfg := SchedulerCfg{
+		Cfg:              cfg,
+		C:                mockedClock,
+		Evaluator:        eval.NewEvaluator(&setting.Cfg{ExpressionsEnabled: true}, logger, nil, secretsService, expr.ProvideService(&setting.Cfg{ExpressionsEnabled: true}, nil, nil)),
+		RuleStore:        rs,
+		InstanceStore:    is,
+		AdminConfigStore: acs,
+		MultiOrgNotifier: moa,
+		Logger:           logger,
+		Metrics:          m.GetSchedulerMetrics(),
+	}
 	st := state.NewManager(schedCfg.Logger, m.GetStateMetrics(), nil, rs, is, &dashboards.FakeDashboardService{}, &image.NoopImageService{}, clock.NewMock())
 	appUrl := &url.URL{
 		Scheme: "http",

pkg/setting/setting_unified_alerting.go

@@ -82,6 +82,7 @@ type UnifiedAlertingSettings struct {
 	// DefaultRuleEvaluationInterval default interval between evaluations of a rule.
 	DefaultRuleEvaluationInterval time.Duration
 	Screenshots                   UnifiedAlertingScreenshotSettings
+	ReservedLabels                UnifiedAlertingReservedLabelSettings
 }
 
 type UnifiedAlertingScreenshotSettings struct {
@@ -90,12 +91,22 @@ type UnifiedAlertingScreenshotSettings struct {
 	UploadExternalImageStorage bool
 }
 
+type UnifiedAlertingReservedLabelSettings struct {
+	DisabledLabels map[string]struct{}
+}
+
 // IsEnabled returns true if UnifiedAlertingSettings.Enabled is either nil or true.
 // It hides the implementation details of the Enabled and simplifies its usage.
 func (u *UnifiedAlertingSettings) IsEnabled() bool {
 	return u.Enabled == nil || *u.Enabled
 }
 
+// IsReservedLabelDisabled returns true if UnifiedAlertingReservedLabelSettings.DisabledLabels contains the given reserved label.
+func (u *UnifiedAlertingReservedLabelSettings) IsReservedLabelDisabled(label string) bool {
+	_, ok := u.DisabledLabels[label]
+	return ok
+}
+
 // readUnifiedAlertingEnabledSettings reads the settings for unified alerting.
 // It returns a non-nil bool and a nil error when unified alerting is enabled either
 // because it has been enabled in the settings or by default. It returns nil and
@@ -274,6 +285,15 @@ func (cfg *Cfg) ReadUnifiedAlertingSettings(iniFile *ini.File) error {
 	uaCfgScreenshots.UploadExternalImageStorage = screenshots.Key("upload_external_image_storage").MustBool(screenshotsDefaultUploadImageStorage)
 	uaCfg.Screenshots = uaCfgScreenshots
 
+	reservedLabels := iniFile.Section("unified_alerting.reserved_labels")
+	uaCfgReservedLabels := UnifiedAlertingReservedLabelSettings{
+		DisabledLabels: make(map[string]struct{}),
+	}
+	for _, label := range util.SplitString(reservedLabels.Key("disabled_labels").MustString("")) {
+		uaCfgReservedLabels.DisabledLabels[label] = struct{}{}
+	}
+	uaCfg.ReservedLabels = uaCfgReservedLabels
+
 	cfg.UnifiedAlerting = uaCfg
 	return nil
 }