Alerting: Add locking to avoid race conditions when creating alert rule folders

pull/101691/head
Alexander Akhmetov 5 months ago
parent 7acfdfa0af
commit c9cdfc51a6
No known key found for this signature in database
GPG Key ID: A5A8947133B1B31B
  1. 2
      pkg/services/cloudmigration/cloudmigrationimpl/cloudmigration_test.go
  2. 9
      pkg/services/ngalert/api/api.go
  3. 43
      pkg/services/ngalert/api/api_convert_prometheus.go
  4. 4
      pkg/services/ngalert/api/persist.go
  5. 7
      pkg/services/ngalert/ngalert.go
  6. 14
      pkg/services/ngalert/store/alert_rule_test.go
  7. 145
      pkg/services/ngalert/store/namespace.go
  8. 142
      pkg/services/ngalert/store/namespace_test.go
  9. 135
      pkg/services/ngalert/tests/fakes/namespaces.go
  10. 3
      pkg/services/ngalert/tests/util.go
  11. 4
      pkg/services/quota/quotaimpl/quota_test.go

@ -23,6 +23,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/plugins"
"github.com/grafana/grafana/pkg/services/accesscontrol/actest"
@ -880,6 +881,7 @@ func setUpServiceTest(t *testing.T, withDashboardMock bool, cfgOverrides ...conf
secretsService, nil, alertMetrics, mockFolder, fakeAccessControl, dashboardService, nil, bus, fakeAccessControlService,
annotationstest.NewFakeAnnotationsRepo(), &pluginstore.FakePluginStore{}, tracer, ruleStore,
httpclient.NewProvider(), ngalertfakes.NewFakeReceiverPermissionsService(), usertest.NewUserServiceFake(),
serverlock.ProvideService(sqlStore, tracer),
)
require.NoError(t, err)

@ -80,6 +80,7 @@ type API struct {
Tracer tracing.Tracer
AppUrl *url.URL
UserService user.Service
AlertingFolderService alertingFolderService
// Hooks can be used to replace API handlers for specific paths.
Hooks *Hooks
@ -188,7 +189,13 @@ func (api *API) RegisterAPIEndpoints(m *metrics.API) {
if api.FeatureManager.IsEnabledGlobally(featuremgmt.FlagAlertingConversionAPI) {
api.RegisterConvertPrometheusApiEndpoints(NewConvertPrometheusApi(
NewConvertPrometheusSrv(&api.Cfg.UnifiedAlerting, logger, api.RuleStore, api.DatasourceCache, api.AlertRules),
NewConvertPrometheusSrv(
&api.Cfg.UnifiedAlerting,
logger,
api.AlertingFolderService,
api.DatasourceCache,
api.AlertRules,
),
), m)
}
}

@ -1,6 +1,7 @@
package api
import (
"context"
"errors"
"fmt"
"net/http"
@ -14,6 +15,7 @@ import (
"github.com/grafana/grafana/pkg/api/response"
"github.com/grafana/grafana/pkg/apimachinery/errutil"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/log"
contextmodel "github.com/grafana/grafana/pkg/services/contexthandler/model"
"github.com/grafana/grafana/pkg/services/dashboards"
@ -54,6 +56,13 @@ func errInvalidHeaderValue(header string) error {
return errInvalidHeaderValueBase.Build(errutil.TemplateData{Public: map[string]any{"Header": header}})
}
type alertingFolderService interface {
GetNamespaceByTitle(ctx context.Context, fullpath string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error)
GetOrCreateNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error)
// GetNamespaceChildren returns all children (first level) of the namespace with the given id.
GetNamespaceChildren(ctx context.Context, uid string, orgID int64, user identity.Requester) ([]*folder.Folder, error)
}
// ConvertPrometheusSrv converts Prometheus rules to Grafana rules
// and retrieves them in a Prometheus-compatible format.
//
@ -95,16 +104,22 @@ func errInvalidHeaderValue(header string) error {
type ConvertPrometheusSrv struct {
cfg *setting.UnifiedAlertingSettings
logger log.Logger
ruleStore RuleStore
folderService alertingFolderService
datasourceCache datasources.CacheService
alertRuleService *provisioning.AlertRuleService
}
func NewConvertPrometheusSrv(cfg *setting.UnifiedAlertingSettings, logger log.Logger, ruleStore RuleStore, datasourceCache datasources.CacheService, alertRuleService *provisioning.AlertRuleService) *ConvertPrometheusSrv {
func NewConvertPrometheusSrv(
cfg *setting.UnifiedAlertingSettings,
logger log.Logger,
folderService alertingFolderService,
datasourceCache datasources.CacheService,
alertRuleService *provisioning.AlertRuleService,
) *ConvertPrometheusSrv {
return &ConvertPrometheusSrv{
cfg: cfg,
logger: logger,
ruleStore: ruleStore,
folderService: folderService,
datasourceCache: datasourceCache,
alertRuleService: alertRuleService,
}
@ -119,7 +134,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusGetRules(c *contextmodel.
workingFolderUID := getWorkingFolderUID(c)
logger = logger.New("working_folder_uid", workingFolderUID)
folders, err := srv.ruleStore.GetNamespaceChildren(c.Req.Context(), workingFolderUID, c.SignedInUser.GetOrgID(), c.SignedInUser)
folders, err := srv.folderService.GetNamespaceChildren(c.Req.Context(), workingFolderUID, c.SignedInUser.GetOrgID(), c.SignedInUser)
if len(folders) == 0 || errors.Is(err, dashboards.ErrFolderNotFound) {
// If there is no such folder or no children, return empty response
// because mimirtool expects 200 OK response in this case.
@ -162,7 +177,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusDeleteNamespace(c *contex
logger = logger.New("working_folder_uid", workingFolderUID)
logger.Debug("Looking up folder by title", "folder_title", namespaceTitle)
namespace, err := srv.ruleStore.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
namespace, err := srv.folderService.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
if err != nil {
return namespaceErrorResponse(err)
}
@ -192,7 +207,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusDeleteRuleGroup(c *contex
logger = logger.New("working_folder_uid", workingFolderUID)
logger.Debug("Looking up folder by title", "folder_title", namespaceTitle)
folder, err := srv.ruleStore.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
folder, err := srv.folderService.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
if err != nil {
return namespaceErrorResponse(err)
}
@ -219,7 +234,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusGetNamespace(c *contextmo
logger = logger.New("working_folder_uid", workingFolderUID)
logger.Debug("Looking up folder by title", "folder_title", namespaceTitle)
namespace, err := srv.ruleStore.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
namespace, err := srv.folderService.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
if err != nil {
logger.Error("Failed to get folder", "error", err)
return namespaceErrorResponse(err)
@ -253,7 +268,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusGetRuleGroup(c *contextmo
logger = logger.New("working_folder_uid", workingFolderUID)
logger.Debug("Looking up folder by title", "folder_title", namespaceTitle)
namespace, err := srv.ruleStore.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
namespace, err := srv.folderService.GetNamespaceByTitle(c.Req.Context(), namespaceTitle, c.SignedInUser.GetOrgID(), c.SignedInUser, workingFolderUID)
if err != nil {
logger.Error("Failed to get folder", "error", err)
return namespaceErrorResponse(err)
@ -303,11 +318,6 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm
logger.Info("Converting Prometheus rule group", "rules", len(promGroup.Rules))
ns, errResp := srv.getOrCreateNamespace(c, namespaceTitle, logger, workingFolderUID)
if errResp != nil {
return errResp
}
datasourceUID := strings.TrimSpace(c.Req.Header.Get(datasourceUIDHeader))
if datasourceUID == "" {
return response.Err(errDatasourceUIDHeaderMissing)
@ -318,6 +328,11 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm
return errorToResponse(err)
}
ns, errResp := srv.getOrCreateNamespace(c, namespaceTitle, logger, workingFolderUID)
if errResp != nil {
return errResp
}
group, err := srv.convertToGrafanaRuleGroup(c, ds, ns.UID, promGroup, logger)
if err != nil {
logger.Error("Failed to convert Prometheus rules to Grafana rules", "error", err)
@ -336,7 +351,7 @@ func (srv *ConvertPrometheusSrv) RouteConvertPrometheusPostRuleGroup(c *contextm
func (srv *ConvertPrometheusSrv) getOrCreateNamespace(c *contextmodel.ReqContext, title string, logger log.Logger, workingFolderUID string) (*folder.Folder, response.Response) {
logger.Debug("Getting or creating a new folder")
ns, err := srv.ruleStore.GetOrCreateNamespaceByTitle(
ns, err := srv.folderService.GetOrCreateNamespaceByTitle(
c.Req.Context(),
title,
c.SignedInUser.GetOrgID(),

@ -15,10 +15,6 @@ type RuleStore interface {
// by returning map[string]struct{} instead of map[string]*folder.Folder
GetUserVisibleNamespaces(context.Context, int64, identity.Requester) (map[string]*folder.Folder, error)
GetNamespaceByUID(ctx context.Context, uid string, orgID int64, user identity.Requester) (*folder.Folder, error)
GetNamespaceByTitle(ctx context.Context, fullpath string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error)
GetOrCreateNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error)
// GetNamespaceChildren returns all children (first level) of the namespace with the given id.
GetNamespaceChildren(ctx context.Context, uid string, orgID int64, user identity.Requester) ([]*folder.Folder, error)
GetAlertRuleByUID(ctx context.Context, query *ngmodels.GetAlertRuleByUIDQuery) (*ngmodels.AlertRule, error)
GetAlertRulesGroupByRuleUID(ctx context.Context, query *ngmodels.GetAlertRulesGroupByRuleUIDQuery) ([]*ngmodels.AlertRule, error)

@ -19,6 +19,7 @@ import (
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/infra/tracing"
"github.com/grafana/grafana/pkg/services/accesscontrol"
"github.com/grafana/grafana/pkg/services/annotations"
@ -80,6 +81,7 @@ func ProvideService(
httpClientProvider httpclient.Provider,
resourcePermissions accesscontrol.ReceiverPermissionsService,
userService user.Service,
serverLockService *serverlock.ServerLockService,
) (*AlertNG, error) {
ng := &AlertNG{
Cfg: cfg,
@ -109,6 +111,7 @@ func ProvideService(
httpClientProvider: httpClientProvider,
ResourcePermissions: resourcePermissions,
userService: userService,
serverLockService: serverLockService,
}
if ng.IsDisabled() {
@ -164,6 +167,7 @@ type AlertNG struct {
bus bus.Bus
pluginsStore pluginstore.Store
tracer tracing.Tracer
serverLockService *serverlock.ServerLockService
}
func (ng *AlertNG) init() error {
@ -406,6 +410,8 @@ func (ng *AlertNG) init() error {
return err
}
alertingFolderService := store.NewAlertingFolderService(ng.folderService, ng.Log, ng.serverLockService)
ng.InstanceStore, ng.StartupInstanceReader = initInstanceStore(ng.store.SQLStore, ng.Log, ng.FeatureToggles)
stateManagerCfg := state.ManagerCfg{
@ -502,6 +508,7 @@ func (ng *AlertNG) init() error {
Hooks: api.NewHooks(ng.Log),
Tracer: ng.tracer,
UserService: ng.userService,
AlertingFolderService: alertingFolderService,
}
ng.Api.RegisterAPIEndpoints(ng.Metrics.GetAPIMetrics())

@ -508,15 +508,15 @@ func TestIntegration_GetAlertRulesForScheduling(t *testing.T) {
parentFolderUid := uuid.NewString()
parentFolderTitle := "Very Parent Folder"
createFolder(t, store, parentFolderUid, parentFolderTitle, rule1.OrgID, "")
createFolder(t, store.FolderService, parentFolderUid, parentFolderTitle, rule1.OrgID, "")
rule1FolderTitle := "folder-" + rule1.Title
rule2FolderTitle := "folder-" + rule2.Title
rule3FolderTitle := "folder-" + rule3.Title
createFolder(t, store, rule1.NamespaceUID, rule1FolderTitle, rule1.OrgID, parentFolderUid)
createFolder(t, store, rule2.NamespaceUID, rule2FolderTitle, rule2.OrgID, "")
createFolder(t, store, rule3.NamespaceUID, rule3FolderTitle, rule3.OrgID, "")
createFolder(t, store.FolderService, rule1.NamespaceUID, rule1FolderTitle, rule1.OrgID, parentFolderUid)
createFolder(t, store.FolderService, rule2.NamespaceUID, rule2FolderTitle, rule2.OrgID, "")
createFolder(t, store.FolderService, rule3.NamespaceUID, rule3FolderTitle, rule3.OrgID, "")
createFolder(t, store, rule2.NamespaceUID, "same UID folder", gen.GenerateRef().OrgID, "") // create a folder with the same UID but in the different org
createFolder(t, store.FolderService, rule2.NamespaceUID, "same UID folder", gen.GenerateRef().OrgID, "") // create a folder with the same UID but in the different org
tc := []struct {
name string
@ -1739,7 +1739,7 @@ func createRule(t *testing.T, store *DBstore, generator *models.AlertRuleGenerat
return rule
}
func createFolder(t *testing.T, store *DBstore, uid, title string, orgID int64, parentUID string) {
func createFolder(t *testing.T, folderService folder.Service, uid, title string, orgID int64, parentUID string) {
t.Helper()
u := &user.SignedInUser{
UserID: 1,
@ -1748,7 +1748,7 @@ func createFolder(t *testing.T, store *DBstore, uid, title string, orgID int64,
IsGrafanaAdmin: true,
}
_, err := store.FolderService.Create(context.Background(), &folder.CreateFolderCommand{
_, err := folderService.Create(context.Background(), &folder.CreateFolderCommand{
UID: uid,
OrgID: orgID,
Title: title,

@ -3,13 +3,30 @@ package store
import (
"context"
"errors"
"fmt"
"hash/fnv"
"sort"
"time"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
)
const (
// folderOperationTimeout is the timeout for individual folder operations (get or create)
// in the GetOrCreateNamespaceByTitle method. The lock timeout is longer to accommodate
// both operations.
folderOperationTimeout = 60 * time.Second
// getOrCreateFolderMaxRetries is the maximum number of retries allowed when
// trying to acquire a lock for folder creation. After this many failed attempts,
// the operation will fail with a "max retries exceeded" error.
getOrCreateFolderMaxRetries = 10
)
// GetUserVisibleNamespaces returns the folders that are visible to the user
func (st DBstore) GetUserVisibleNamespaces(ctx context.Context, orgID int64, user identity.Requester) (map[string]*folder.Folder, error) {
folders, err := st.FolderService.GetFolders(ctx, folder.GetFoldersQuery{
@ -40,14 +57,34 @@ func (st DBstore) GetNamespaceByUID(ctx context.Context, uid string, orgID int64
return f[0], nil
}
// ServerLockService defines the interface for distributed locking functionality
type ServerLockService interface {
// LockExecuteAndReleaseWithRetries acquires a lock, executes a function, and releases the lock with retries
LockExecuteAndReleaseWithRetries(ctx context.Context, actionName string, timeConfig serverlock.LockTimeConfig, fn func(ctx context.Context), retryOpts ...serverlock.RetryOpt) error
}
type AlertingFolderService struct {
FolderService folder.Service
Logger log.Logger
ServerLockService ServerLockService
}
func NewAlertingFolderService(fs folder.Service, logger log.Logger, serverLockService ServerLockService) *AlertingFolderService {
return &AlertingFolderService{
FolderService: fs,
Logger: logger,
ServerLockService: serverLockService,
}
}
// GetNamespaceChildren gets namespace (folder) children (first level) by its UID.
func (st DBstore) GetNamespaceChildren(ctx context.Context, uid string, orgID int64, user identity.Requester) ([]*folder.Folder, error) {
func (s AlertingFolderService) GetNamespaceChildren(ctx context.Context, uid string, orgID int64, user identity.Requester) ([]*folder.Folder, error) {
q := &folder.GetChildrenQuery{
UID: uid,
OrgID: orgID,
SignedInUser: user,
}
folders, err := st.FolderService.GetChildren(ctx, q)
folders, err := s.FolderService.GetChildren(ctx, q)
if err != nil {
return nil, err
}
@ -63,8 +100,8 @@ func (st DBstore) GetNamespaceChildren(ctx context.Context, uid string, orgID in
}
// GetNamespaceByTitle gets namespace by its title in the specified folder.
func (st DBstore) GetNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
folders, err := st.GetNamespaceChildren(ctx, parentUID, orgID, user)
func (s AlertingFolderService) GetNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
folders, err := s.GetNamespaceChildren(ctx, parentUID, orgID, user)
if err != nil {
return nil, err
}
@ -88,28 +125,102 @@ func (st DBstore) GetNamespaceByTitle(ctx context.Context, title string, orgID i
return foundByTitle[0], nil
}
// GetOrCreateNamespaceByTitle gets or creates a namespace by title in the specified folder.
func (st DBstore) GetOrCreateNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
var f *folder.Folder
var err error
// GetOrCreateNamespaceByTitle retrieves a folder with the given title from the specified parent,
// or creates it if it doesn't exist.
//
// This method uses locking to prevent race conditions when multiple
// requests attempt to create the same folder simultaneously. The lock is based on
// the combination of parent folder UID, title, and organization ID.
func (s AlertingFolderService) GetOrCreateNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
logger := s.Logger.New("parentUID", parentUID, "title", title, "orgID", orgID)
f, err = st.GetNamespaceByTitle(ctx, title, orgID, user, parentUID)
if err != nil && !errors.Is(err, dashboards.ErrFolderNotFound) {
return nil, err
// Configure lock retry behavior
// Make sure the lock timeout (MaxInterval) is enough for both operations.
timeConfig := serverlock.LockTimeConfig{
MaxInterval: folderOperationTimeout*2 + 10*time.Second,
MinWait: 100 * time.Millisecond,
MaxWait: 1 * time.Second,
}
var folder *folder.Folder
var folderErr error
retryLimiter := func(attempt int) error {
if attempt > getOrCreateFolderMaxRetries {
return errors.New("unable to lock: max retries exceeded")
}
return nil
}
// Execute the folder get/create operation with a lock
lockName, err := lockName(parentUID, title, orgID)
if err != nil {
return nil, fmt.Errorf("failed to generate lock name: %w", err)
}
logger.Debug("Acquiring lock for folder creation", "lockName", lockName)
errLock := s.ServerLockService.LockExecuteAndReleaseWithRetries(ctx, lockName, timeConfig, func(ctx context.Context) {
// Try to get the folder
logger.Debug("Trying to get existing folder")
folder, folderErr = s.getFolder(ctx, title, orgID, user, parentUID, folderOperationTimeout)
if folder != nil {
return
}
// If this is not the folder not found error, return
if !errors.Is(folderErr, dashboards.ErrFolderNotFound) {
return
}
// Folder doesn't exist, create a new one
logger.Debug("Folder not found, creating a new one")
folder, folderErr = s.createFolder(ctx, title, orgID, user, parentUID, folderOperationTimeout)
}, []serverlock.RetryOpt{retryLimiter}...)
// Handle lock acquisition failures
if errLock != nil {
logger.Error("Failed to acquire or execute with lock", "error", errLock)
return nil, fmt.Errorf("failed to acquire lock: %w", errLock)
}
// Handle folder operation errors
if folderErr != nil {
return nil, folderErr
}
if folder == nil {
// This should never happen if the code is correct
logger.Error("Both error and folder are nil after GetOrCreateNamespaceByTitle execution")
return nil, fmt.Errorf("unexpected error: could not get or create a folder")
}
return folder, nil
}
if f == nil {
func (s AlertingFolderService) getFolder(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string, timeout time.Duration) (*folder.Folder, error) {
getCtx, getCancel := context.WithTimeout(ctx, timeout)
defer getCancel()
return s.GetNamespaceByTitle(getCtx, title, orgID, user, parentUID)
}
func (s AlertingFolderService) createFolder(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string, timeout time.Duration) (*folder.Folder, error) {
createCtx, createCancel := context.WithTimeout(ctx, timeout)
defer createCancel()
cmd := &folder.CreateFolderCommand{
OrgID: orgID,
Title: title,
SignedInUser: user,
ParentUID: parentUID,
}
f, err = st.FolderService.Create(ctx, cmd)
if err != nil {
return nil, err
}
return s.FolderService.Create(createCtx, cmd)
}
return f, nil
func lockName(parentUID, title string, orgID int64) (string, error) {
h := fnv.New64a()
data := fmt.Sprintf("%s|%s|%d", parentUID, title, orgID)
_, err := h.Write([]byte(data))
if err != nil {
return "", err
}
return fmt.Sprintf("alerting-folder-create-%x", h.Sum64()), nil
}

@ -2,21 +2,23 @@ package store
import (
"context"
"errors"
"fmt"
"testing"
"github.com/google/uuid"
"github.com/stretchr/testify/require"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/featuremgmt"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/services/org"
"github.com/grafana/grafana/pkg/services/user"
"github.com/grafana/grafana/pkg/util"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/setting"
"github.com/grafana/grafana/pkg/util"
)
func TestIntegration_GetUserVisibleNamespaces(t *testing.T) {
@ -49,7 +51,7 @@ func TestIntegration_GetUserVisibleNamespaces(t *testing.T) {
}
for _, f := range folders {
createFolder(t, store, f.uid, f.title, 1, f.parentUid)
createFolder(t, folderService, f.uid, f.title, 1, f.parentUid)
}
t.Run("returns all folders", func(t *testing.T) {
@ -89,8 +91,8 @@ func TestIntegration_GetNamespaceByUID(t *testing.T) {
parentUid := uuid.NewString()
title := "folder/title"
parentTitle := "parent-title"
createFolder(t, store, parentUid, parentTitle, 1, "")
createFolder(t, store, uid, title, 1, parentUid)
createFolder(t, folderService, parentUid, parentTitle, 1, "")
createFolder(t, folderService, uid, title, 1, parentUid)
actual, err := store.GetNamespaceByUID(context.Background(), uid, 1, u)
require.NoError(t, err)
@ -132,10 +134,7 @@ func TestIntegration_GetNamespaceByTitle(t *testing.T) {
sqlStore := db.InitTestDB(t)
cfg := setting.NewCfg()
folderService := setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures())
b := &fakeBus{}
logger := log.New("test-dbstore")
store := createTestStore(sqlStore, folderService, logger, cfg.UnifiedAlerting, b)
store.FolderService = setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures(featuremgmt.FlagNestedFolders))
store := createAlertingFolderService(t, folderService, &mockServerLockService{})
u := &user.SignedInUser{
UserID: 1,
@ -147,16 +146,16 @@ func TestIntegration_GetNamespaceByTitle(t *testing.T) {
// Create parent folder
parentUID := uuid.NewString()
parentTitle := "parent-folder"
createFolder(t, store, parentUID, parentTitle, 1, "")
createFolder(t, folderService, parentUID, parentTitle, 1, "")
// Create child folder under parent
childUID := uuid.NewString()
childTitle := "child-folder"
createFolder(t, store, childUID, childTitle, 1, parentUID)
createFolder(t, folderService, childUID, childTitle, 1, parentUID)
// Create another folder with same title but under root
sameTitleInRoot := uuid.NewString()
createFolder(t, store, sameTitleInRoot, childTitle, 1, "")
createFolder(t, folderService, sameTitleInRoot, childTitle, 1, "")
t.Run("should find folder by title and parent UID", func(t *testing.T) {
actual, err := store.GetNamespaceByTitle(context.Background(), childTitle, 1, u, parentUID)
@ -194,16 +193,12 @@ func TestIntegration_GetOrCreateNamespaceByTitle(t *testing.T) {
IsGrafanaAdmin: true,
}
setupStore := func(t *testing.T) *DBstore {
setupStore := func(t *testing.T) *AlertingFolderService {
sqlStore := db.InitTestDB(t)
cfg := setting.NewCfg()
folderService := setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures())
b := &fakeBus{}
logger := log.New("test-dbstore")
store := createTestStore(sqlStore, folderService, logger, cfg.UnifiedAlerting, b)
store.FolderService = setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures(featuremgmt.FlagNestedFolders))
return store
mockLock := &mockServerLockService{}
return createAlertingFolderService(t, folderService, mockLock)
}
t.Run("should create folder when it does not exist", func(t *testing.T) {
@ -231,7 +226,7 @@ func TestIntegration_GetOrCreateNamespaceByTitle(t *testing.T) {
store := setupStore(t)
title := "existing folder"
createFolder(t, store, "", title, 1, "")
createFolder(t, store.FolderService, "", title, 1, "")
f, err := store.GetOrCreateNamespaceByTitle(context.Background(), title, 1, u, folder.RootFolderUID)
require.NoError(t, err)
require.Equal(t, title, f.Title)
@ -327,10 +322,9 @@ func TestIntegration_GetNamespaceChildren(t *testing.T) {
sqlStore := db.InitTestDB(t)
cfg := setting.NewCfg()
folderService := setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures())
b := &fakeBus{}
logger := log.New("test-dbstore")
store := createTestStore(sqlStore, folderService, logger, cfg.UnifiedAlerting, b)
store.FolderService = setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures(featuremgmt.FlagNestedFolders))
mockLock := &mockServerLockService{}
store := createAlertingFolderService(t, folderService, mockLock)
admin := &user.SignedInUser{
UserID: 1,
@ -342,21 +336,21 @@ func TestIntegration_GetNamespaceChildren(t *testing.T) {
// Create root folders
rootFolder1 := uuid.NewString()
rootFolder2 := uuid.NewString()
createFolder(t, store, rootFolder1, "Root Folder 1", 1, "")
createFolder(t, store, rootFolder2, "Root Folder 2", 1, "")
createFolder(t, folderService, rootFolder1, "Root Folder 1", 1, "")
createFolder(t, folderService, rootFolder2, "Root Folder 2", 1, "")
// Create child folders under root folder 1
child1 := uuid.NewString()
child2 := uuid.NewString()
createFolder(t, store, child1, "Child Folder 1", 1, rootFolder1)
createFolder(t, store, child2, "Child Folder 2", 1, rootFolder1)
createFolder(t, folderService, child1, "Child Folder 1", 1, rootFolder1)
createFolder(t, folderService, child2, "Child Folder 2", 1, rootFolder1)
// Create nested child under child1
nestedChild := uuid.NewString()
createFolder(t, store, nestedChild, "Nested Child", 1, child1)
createFolder(t, folderService, nestedChild, "Nested Child", 1, child1)
differentOrgID := int64(999)
createFolder(t, store, util.GenerateShortUID(), "Root Folder 1", differentOrgID, "")
createFolder(t, folderService, util.GenerateShortUID(), "Root Folder 1", differentOrgID, "")
/*
* Folder structure:
@ -417,3 +411,89 @@ func TestIntegration_GetNamespaceChildren(t *testing.T) {
require.ElementsMatch(t, []string{rootFolder1, rootFolder2}, []string{children[0].UID, children[1].UID})
})
}
func TestServerLockInGetOrCreateNamespace(t *testing.T) {
if testing.Short() {
t.Skip("skipping integration test")
}
u := &user.SignedInUser{
UserID: 1,
OrgID: 1,
OrgRole: org.RoleAdmin,
IsGrafanaAdmin: true,
}
setupStoreWithMockLock := func(t *testing.T, mockLock ServerLockService) *AlertingFolderService {
sqlStore := db.InitTestDB(t)
cfg := setting.NewCfg()
folderService := setupFolderService(t, sqlStore, cfg, featuremgmt.WithFeatures())
return createAlertingFolderService(t, folderService, mockLock)
}
t.Run("should handle lock acquisition failure", func(t *testing.T) {
mockLock := &mockServerLockService{ShouldFail: true}
store := setupStoreWithMockLock(t, mockLock)
_, err := store.GetOrCreateNamespaceByTitle(context.Background(), "new folder", 1, u, folder.RootFolderUID)
require.Error(t, err)
require.Contains(t, err.Error(), "mock lock acquisition failed")
require.True(t, mockLock.LockWasAcquired)
require.Equal(t, 0, mockLock.ExecutionCount)
})
t.Run("should handle lock timeout", func(t *testing.T) {
mockLock := &mockServerLockService{ShouldTimeout: true}
store := setupStoreWithMockLock(t, mockLock)
_, err := store.GetOrCreateNamespaceByTitle(context.Background(), "new folder", 1, u, folder.RootFolderUID)
require.Error(t, err)
require.True(t, errors.Is(err, context.DeadlineExceeded))
require.True(t, mockLock.LockWasAcquired)
require.Equal(t, 0, mockLock.ExecutionCount)
})
t.Run("should successfully use lock when creating folder", func(t *testing.T) {
mockLock := &mockServerLockService{}
store := setupStoreWithMockLock(t, mockLock)
f, err := store.GetOrCreateNamespaceByTitle(context.Background(), "new folder", 1, u, folder.RootFolderUID)
require.NoError(t, err)
require.Equal(t, "new folder", f.Title)
require.True(t, mockLock.LockWasAcquired)
require.Equal(t, 1, mockLock.ExecutionCount)
})
}
type mockServerLockService struct {
LockWasAcquired bool
ExecutionCount int
ShouldFail bool
ShouldTimeout bool
}
func (m *mockServerLockService) LockExecuteAndReleaseWithRetries(ctx context.Context, actionName string, timeConfig serverlock.LockTimeConfig, fn func(ctx context.Context), retryOpts ...serverlock.RetryOpt) error {
m.LockWasAcquired = true
if m.ShouldFail {
return fmt.Errorf("mock lock acquisition failed")
}
if m.ShouldTimeout {
return context.DeadlineExceeded
}
fn(ctx)
m.ExecutionCount++
return nil
}
func createAlertingFolderService(t *testing.T, folderService folder.Service, lockService ServerLockService) *AlertingFolderService {
t.Helper()
return NewAlertingFolderService(
folderService,
log.NewNopLogger(),
lockService,
)
}

@ -0,0 +1,135 @@
package fakes
import (
"context"
"fmt"
"math/rand"
"sync"
"testing"
"github.com/grafana/grafana/pkg/apimachinery/identity"
"github.com/grafana/grafana/pkg/services/dashboards"
"github.com/grafana/grafana/pkg/services/folder"
"github.com/grafana/grafana/pkg/util"
)
type FakeAlertingFolderService struct {
t *testing.T
mtx sync.Mutex
Hook func(cmd any) error // use Hook if you need to intercept some query and return an error
RecordedOps []any
Folders map[int64][]*folder.Folder
}
func NewFakeAlertingFolderService(t *testing.T) *FakeAlertingFolderService {
return &FakeAlertingFolderService{
t: t,
Hook: func(any) error {
return nil
},
Folders: map[int64][]*folder.Folder{},
}
}
// GetRecordedCommands filters recorded commands using predicate function. Returns the subset of the recorded commands that meet the predicate
func (f *FakeAlertingFolderService) GetRecordedCommands(predicate func(cmd any) (any, bool)) []any {
f.mtx.Lock()
defer f.mtx.Unlock()
result := make([]any, 0, len(f.RecordedOps))
for _, op := range f.RecordedOps {
cmd, ok := predicate(op)
if !ok {
continue
}
result = append(result, cmd)
}
return result
}
func (f *FakeAlertingFolderService) GetUserVisibleNamespaces(_ context.Context, orgID int64, _ identity.Requester) (map[string]*folder.Folder, error) {
f.mtx.Lock()
defer f.mtx.Unlock()
namespacesMap := map[string]*folder.Folder{}
for _, folder := range f.Folders[orgID] {
namespacesMap[folder.UID] = folder
}
return namespacesMap, nil
}
func (f *FakeAlertingFolderService) GetNamespaceByUID(_ context.Context, uid string, orgID int64, user identity.Requester) (*folder.Folder, error) {
q := GenericRecordedQuery{
Name: "GetNamespaceByUID",
Params: []any{orgID, uid, user},
}
defer func() {
f.RecordedOps = append(f.RecordedOps, q)
}()
err := f.Hook(q)
if err != nil {
return nil, err
}
folders := f.Folders[orgID]
for _, folder := range folders {
if folder.UID == uid {
return folder, nil
}
}
return nil, fmt.Errorf("not found")
}
func (f *FakeAlertingFolderService) GetOrCreateNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
f.mtx.Lock()
defer f.mtx.Unlock()
for _, folder := range f.Folders[orgID] {
if folder.Title == title && folder.ParentUID == parentUID {
return folder, nil
}
}
newFolder := &folder.Folder{
ID: rand.Int63(), // nolint:staticcheck
UID: util.GenerateShortUID(),
Title: title,
ParentUID: parentUID,
Fullpath: "fullpath_" + title,
}
f.Folders[orgID] = append(f.Folders[orgID], newFolder)
return newFolder, nil
}
func (f *FakeAlertingFolderService) GetNamespaceByTitle(ctx context.Context, title string, orgID int64, user identity.Requester, parentUID string) (*folder.Folder, error) {
f.mtx.Lock()
defer f.mtx.Unlock()
for _, folder := range f.Folders[orgID] {
if folder.Title == title && folder.ParentUID == parentUID {
return folder, nil
}
}
return nil, dashboards.ErrFolderNotFound
}
func (f *FakeAlertingFolderService) GetNamespaceChildren(ctx context.Context, uid string, orgID int64, user identity.Requester) ([]*folder.Folder, error) {
f.mtx.Lock()
defer f.mtx.Unlock()
result := []*folder.Folder{}
for _, folder := range f.Folders[orgID] {
if folder.ParentUID == uid {
result = append(result, folder)
}
}
if len(result) == 0 {
return nil, dashboards.ErrFolderNotFound
}
return result, nil
}

@ -18,6 +18,7 @@ import (
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/kvstore"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/infra/tracing"
acmock "github.com/grafana/grafana/pkg/services/accesscontrol/mock"
"github.com/grafana/grafana/pkg/services/annotations/annotationstest"
@ -86,12 +87,14 @@ func SetupTestEnv(tb testing.TB, baseInterval time.Duration, opts ...TestEnvOpti
folderStore := folderimpl.ProvideDashboardFolderStore(sqlStore)
dashboardService, dashboardStore := testutil.SetupDashboardService(tb, sqlStore, folderStore, cfg)
folderService := testutil.SetupFolderService(tb, cfg, sqlStore, dashboardStore, folderStore, bus, options.featureToggles, ac)
serverLockService := serverlock.ProvideService(sqlStore, tracer)
ruleStore, err := store.ProvideDBStore(cfg, options.featureToggles, sqlStore, folderService, &dashboards.FakeDashboardService{}, ac, bus)
require.NoError(tb, err)
ng, err := ngalert.ProvideService(
cfg, options.featureToggles, nil, nil, routing.NewRouteRegister(), sqlStore, kvstore.NewFakeKVStore(), nil, nil, quotatest.New(false, nil),
secretsService, nil, m, folderService, ac, &dashboards.FakeDashboardService{}, nil, bus, ac,
annotationstest.NewFakeAnnotationsRepo(), &pluginstore.FakePluginStore{}, tracer, ruleStore, httpclient.NewProvider(), ngalertfakes.NewFakeReceiverPermissionsService(), usertest.NewUserServiceFake(),
serverLockService,
)
require.NoError(tb, err)

@ -13,6 +13,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/httpclient"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/infra/tracing"
pluginfakes "github.com/grafana/grafana/pkg/plugins/manager/fakes"
"github.com/grafana/grafana/pkg/services/accesscontrol/acimpl"
@ -516,7 +517,8 @@ func setupEnv(t *testing.T, sqlStore db.DB, cfg *setting.Cfg, b bus.Bus, quotaSe
_, err = ngalert.ProvideService(
cfg, featuremgmt.WithFeatures(), nil, nil, routing.NewRouteRegister(), sqlStore, ngalertfakes.NewFakeKVStore(t), nil, nil, quotaService,
secretsService, nil, m, &foldertest.FakeService{}, &acmock.Mock{}, &dashboards.FakeDashboardService{}, nil, b, &acmock.Mock{},
annotationstest.NewFakeAnnotationsRepo(), &pluginstore.FakePluginStore{}, tracer, ruleStore, httpclient.NewProvider(), ngalertfakes.NewFakeReceiverPermissionsService(), usertest.NewUserServiceFake(),
annotationstest.NewFakeAnnotationsRepo(), &pluginstore.FakePluginStore{}, tracer, ruleStore, httpclient.NewProvider(),
ngalertfakes.NewFakeReceiverPermissionsService(), usertest.NewUserServiceFake(), serverlock.ProvideService(sqlStore, tracer),
)
require.NoError(t, err)
_, err = storesrv.ProvideService(sqlStore, featuremgmt.WithFeatures(), cfg, quotaService, storesrv.ProvideSystemUsersService())

Loading…
Cancel
Save