From c3ca5405cedf19e71ae644bbe62f7c1231eea3ec Mon Sep 17 00:00:00 2001 From: Artur Wierzbicki Date: Wed, 21 Sep 2022 01:09:55 +0200 Subject: [PATCH] Search: Add search index configuration options (#55525) * Search: externalize config * Search: update config descriptions * Search: fix value * Search: fix * update * Search: revert config values * Search: rename copy/paste * Search: fix tests --- conf/defaults.ini | 16 +++++++++++++ pkg/services/searchV2/allowed_actions_test.go | 3 ++- pkg/services/searchV2/index.go | 22 +++++++++++------- pkg/services/searchV2/index_test.go | 3 ++- pkg/services/searchV2/service.go | 3 ++- pkg/setting/setting.go | 3 +++ pkg/setting/setting_search.go | 23 +++++++++++++++++++ 7 files changed, 62 insertions(+), 11 deletions(-) create mode 100644 pkg/setting/setting_search.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 08c6f70c192..630679839e2 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -1296,3 +1296,19 @@ scheduler_interval = [storage] # Allow uploading SVG files without sanitization. allow_unsanitized_svg_upload = false + + +#################################### Search ################################################ + +[search] +# Defines the number of dashboards loaded at once in a batch during a full reindex. +# This is a temporary settings that might be removed in the future. +dashboard_loading_batch_size = 200 + +# Defines the frequency of a full search reindex. +# This is a temporary settings that might be removed in the future. +full_reindex_interval = 5m + +# Defines the frequency of partial index updates based on recent changes such as dashboard updates. +# This is a temporary settings that might be removed in the future. +index_update_interval = 10s diff --git a/pkg/services/searchV2/allowed_actions_test.go b/pkg/services/searchV2/allowed_actions_test.go index b65b83f8d76..9f69da83b29 100644 --- a/pkg/services/searchV2/allowed_actions_test.go +++ b/pkg/services/searchV2/allowed_actions_test.go @@ -16,6 +16,7 @@ import ( "github.com/grafana/grafana/pkg/services/datasources" "github.com/grafana/grafana/pkg/services/featuremgmt" "github.com/grafana/grafana/pkg/services/user" + "github.com/grafana/grafana/pkg/setting" "github.com/stretchr/testify/require" ) @@ -82,7 +83,7 @@ var ( ) func service(t *testing.T) *StandardSearchService { - service, ok := ProvideService(nil, nil, nil, accesscontrolmock.New(), tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(), nil).(*StandardSearchService) + service, ok := ProvideService(&setting.Cfg{Search: setting.SearchSettings{}}, nil, nil, accesscontrolmock.New(), tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(), nil).(*StandardSearchService) require.True(t, ok) return service } diff --git a/pkg/services/searchV2/index.go b/pkg/services/searchV2/index.go index b6a61f39308..ea52b66e24c 100644 --- a/pkg/services/searchV2/index.go +++ b/pkg/services/searchV2/index.go @@ -20,6 +20,7 @@ import ( "github.com/grafana/grafana/pkg/services/searchV2/extract" "github.com/grafana/grafana/pkg/services/sqlstore" "github.com/grafana/grafana/pkg/services/store" + "github.com/grafana/grafana/pkg/setting" "go.opentelemetry.io/otel/attribute" "github.com/blugelabs/bluge" @@ -98,9 +99,10 @@ type searchIndex struct { syncCh chan chan struct{} tracer tracing.Tracer features featuremgmt.FeatureToggles + settings setting.SearchSettings } -func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup, tracer tracing.Tracer, features featuremgmt.FeatureToggles) *searchIndex { +func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup, tracer tracing.Tracer, features featuremgmt.FeatureToggles, settings setting.SearchSettings) *searchIndex { return &searchIndex{ loader: dashLoader, eventStore: evStore, @@ -113,6 +115,7 @@ func newSearchIndex(dashLoader dashboardLoader, evStore eventStore, extender Doc syncCh: make(chan chan struct{}), tracer: tracer, features: features, + settings: settings, } } @@ -176,11 +179,13 @@ func (i *searchIndex) sync(ctx context.Context) error { } func (i *searchIndex) run(ctx context.Context, orgIDs []int64, reIndexSignalCh chan struct{}) error { - reIndexInterval := 5 * time.Minute + i.logger.Info("Initializing SearchV2", "dashboardLoadingBatchSize", i.settings.DashboardLoadingBatchSize, "fullReindexInterval", i.settings.FullReindexInterval, "indexUpdateInterval", i.settings.IndexUpdateInterval) + + reIndexInterval := i.settings.FullReindexInterval fullReIndexTimer := time.NewTimer(reIndexInterval) defer fullReIndexTimer.Stop() - partialUpdateInterval := 5 * time.Second + partialUpdateInterval := i.settings.IndexUpdateInterval partialUpdateTimer := time.NewTimer(partialUpdateInterval) defer partialUpdateTimer.Stop() @@ -786,12 +791,13 @@ func (i *searchIndex) updateDashboard(ctx context.Context, orgID int64, index *o } type sqlDashboardLoader struct { - sql *sqlstore.SQLStore - logger log.Logger + sql *sqlstore.SQLStore + logger log.Logger + settings setting.SearchSettings } -func newSQLDashboardLoader(sql *sqlstore.SQLStore) *sqlDashboardLoader { - return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader")} +func newSQLDashboardLoader(sql *sqlstore.SQLStore, settings setting.SearchSettings) *sqlDashboardLoader { + return &sqlDashboardLoader{sql: sql, logger: log.New("sqlDashboardLoader"), settings: settings} } func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, dashboardUID string) ([]dashboard, error) { @@ -800,7 +806,7 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das limit := 1 if dashboardUID == "" { - limit = 200 + limit = l.settings.DashboardLoadingBatchSize dashboards = make([]dashboard, 0, limit+1) // Add the root folder ID (does not exist in SQL). diff --git a/pkg/services/searchV2/index_test.go b/pkg/services/searchV2/index_test.go index 70f5c45d060..bd1f41a98d3 100644 --- a/pkg/services/searchV2/index_test.go +++ b/pkg/services/searchV2/index_test.go @@ -10,6 +10,7 @@ import ( "github.com/grafana/grafana-plugin-sdk-go/data" "github.com/grafana/grafana/pkg/infra/tracing" "github.com/grafana/grafana/pkg/services/featuremgmt" + "github.com/grafana/grafana/pkg/setting" "github.com/grafana/grafana/pkg/infra/log" "github.com/grafana/grafana/pkg/services/searchV2/extract" @@ -62,7 +63,7 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten dashboardLoader := &testDashboardLoader{ dashboards: dashboards, } - index := newSearchIndex(dashboardLoader, &store.MockEntityEventsService{}, extender, func(ctx context.Context, folderId int64) (string, error) { return "x", nil }, tracing.InitializeTracerForTest(), featuremgmt.WithFeatures()) + index := newSearchIndex(dashboardLoader, &store.MockEntityEventsService{}, extender, func(ctx context.Context, folderId int64) (string, error) { return "x", nil }, tracing.InitializeTracerForTest(), featuremgmt.WithFeatures(), setting.SearchSettings{}) require.NotNil(t, index) numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID) require.NoError(t, err) diff --git a/pkg/services/searchV2/service.go b/pkg/services/searchV2/service.go index 08f8a2ef03f..c1723f09fc1 100644 --- a/pkg/services/searchV2/service.go +++ b/pkg/services/searchV2/service.go @@ -81,12 +81,13 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s ac: ac, }, dashboardIndex: newSearchIndex( - newSQLDashboardLoader(sql), + newSQLDashboardLoader(sql, cfg.Search), entityEventStore, extender.GetDocumentExtender(), newFolderIDLookup(sql), tracer, features, + cfg.Search, ), logger: log.New("searchV2"), extender: extender, diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index d1bbb432017..2777bfea0a4 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -455,6 +455,8 @@ type Cfg struct { Storage StorageSettings + Search SearchSettings + // Access Control RBACEnabled bool RBACPermissionCache bool @@ -1028,6 +1030,7 @@ func (cfg *Cfg) Load(args CommandLineArgs) error { cfg.DashboardPreviews = readDashboardPreviewsSettings(iniFile) cfg.Storage = readStorageSettings(iniFile) + cfg.Search = readSearchSettings(iniFile) if VerifyEmailEnabled && !cfg.Smtp.Enabled { cfg.Logger.Warn("require_email_validation is enabled but smtp is disabled") diff --git a/pkg/setting/setting_search.go b/pkg/setting/setting_search.go new file mode 100644 index 00000000000..b889fdd6319 --- /dev/null +++ b/pkg/setting/setting_search.go @@ -0,0 +1,23 @@ +package setting + +import ( + "time" + + "gopkg.in/ini.v1" +) + +type SearchSettings struct { + FullReindexInterval time.Duration + IndexUpdateInterval time.Duration + DashboardLoadingBatchSize int +} + +func readSearchSettings(iniFile *ini.File) SearchSettings { + s := SearchSettings{} + + searchSection := iniFile.Section("search") + s.DashboardLoadingBatchSize = searchSection.Key("dashboard_loading_batch_size").MustInt(200) + s.FullReindexInterval = searchSection.Key("full_reindex_interval").MustDuration(5 * time.Minute) + s.IndexUpdateInterval = searchSection.Key("index_update_interval").MustDuration(10 * time.Second) + return s +}