apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Search: remove internal ID from bluge index (#49205)

Browse Source 
Co-authored-by: Alexander Emelin <frvzmb@gmail.com>
pull/49362/head
Ryan McKinley 3 years ago committed by GitHub
parent fe16680c6d
commit 9a59ccd644
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 76 additions and 86 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 70
      pkg/services/searchV2/bluge.go
  2. 69
      pkg/services/searchV2/index.go
  3. 12
      pkg/services/searchV2/index_test.go
  4. 11
      pkg/services/searchV2/service.go

70
pkg/services/searchV2/bluge.go
Unescape View File

@ -25,14 +25,13 @@ const (
documentFieldTag = "tag"
documentFieldURL = "url"
documentFieldName = "name"
documentFieldName_ngram = "name_ngram"
documentFieldNameNgram = "name_ngram"
documentFieldDescription = "description"
documentFieldLocation = "location" // parent path
documentFieldPanelType = "panel_type"
documentFieldTransformer = "transformer"
documentFieldDSUID = "ds_uid"
documentFieldDSType = "ds_type"
documentFieldInternalID = "__internal_id" // only for migrations! (indexed as a string)
)
func initIndex(dashboards []dashboard, logger log.Logger, extendDoc ExtendDashboardFunc) (*bluge.Reader, *bluge.Writer, error) {
@ -149,10 +148,9 @@ func getFolderDashboardDoc(dash dashboard) *bluge.Document {
AddField(bluge.NewKeywordField(documentFieldKind, string(entityKindFolder)).Aggregatable().StoreValue()).
AddField(bluge.NewKeywordField(documentFieldURL, url).StoreValue()).
AddField(bluge.NewTextField(documentFieldName, dash.info.Title).StoreValue().SearchTermPositions()).
AddField(getNameNGramField(dash.info.Title)).
AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions()).
// Add legacy ID (for lookup by internal ID)
AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dash.id)).Aggregatable().StoreValue())
AddField(getNameNGramField(dash.info.Title)).
AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions())
}
func getNonFolderDashboardDoc(dash dashboard, location string) *bluge.Document {
@ -167,9 +165,6 @@ func getNonFolderDashboardDoc(dash dashboard, location string) *bluge.Document {
AddField(getNameNGramField(dash.info.Title)).
AddField(bluge.NewTextField(documentFieldDescription, dash.info.Description).SearchTermPositions())
// Add legacy ID (for lookup by internal ID)
doc.AddField(bluge.NewKeywordField(documentFieldInternalID, fmt.Sprintf("%d", dash.id)))
for _, tag := range dash.info.Tags {
doc.AddField(bluge.NewKeywordField(documentFieldTag, tag).
StoreValue().
@ -255,36 +250,7 @@ var ngramQueryAnalyzer = &analysis.Analyzer{
}
func getNameNGramField(name string) bluge.Field {
return bluge.NewTextField(documentFieldName_ngram, name).WithAnalyzer(ngramIndexAnalyzer)
}
func getDashboardFolderUID(reader *bluge.Reader, folderID int64) (string, error) {
fullQuery := bluge.NewBooleanQuery()
fullQuery.AddMust(bluge.NewTermQuery(strconv.FormatInt(folderID, 10)).SetField(documentFieldInternalID))
fullQuery.AddMust(bluge.NewTermQuery(string(entityKindFolder)).SetField(documentFieldKind))
req := bluge.NewAllMatches(fullQuery)
req.WithStandardAggregations()
documentMatchIterator, err := reader.Search(context.Background(), req)
if err != nil {
return "", err
}
var uid string
match, err := documentMatchIterator.Next()
for err == nil && match != nil {
// load the identifier for this match
err = match.VisitStoredFields(func(field string, value []byte) bool {
if field == documentFieldUID {
uid = string(value)
}
return true
})
if err != nil {
return "", err
}
// load the next document match
match, err = documentMatchIterator.Next()
}
return uid, err
return bluge.NewTextField(documentFieldNameNgram, name).WithAnalyzer(ngramIndexAnalyzer)
}
func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string, error) {
@ -293,7 +259,6 @@ func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string,
fullQuery.AddMust(bluge.NewTermQuery(dashboardUID).SetField(documentFieldDSUID))
fullQuery.AddMust(bluge.NewTermQuery(string(entityKindPanel)).SetField(documentFieldKind))
req := bluge.NewAllMatches(fullQuery)
req.WithStandardAggregations()
documentMatchIterator, err := reader.Search(context.Background(), req)
if err != nil {
return nil, err
@ -320,7 +285,7 @@ func getDashboardPanelIDs(reader *bluge.Reader, dashboardUID string) ([]string,
func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader, filter ResourceFilter, q DashboardQuery, extender QueryExtender) *backend.DataResponse {
response := &backend.DataResponse{}
// Folder listing structure
// Folder listing structure.
idx := strings.Index(q.Query, ":")
if idx > 0 {
key := q.Query[0:idx]
@ -345,7 +310,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
fullQuery := bluge.NewBooleanQuery()
fullQuery.AddMust(newPermissionFilter(filter, logger))
// Only show dashboard / folders
// Only show dashboard / folders / panels.
if len(q.Kind) > 0 {
bq := bluge.NewBooleanQuery()
for _, k := range q.Kind {
@ -400,7 +365,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField(documentFieldName).SetBoost(6)).
AddShould(bluge.NewMatchPhraseQuery(q.Query).SetField(documentFieldDescription).SetBoost(3)).
AddShould(bluge.NewMatchQuery(q.Query).
SetField(documentFieldName_ngram).
SetField(documentFieldNameNgram).
SetAnalyzer(ngramQueryAnalyzer).SetBoost(1))
if len(q.Query) > 4 {
@ -447,8 +412,6 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
dvfieldNames := []string{"type"}
sctx := search.NewSearchContext(0, 0)
// numericFields := map[string]bool{"schemaVersion": true, "panelCount": true}
fScore := data.NewFieldFromFieldType(data.FieldTypeFloat64, 0)
fUID := data.NewFieldFromFieldType(data.FieldTypeString, 0)
fKind := data.NewFieldFromFieldType(data.FieldTypeString, 0)
@ -500,19 +463,10 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
name := ""
url := ""
loc := ""
var ds_uids []string
var dsUIDs []string
var tags []string
err = match.VisitStoredFields(func(field string, value []byte) bool {
// if numericFields[field] {
// num, err2 := bluge.DecodeNumericFloat64(value)
// if err2 != nil {
// vals[field] = num
// }
// } else {
// vals[field] = string(value)
// }
switch field {
case documentFieldUID:
uid = string(value)
@ -527,7 +481,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
case documentFieldLocation:
loc = string(value)
case documentFieldDSUID:
ds_uids = append(ds_uids, string(value))
dsUIDs = append(dsUIDs, string(value))
case documentFieldTag:
tags = append(tags, string(value))
default:
@ -563,8 +517,8 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
fTags.Append(nil)
}
if len(ds_uids) > 0 {
js, _ := json.Marshal(ds_uids)
if len(dsUIDs) > 0 {
js, _ := json.Marshal(dsUIDs)
jsb := json.RawMessage(js)
fDSUIDs.Append(&jsb)
} else {
@ -598,7 +552,7 @@ func doSearchQuery(ctx context.Context, logger log.Logger, reader *bluge.Reader,
aggs := documentMatchIterator.Aggregations()
header := &customMeta{
Count: aggs.Count(), // Total cound
Count: aggs.Count(), // Total count.
}
if q.Explain {
header.MaxScore = aggs.Metric("max_score")

69
pkg/services/searchV2/index.go
Unescape View File

@ -34,6 +34,10 @@ type eventStore interface {
GetAllEventsAfter(ctx context.Context, id int64) ([]*store.EntityEvent, error)
}
// While we migrate away from internal IDs... this lets us lookup values in SQL
// NOTE: folderId is unique across all orgs
type folderUIDLookup = func(ctx context.Context, folderId int64) (string, error)
type dashboard struct {
id int64
uid string
@ -46,25 +50,27 @@ type dashboard struct {
}
type dashboardIndex struct {
mu sync.RWMutex
loader dashboardLoader
perOrgReader map[int64]*bluge.Reader // orgId -> bluge reader
perOrgWriter map[int64]*bluge.Writer // orgId -> bluge writer
eventStore eventStore
logger log.Logger
buildSignals chan int64
extender DocumentExtender
mu sync.RWMutex
loader dashboardLoader
perOrgReader map[int64]*bluge.Reader // orgId -> bluge reader
perOrgWriter map[int64]*bluge.Writer // orgId -> bluge writer
eventStore eventStore
logger log.Logger
buildSignals chan int64
extender DocumentExtender
folderIdLookup folderUIDLookup
}
func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender) *dashboardIndex {
func newDashboardIndex(dashLoader dashboardLoader, evStore eventStore, extender DocumentExtender, folderIDs folderUIDLookup) *dashboardIndex {
return &dashboardIndex{
loader: dashLoader,
eventStore: evStore,
perOrgReader: map[int64]*bluge.Reader{},
perOrgWriter: map[int64]*bluge.Writer{},
logger: log.New("dashboardIndex"),
buildSignals: make(chan int64),
extender: extender,
loader: dashLoader,
eventStore: evStore,
perOrgReader: map[int64]*bluge.Reader{},
perOrgWriter: map[int64]*bluge.Writer{},
logger: log.New("dashboardIndex"),
buildSignals: make(chan int64),
extender: extender,
folderIdLookup: folderIDs,
}
}
@ -351,9 +357,9 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
// In the future we can rely on operation types to reduce work here.
if len(dbDashboards) == 0 {
newReader, err = i.removeDashboard(writer, reader, dashboardUID)
newReader, err = i.removeDashboard(ctx, writer, reader, dashboardUID)
} else {
newReader, err = i.updateDashboard(orgID, writer, reader, dbDashboards[0])
newReader, err = i.updateDashboard(ctx, orgID, writer, reader, dbDashboards[0])
}
if err != nil {
return err
@ -362,7 +368,7 @@ func (i *dashboardIndex) applyDashboardEvent(ctx context.Context, orgID int64, d
return nil
}
func (i *dashboardIndex) removeDashboard(writer *bluge.Writer, reader *bluge.Reader, dashboardUID string) (*bluge.Reader, error) {
func (i *dashboardIndex) removeDashboard(_ context.Context, writer *bluge.Writer, reader *bluge.Reader, dashboardUID string) (*bluge.Reader, error) {
// Find all panel docs to remove with dashboard.
panelIDs, err := getDashboardPanelIDs(reader, dashboardUID)
if err != nil {
@ -392,7 +398,7 @@ func stringInSlice(str string, slice []string) bool {
return false
}
func (i *dashboardIndex) updateDashboard(orgID int64, writer *bluge.Writer, reader *bluge.Reader, dash dashboard) (*bluge.Reader, error) {
func (i *dashboardIndex) updateDashboard(ctx context.Context, orgID int64, writer *bluge.Writer, reader *bluge.Reader, dash dashboard) (*bluge.Reader, error) {
batch := bluge.NewBatch()
extendDoc := i.extender.GetDashboardExtender(orgID, dash.uid)
@ -409,7 +415,7 @@ func (i *dashboardIndex) updateDashboard(orgID int64, writer *bluge.Writer, read
folderUID = "general"
} else {
var err error
folderUID, err = getDashboardFolderUID(reader, dash.folderID)
folderUID, err = i.folderIdLookup(ctx, dash.folderID)
if err != nil {
return nil, err
}
@ -547,6 +553,27 @@ func (l sqlDashboardLoader) LoadDashboards(ctx context.Context, orgID int64, das
return dashboards, err
}
func newFolderIDLookup(sql *sqlstore.SQLStore) folderUIDLookup {
return func(ctx context.Context, folderID int64) (string, error) {
uid := ""
err := sql.WithDbSession(ctx, func(sess *sqlstore.DBSession) error {
sess.Table("dashboard").
Where("id = ?", folderID).
Cols("uid")
res, err := sess.Query("SELECT uid FROM dashboard WHERE id=?", folderID)
if err != nil {
return err
}
if len(res) > 0 {
uid = string(res[0]["uid"])
}
return nil
})
return uid, err
}
}
type dashboardQueryResult struct {
Id int64
Uid string

12
pkg/services/searchV2/index_test.go
Unescape View File

@ -48,7 +48,11 @@ func initTestIndexFromDashesExtended(t *testing.T, dashboards []dashboard, exten
dashboardLoader := &testDashboardLoader{
dashboards: dashboards,
}
index := newDashboardIndex(dashboardLoader, &store.MockEntityEventsService{}, extender)
index := newDashboardIndex(
dashboardLoader,
&store.MockEntityEventsService{},
extender,
func(ctx context.Context, folderId int64) (string, error) { return "x", nil })
require.NotNil(t, index)
numDashboards, err := index.buildOrgIndex(context.Background(), testOrgID)
require.NoError(t, err)
@ -110,7 +114,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
t.Run("dashboard-delete", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, testDashboards)
newReader, err := index.removeDashboard(writer, reader, "2")
newReader, err := index.removeDashboard(context.Background(), writer, reader, "2")
require.NoError(t, err)
checkSearchResponse(t, filepath.Base(t.Name())+".txt", newReader, testAllowAllFilter,
@ -121,7 +125,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
t.Run("dashboard-create", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, testDashboards)
newReader, err := index.updateDashboard(testOrgID, writer, reader, dashboard{
newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
id: 3,
uid: "3",
info: &extract.DashboardInfo{
@ -138,7 +142,7 @@ func TestDashboardIndexUpdates(t *testing.T) {
t.Run("dashboard-update", func(t *testing.T) {
index, reader, writer := initTestIndexFromDashes(t, testDashboards)
newReader, err := index.updateDashboard(testOrgID, writer, reader, dashboard{
newReader, err := index.updateDashboard(context.Background(), testOrgID, writer, reader, dashboard{
id: 2,
uid: "2",
info: &extract.DashboardInfo{

11
pkg/services/searchV2/service.go
Unescape View File

@ -39,9 +39,14 @@ func ProvideService(cfg *setting.Cfg, sql *sqlstore.SQLStore, entityEventStore s
sql: sql,
ac: ac,
},
dashboardIndex: newDashboardIndex(newSQLDashboardLoader(sql), entityEventStore, extender.GetDocumentExtender()),
logger: log.New("searchV2"),
extender: extender,
dashboardIndex: newDashboardIndex(
newSQLDashboardLoader(sql),
entityEventStore,
extender.GetDocumentExtender(),
newFolderIDLookup(sql),
),
logger: log.New("searchV2"),
extender: extender,
}
return s
}

Write Preview
Loading…
Cancel
Save