apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

[search] grouping - support indexing dashboard tags, filtering by tag, listing tags (#95670)

Browse Source 
[search] grouping
pull/95682/head
Scott Lepper 7 months ago committed by GitHub
parent f539a70d6d
commit f8a5813573
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
8 changed files with 1226 additions and 909 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 61
      pkg/storage/unified/resource/index.go
  2. 11
      pkg/storage/unified/resource/index_mapping.go
  3. 5
      pkg/storage/unified/resource/index_server.go
  4. 58
      pkg/storage/unified/resource/index_test.go
  5. 1417
      pkg/storage/unified/resource/resource.pb.go
  6. 15
      pkg/storage/unified/resource/resource.proto
  7. 284
      pkg/storage/unified/resource/testdata/dashboard-tagged-resource.json
  8. 284
      pkg/storage/unified/resource/testdata/dashboard-tagged-resource2.json

61
pkg/storage/unified/resource/index.go
Unescape View File

@ -151,13 +151,7 @@ func (i *Index) Init(ctx context.Context) error {
logger.Info("indexing batch", "kind", rt.Key.Resource, "count", len(list.Items))
//add changes to batches for shards with changes in the List
tenants, err := i.AddToBatches(ctx, list)
if err != nil {
return err
}
// Index the batches for tenants with changes if the batch is large enough
err = i.IndexBatches(ctx, i.opts.BatchSize, tenants)
err = i.writeBatch(ctx, list)
if err != nil {
return err
}
@ -187,6 +181,20 @@ func (i *Index) Init(ctx context.Context) error {
return nil
}
func (i *Index) writeBatch(ctx context.Context, list *ListResponse) error {
tenants, err := i.AddToBatches(ctx, list)
if err != nil {
return err
}
// Index the batches for tenants with changes if the batch is large enough
err = i.IndexBatches(ctx, i.opts.BatchSize, tenants)
if err != nil {
return err
}
return nil
}
func (i *Index) Index(ctx context.Context, data *Data) error {
ctx, span := i.tracer.Start(ctx, tracingPrexfixIndex+"Index")
defer span.End()
@ -235,15 +243,15 @@ func (i *Index) Delete(ctx context.Context, uid string, key *ResourceKey) error
return nil
}
func (i *Index) Search(ctx context.Context, tenant string, query string, limit int, offset int) ([]IndexedResource, error) {
func (i *Index) Search(ctx context.Context, request *SearchRequest) (*IndexResults, error) {
ctx, span := i.tracer.Start(ctx, tracingPrexfixIndex+"Search")
defer span.End()
logger := i.log.FromContext(ctx)
if tenant == "" {
tenant = "default"
if request.Tenant == "" {
request.Tenant = "default"
}
shard, err := i.getShard(tenant)
shard, err := i.getShard(request.Tenant)
if err != nil {
return nil, err
}
@ -251,23 +259,30 @@ func (i *Index) Search(ctx context.Context, tenant string, query string, limit i
if err != nil {
return nil, err
}
logger.Info("got index for tenant", "tenant", tenant, "docCount", docCount)
logger.Info("got index for tenant", "tenant", request.Tenant, "docCount", docCount)
fields, _ := shard.index.Fields()
logger.Debug("indexed fields", "fields", fields)
// use 10 as a default limit for now
if limit <= 0 {
limit = 10
if request.Limit <= 0 {
request.Limit = 10
}
query := bleve.NewQueryStringQuery(request.Query)
req := bleve.NewSearchRequest(query)
for _, group := range request.GroupBy {
facet := bleve.NewFacetRequest("Spec."+group.Name, int(group.Limit))
req.AddFacet(group.Name+"_facet", facet)
}
req := bleve.NewSearchRequest(bleve.NewQueryStringQuery(query))
req.From = offset
req.Size = limit
req.From = int(request.Offset)
req.Size = int(request.Size)
req.Fields = []string{"*"} // return all indexed fields in search results
logger.Info("searching index", "query", query, "tenant", tenant)
logger.Info("searching index", "query", request.Query, "tenant", request.Tenant)
res, err := shard.index.Search(req)
if err != nil {
return nil, err
@ -282,7 +297,15 @@ func (i *Index) Search(ctx context.Context, tenant string, query string, limit i
results[resKey] = ir
}
return results, nil
groups := []*Group{}
for _, group := range request.GroupBy {
groupByFacet := res.Facets[group.Name+"_facet"]
for _, term := range groupByFacet.Terms.Terms() {
groups = append(groups, &Group{Name: term.Term, Count: int64(term.Count)})
}
}
return &IndexResults{Values: results, Groups: groups}, nil
}
func (i *Index) Count() (uint64, error) {

11
pkg/storage/unified/resource/index_mapping.go
Unescape View File

@ -25,6 +25,11 @@ type IndexedResource struct {
Spec any
}
type IndexResults struct {
Values []IndexedResource
Groups []*Group
}
func (ir IndexedResource) FromSearchHit(hit *search.DocumentMatch) IndexedResource {
ir.Uid = hit.Fields["Uid"].(string)
ir.Kind = hit.Fields["Kind"].(string)
@ -178,6 +183,10 @@ func getSpecObjectMappings() map[string][]SpecFieldMapping {
Field: "description",
Type: "string",
},
{
Field: "tags",
Type: "string[]",
},
},
}
@ -198,7 +207,7 @@ func createSpecObjectMapping(kind string) *mapping.DocumentMapping {
// Create a field mapping based on field type
switch fieldType {
case "string":
case "string", "string[]":
specMapping.AddFieldMappingsAt(fieldName, bleve.NewTextFieldMapping())
case "int", "int64", "float64":
specMapping.AddFieldMappingsAt(fieldName, bleve.NewNumericFieldMapping())

5
pkg/storage/unified/resource/index_server.go
Unescape View File

@ -28,17 +28,18 @@ func (is *IndexServer) Search(ctx context.Context, req *SearchRequest) (*SearchR
ctx, span := is.tracer.Start(ctx, tracingPrefixIndexServer+"Search")
defer span.End()
results, err := is.index.Search(ctx, req.Tenant, req.Query, int(req.Limit), int(req.Offset))
results, err := is.index.Search(ctx, req)
if err != nil {
return nil, err
}
res := &SearchResponse{}
for _, r := range results {
for _, r := range results.Values {
resJsonBytes, err := json.Marshal(r)
if err != nil {
return nil, err
}
res.Items = append(res.Items, &ResourceWrapper{Value: resJsonBytes})
res.Groups = results.Groups
}
return res, nil
}

58
pkg/storage/unified/resource/index_test.go
Unescape View File

@ -21,22 +21,35 @@ const testTenant = "default"
var testContext = context.Background()
func TestIndexDashboard(t *testing.T) {
data, err := os.ReadFile("./testdata/dashboard-resource.json")
require.NoError(t, err)
data := readTestData(t, "dashboard-resource.json")
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}}}
index := newTestIndex(t)
_, err = index.AddToBatches(testContext, list)
require.NoError(t, err)
index := newTestIndex(t, 1)
err = index.IndexBatches(testContext, 1, []string{testTenant})
err := index.writeBatch(testContext, list)
require.NoError(t, err)
assertCountEquals(t, index, 1)
assertSearchCountEquals(t, index, 1)
assertSearchCountEquals(t, index, "*", 1)
}
func TestIndexDashboardWithTags(t *testing.T) {
data := readTestData(t, "dashboard-tagged-resource.json")
data2 := readTestData(t, "dashboard-tagged-resource2.json")
list := &ListResponse{Items: []*ResourceWrapper{{Value: data}, {Value: data2}}}
index := newTestIndex(t, 2)
err := index.writeBatch(testContext, list)
require.NoError(t, err)
assertCountEquals(t, index, 2)
assertSearchCountEquals(t, index, "tag1", 2)
assertSearchCountEquals(t, index, "tag4", 1)
assertSearchGroupCountEquals(t, index, "*", "tags", 4)
assertSearchGroupCountEquals(t, index, "tag4", "tags", 3)
}
func TestIndexBatch(t *testing.T) {
index := newTestIndex(t)
index := newTestIndex(t, 1000)
startAll := time.Now()
ns := namespaces()
@ -105,7 +118,7 @@ func namespaces() []string {
return ns
}
func newTestIndex(t *testing.T) *Index {
func newTestIndex(t *testing.T, batchSize int) *Index {
tracingCfg := tracing.NewEmptyTracingConfig()
trace, err := tracing.ProvideService(tracingCfg)
require.NoError(t, err)
@ -117,7 +130,7 @@ func newTestIndex(t *testing.T) *Index {
opts: Opts{
ListLimit: 5000,
Workers: 10,
BatchSize: 1000,
BatchSize: batchSize,
},
}
}
@ -128,8 +141,25 @@ func assertCountEquals(t *testing.T, index *Index, expected uint64) {
assert.Equal(t, expected, total)
}
func assertSearchCountEquals(t *testing.T, index *Index, expected int) {
results, err := index.Search(testContext, testTenant, "*", expected+1, 0)
func assertSearchCountEquals(t *testing.T, index *Index, search string, expected int64) {
req := &SearchRequest{Query: search, Tenant: testTenant, Limit: expected + 1, Offset: 0, Size: expected + 1}
results, err := index.Search(testContext, req)
require.NoError(t, err)
assert.Equal(t, expected, int64(len(results.Values)))
}
func assertSearchGroupCountEquals(t *testing.T, index *Index, search string, group string, expected int64) {
groupBy := []*GroupBy{{Name: group, Limit: 100}}
req := &SearchRequest{Query: search, Tenant: testTenant, Limit: expected + 1, Offset: 0, Size: expected + 1, GroupBy: groupBy}
results, err := index.Search(testContext, req)
require.NoError(t, err)
assert.Equal(t, expected, int64(len(results.Groups)))
}
func readTestData(t *testing.T, name string) []byte {
// We can ignore the gosec G304 because this is only for tests
// nolint:gosec
data, err := os.ReadFile("./testdata/" + name)
require.NoError(t, err)
assert.Equal(t, expected, len(results))
return data
}

1417
pkg/storage/unified/resource/resource.pb.go
View File

File diff suppressed because it is too large Load Diff

15
pkg/storage/unified/resource/resource.proto
Unescape View File

@ -334,10 +334,25 @@ message SearchRequest {
// pagination support
int64 limit = 5;
int64 offset = 6;
// batch size (optional)
int64 size = 7;
// grouping (optional)
repeated GroupBy groupBy = 8;
}
message GroupBy {
string name = 1;
int64 limit = 2;
}
message Group {
string name = 1;
int64 count = 2;
}
message SearchResponse {
repeated ResourceWrapper items = 1;
repeated Group groups = 2;
}
message HistoryRequest {

284
pkg/storage/unified/resource/testdata/dashboard-tagged-resource.json vendored
Unescape View File

@ -0,0 +1,284 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "adg5xd8",
"namespace": "default",
"uid": "86ab200a-e8b0-47ce-bbc1-8c2e078b0956",
"creationTimestamp": "2024-10-30T20:24:07Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T20:24:07Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "A dashboard with tags",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"refId": "A"
}
],
"title": "Panel 2",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Another Panel",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
"tag1",
"tag2",
"tag3"
],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "Dashboard with Tags",
"uid": "",
"version": 0,
"weekStart": ""
}
}

284
pkg/storage/unified/resource/testdata/dashboard-tagged-resource2.json vendored
Unescape View File

@ -0,0 +1,284 @@
{
"kind": "Dashboard",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"metadata": {
"name": "foo",
"namespace": "default",
"uid": "86ab200a-e8b0-47ce-bbc1-8c2e078b0956-2",
"creationTimestamp": "2024-10-30T20:24:07Z",
"annotations": {
"grafana.app/createdBy": "user:be2g71ke8yoe8b",
"grafana.app/originHash": "Grafana v9.2.0 (NA)",
"grafana.app/originName": "UI",
"grafana.app/originPath": "/dashboard/new"
},
"managedFields": [
{
"manager": "Mozilla",
"operation": "Update",
"apiVersion": "dashboard.grafana.app/v0alpha1",
"time": "2024-10-30T20:24:07Z",
"fieldsType": "FieldsV1",
"fieldsV1": {
"f:metadata": {
"f:annotations": {
".": {},
"f:grafana.app/originHash": {},
"f:grafana.app/originName": {},
"f:grafana.app/originPath": {}
},
"f:generateName": {}
},
"f:spec": {
"f:annotations": {
".": {},
"f:list": {}
},
"f:description": {},
"f:editable": {},
"f:fiscalYearStartMonth": {},
"f:graphTooltip": {},
"f:id": {},
"f:links": {},
"f:panels": {},
"f:preload": {},
"f:schemaVersion": {},
"f:tags": {},
"f:templating": {
".": {},
"f:list": {}
},
"f:timepicker": {},
"f:timezone": {},
"f:title": {},
"f:uid": {},
"f:version": {},
"f:weekStart": {}
}
}
}
]
},
"spec": {
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": {
"type": "grafana",
"uid": "-- Grafana --"
},
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations \u0026 Alerts",
"type": "dashboard"
}
]
},
"description": "Another dashboard with tags",
"editable": true,
"fiscalYearStartMonth": 0,
"graphTooltip": 0,
"id": null,
"links": [],
"panels": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 0
},
"id": 2,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"refId": "A"
}
],
"title": "Panel 2",
"type": "timeseries"
},
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisBorderShow": false,
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"barWidthFactor": 0.6,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"insertNulls": false,
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 8
},
"id": 1,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"pluginVersion": "9.2.0",
"targets": [
{
"datasource": {
"type": "datasource",
"uid": "grafana"
},
"refId": "A"
}
],
"title": "Another Panel",
"type": "timeseries"
}
],
"preload": false,
"schemaVersion": 40,
"tags": [
"tag1",
"tag2",
"tag4"
],
"templating": {
"list": []
},
"timepicker": {},
"timezone": "browser",
"title": "Dashboard with Tags",
"uid": "",
"version": 0,
"weekStart": ""
}
}
Write Preview
Loading…
Cancel
Save