mirror of https://github.com/grafana/grafana
search: add index batching (#104163)
* add basic search backend integration tests * add search backend benchmark * add benchmark indexServer * fix * lint * add more tests * lint * do not use the poller * batch write * refactor and add tests * improvements * improvements * cleanup * only observe index success * add monitorIndexEvents method * nit use switch instead of if * make newIndexQueueProcessor private * simplify runProcessor * go lintpull/105181/head
Georges Chaudy
2 months ago
committed by
GitHub
parent
0ceea29787
commit
15b3de5893
@ -0,0 +1,143 @@ |
||||
package resource |
||||
|
||||
import ( |
||||
"context" |
||||
"sync" |
||||
"time" |
||||
) |
||||
|
||||
// indexQueueProcessor manages queue-based operations for a specific index
|
||||
// It is responsible for ingesting events for a single index
|
||||
// It will batch events and send them to the index in a single bulk request
|
||||
type indexQueueProcessor struct { |
||||
index ResourceIndex |
||||
nsr NamespacedResource |
||||
queue chan *WrittenEvent |
||||
batchSize int |
||||
builder DocumentBuilder |
||||
|
||||
resChan chan *IndexEvent // Channel to send results to the caller
|
||||
|
||||
mu sync.Mutex |
||||
running bool |
||||
} |
||||
|
||||
type IndexEvent struct { |
||||
WrittenEvent *WrittenEvent |
||||
Action IndexAction |
||||
IndexableDocument *IndexableDocument // empty for delete actions
|
||||
Timestamp time.Time |
||||
Latency time.Duration |
||||
Err error |
||||
} |
||||
|
||||
// newIndexQueueProcessor creates a new IndexQueueProcessor for the given index
|
||||
func newIndexQueueProcessor(index ResourceIndex, nsr NamespacedResource, batchSize int, builder DocumentBuilder, resChan chan *IndexEvent) *indexQueueProcessor { |
||||
return &indexQueueProcessor{ |
||||
index: index, |
||||
nsr: nsr, |
||||
queue: make(chan *WrittenEvent, 1000), // Buffer size of 1000 events
|
||||
batchSize: batchSize, |
||||
builder: builder, |
||||
resChan: resChan, |
||||
running: false, |
||||
} |
||||
} |
||||
|
||||
// Add adds an event to the queue and ensures the background processor is running
|
||||
func (b *indexQueueProcessor) Add(evt *WrittenEvent) { |
||||
b.queue <- evt |
||||
|
||||
// Start the processor if it's not already running
|
||||
b.mu.Lock() |
||||
defer b.mu.Unlock() |
||||
if !b.running { |
||||
b.running = true |
||||
go b.runProcessor() |
||||
} |
||||
} |
||||
|
||||
// runProcessor is the task processing the queue of written events
|
||||
func (b *indexQueueProcessor) runProcessor() { |
||||
defer func() { |
||||
b.mu.Lock() |
||||
b.running = false |
||||
b.mu.Unlock() |
||||
}() |
||||
|
||||
for { |
||||
batch := make([]*WrittenEvent, 0, b.batchSize) |
||||
select { |
||||
case evt := <-b.queue: |
||||
batch = append(batch, evt) |
||||
case <-time.After(5 * time.Second): |
||||
// No events in the past few seconds, stop the processor
|
||||
return |
||||
} |
||||
|
||||
prepare: |
||||
for len(batch) < b.batchSize { |
||||
select { |
||||
case evt := <-b.queue: |
||||
batch = append(batch, evt) |
||||
default: |
||||
break prepare |
||||
} |
||||
} |
||||
|
||||
b.process(batch) |
||||
} |
||||
} |
||||
|
||||
// process handles a batch of events
|
||||
func (b *indexQueueProcessor) process(batch []*WrittenEvent) { |
||||
if len(batch) == 0 { |
||||
return |
||||
} |
||||
|
||||
// Create bulk request
|
||||
req := &BulkIndexRequest{ |
||||
Items: make([]*BulkIndexItem, 0, len(batch)), |
||||
} |
||||
resp := make([]*IndexEvent, 0, len(batch)) |
||||
|
||||
for _, evt := range batch { |
||||
result := &IndexEvent{ |
||||
WrittenEvent: evt, |
||||
} |
||||
resp = append(resp, result) |
||||
|
||||
item := &BulkIndexItem{} |
||||
if evt.Type == WatchEvent_DELETED { |
||||
item.Action = ActionDelete |
||||
item.Key = evt.Key |
||||
} else { |
||||
item.Action = ActionIndex |
||||
ctx, cancel := context.WithTimeout(context.Background(), 10*time.Second) |
||||
defer cancel() |
||||
doc, err := b.builder.BuildDocument(ctx, evt.Key, evt.ResourceVersion, evt.Value) |
||||
if err != nil { |
||||
result.Err = err |
||||
} else { |
||||
item.Doc = doc |
||||
result.IndexableDocument = doc |
||||
} |
||||
} |
||||
req.Items = append(req.Items, item) |
||||
} |
||||
|
||||
err := b.index.BulkIndex(req) |
||||
if err != nil { |
||||
for _, r := range resp { |
||||
r.Err = err |
||||
} |
||||
} |
||||
ts := time.Now() |
||||
if b.resChan != nil { |
||||
for _, r := range resp { |
||||
r.Timestamp = ts |
||||
r.Latency = time.Duration(ts.UnixMicro()-r.WrittenEvent.ResourceVersion) * time.Microsecond |
||||
b.resChan <- r |
||||
} |
||||
} |
||||
} |
||||
@ -0,0 +1,173 @@ |
||||
package resource |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/mock" |
||||
) |
||||
|
||||
func TestNewIndexQueueProcessor(t *testing.T) { |
||||
mockIndex := &MockResourceIndex{} |
||||
mockBuilder := &MockDocumentBuilder{} |
||||
nsr := NamespacedResource{Resource: "test"} |
||||
|
||||
resChan := make(chan *IndexEvent) |
||||
|
||||
processor := newIndexQueueProcessor(mockIndex, nsr, 10, mockBuilder, resChan) |
||||
|
||||
assert.NotNil(t, processor) |
||||
assert.Equal(t, 10, processor.batchSize) |
||||
assert.NotNil(t, processor.queue) |
||||
} |
||||
|
||||
func TestIndexQueueProcessor_SingleEvent(t *testing.T) { |
||||
mockIndex := &MockResourceIndex{} |
||||
mockBuilder := &MockDocumentBuilder{} |
||||
nsr := NamespacedResource{Resource: "test"} |
||||
|
||||
resChan := make(chan *IndexEvent) |
||||
|
||||
processor := newIndexQueueProcessor(mockIndex, nsr, 10, mockBuilder, resChan) |
||||
|
||||
// Test data
|
||||
key := ResourceKey{Resource: "test", Name: "obj1", Namespace: "default"} |
||||
evt := &WrittenEvent{ |
||||
Key: &key, |
||||
ResourceVersion: time.Now().UnixMicro(), |
||||
Type: WatchEvent_ADDED, |
||||
Value: []byte(`{"test": "data"}`), |
||||
} |
||||
|
||||
// Setup expectations
|
||||
mockBuilder.On("BuildDocument", mock.Anything, &key, evt.ResourceVersion, evt.Value).Return(&IndexableDocument{Key: &key}, nil) |
||||
mockIndex.On("BulkIndex", mock.MatchedBy(func(req *BulkIndexRequest) bool { |
||||
return len(req.Items) == 1 && req.Items[0].Action == ActionIndex |
||||
})).Return(nil) |
||||
|
||||
// Start processor and wait for the document to be indexed
|
||||
processor.Add(evt) |
||||
|
||||
resp := <-resChan |
||||
assert.NotNil(t, resp) |
||||
assert.Nil(t, resp.Err) |
||||
assert.Equal(t, &key, resp.IndexableDocument.Key) |
||||
|
||||
mockBuilder.AssertExpectations(t) |
||||
mockIndex.AssertExpectations(t) |
||||
} |
||||
|
||||
func TestIndexQueueProcessor_BatchProcessing(t *testing.T) { |
||||
mockIndex := &MockResourceIndex{} |
||||
mockBuilder := &MockDocumentBuilder{} |
||||
nsr := NamespacedResource{Namespace: "default", Resource: "test"} |
||||
|
||||
resChan := make(chan *IndexEvent) |
||||
|
||||
processor := newIndexQueueProcessor(mockIndex, nsr, 2, mockBuilder, resChan) |
||||
|
||||
// Test data for two events
|
||||
events := []*WrittenEvent{ |
||||
{ |
||||
Key: &ResourceKey{Resource: "test", Name: "obj1", Namespace: "default"}, |
||||
ResourceVersion: time.Now().UnixMicro(), |
||||
Type: WatchEvent_ADDED, |
||||
Value: []byte(`{"test": "data1"}`), |
||||
}, |
||||
{ |
||||
Key: &ResourceKey{Resource: "test", Name: "obj2", Namespace: "default"}, |
||||
ResourceVersion: time.Now().UnixMicro(), |
||||
Type: WatchEvent_DELETED, |
||||
}, |
||||
} |
||||
|
||||
// Setup expectations
|
||||
mockBuilder.On("BuildDocument", mock.Anything, events[0].Key, events[0].ResourceVersion, events[0].Value). |
||||
Return(&IndexableDocument{Key: events[0].Key}, nil) |
||||
mockIndex.On("BulkIndex", mock.MatchedBy(func(req *BulkIndexRequest) bool { |
||||
return len(req.Items) == 2 && |
||||
req.Items[0].Action == ActionIndex && |
||||
req.Items[1].Action == ActionDelete |
||||
})).Return(nil) |
||||
|
||||
// Start processor and add events
|
||||
processor.Add(events[0]) |
||||
processor.Add(events[1]) |
||||
|
||||
r0 := <-resChan |
||||
assert.Nil(t, r0.Err) |
||||
assert.Equal(t, events[0].Key, r0.IndexableDocument.Key) |
||||
|
||||
r1 := <-resChan |
||||
assert.Nil(t, r1.Err) |
||||
assert.Nil(t, r1.IndexableDocument) // deleted event
|
||||
|
||||
mockBuilder.AssertExpectations(t) |
||||
mockIndex.AssertExpectations(t) |
||||
} |
||||
|
||||
func TestIndexQueueProcessor_BuildDocumentError(t *testing.T) { |
||||
mockIndex := &MockResourceIndex{} |
||||
mockBuilder := &MockDocumentBuilder{} |
||||
nsr := NamespacedResource{Resource: "test"} |
||||
|
||||
resChan := make(chan *IndexEvent) |
||||
|
||||
processor := newIndexQueueProcessor(mockIndex, nsr, 10, mockBuilder, resChan) |
||||
|
||||
evt := &WrittenEvent{ |
||||
Key: &ResourceKey{Resource: "test", Name: "obj1", Namespace: "default"}, |
||||
ResourceVersion: time.Now().UnixMicro(), |
||||
Type: WatchEvent_ADDED, |
||||
Value: []byte(`invalid json`), |
||||
} |
||||
|
||||
// Setup expectations for error case
|
||||
mockBuilder.On("BuildDocument", mock.Anything, evt.Key, evt.ResourceVersion, evt.Value). |
||||
Return(nil, assert.AnError) |
||||
|
||||
// The bulk index should not be called since document building failed
|
||||
mockIndex.On("BulkIndex", mock.Anything).Return(nil).Maybe() |
||||
|
||||
processor.Add(evt) |
||||
|
||||
resp := <-resChan |
||||
assert.NotNil(t, resp) |
||||
assert.Error(t, resp.Err) |
||||
assert.Nil(t, resp.IndexableDocument) |
||||
|
||||
mockBuilder.AssertExpectations(t) |
||||
mockIndex.AssertExpectations(t) |
||||
} |
||||
|
||||
func TestIndexQueueProcessor_BulkIndexError(t *testing.T) { |
||||
mockIndex := &MockResourceIndex{} |
||||
mockBuilder := &MockDocumentBuilder{} |
||||
nsr := NamespacedResource{Resource: "test"} |
||||
|
||||
resChan := make(chan *IndexEvent) |
||||
|
||||
processor := newIndexQueueProcessor(mockIndex, nsr, 10, mockBuilder, resChan) |
||||
|
||||
evt := &WrittenEvent{ |
||||
Key: &ResourceKey{Resource: "test", Name: "obj1", Namespace: "default"}, |
||||
ResourceVersion: time.Now().UnixMicro(), |
||||
Type: WatchEvent_ADDED, |
||||
Value: []byte(`{"test": "data"}`), |
||||
} |
||||
|
||||
// Setup expectations
|
||||
mockBuilder.On("BuildDocument", mock.Anything, evt.Key, evt.ResourceVersion, evt.Value). |
||||
Return(&IndexableDocument{Key: evt.Key}, nil) |
||||
mockIndex.On("BulkIndex", mock.Anything).Return(assert.AnError) |
||||
|
||||
processor.Add(evt) |
||||
|
||||
resp := <-resChan |
||||
assert.NotNil(t, resp) |
||||
assert.Error(t, resp.Err) |
||||
|
||||
mockBuilder.AssertExpectations(t) |
||||
mockIndex.AssertExpectations(t) |
||||
} |
||||
@ -0,0 +1,54 @@ |
||||
package resource |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
"github.com/grafana/authlib/types" |
||||
"github.com/stretchr/testify/mock" |
||||
) |
||||
|
||||
var _ ResourceIndex = &MockResourceIndex{} |
||||
|
||||
// Mock implementations
|
||||
type MockResourceIndex struct { |
||||
mock.Mock |
||||
} |
||||
|
||||
func (m *MockResourceIndex) BulkIndex(req *BulkIndexRequest) error { |
||||
args := m.Called(req) |
||||
return args.Error(0) |
||||
} |
||||
|
||||
func (m *MockResourceIndex) Search(ctx context.Context, access types.AccessClient, req *ResourceSearchRequest, federate []ResourceIndex) (*ResourceSearchResponse, error) { |
||||
args := m.Called(ctx, access, req, federate) |
||||
return args.Get(0).(*ResourceSearchResponse), args.Error(1) |
||||
} |
||||
|
||||
func (m *MockResourceIndex) CountManagedObjects(ctx context.Context) ([]*CountManagedObjectsResponse_ResourceCount, error) { |
||||
args := m.Called(ctx) |
||||
return args.Get(0).([]*CountManagedObjectsResponse_ResourceCount), args.Error(1) |
||||
} |
||||
|
||||
func (m *MockResourceIndex) DocCount(ctx context.Context, folder string) (int64, error) { |
||||
args := m.Called(ctx, folder) |
||||
return args.Get(0).(int64), args.Error(1) |
||||
} |
||||
|
||||
func (m *MockResourceIndex) ListManagedObjects(ctx context.Context, req *ListManagedObjectsRequest) (*ListManagedObjectsResponse, error) { |
||||
args := m.Called(ctx, req) |
||||
return args.Get(0).(*ListManagedObjectsResponse), args.Error(1) |
||||
} |
||||
|
||||
var _ DocumentBuilder = &MockDocumentBuilder{} |
||||
|
||||
type MockDocumentBuilder struct { |
||||
mock.Mock |
||||
} |
||||
|
||||
func (m *MockDocumentBuilder) BuildDocument(ctx context.Context, key *ResourceKey, resourceVersion int64, value []byte) (*IndexableDocument, error) { |
||||
args := m.Called(ctx, key, resourceVersion, value) |
||||
if args.Get(0) == nil { |
||||
return nil, args.Error(1) |
||||
} |
||||
return args.Get(0).(*IndexableDocument), nil |
||||
} |
||||
@ -0,0 +1,56 @@ |
||||
package search |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/tracing" |
||||
"github.com/grafana/grafana/pkg/services/featuremgmt" |
||||
"github.com/grafana/grafana/pkg/storage/unified/resource" |
||||
unitest "github.com/grafana/grafana/pkg/storage/unified/testing" |
||||
) |
||||
|
||||
func TestBleveSearchBackend(t *testing.T) { |
||||
if testing.Short() { |
||||
t.Skip("skipping integration test") |
||||
} |
||||
|
||||
// Run the search backend test suite
|
||||
unitest.RunSearchBackendTest(t, func(ctx context.Context) resource.SearchBackend { |
||||
tempDir := t.TempDir() |
||||
|
||||
// Create a new bleve backend
|
||||
backend, err := NewBleveBackend(BleveOptions{ |
||||
Root: tempDir, |
||||
FileThreshold: 5, |
||||
}, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorageSearchPermissionFiltering), nil) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, backend) |
||||
|
||||
return backend |
||||
}, &unitest.TestOptions{ |
||||
NSPrefix: "bleve-test", |
||||
}) |
||||
} |
||||
|
||||
func TestSearchBackendBenchmark(t *testing.T) { |
||||
opts := &unitest.BenchmarkOptions{ |
||||
NumResources: 10000, |
||||
Concurrency: 1, // For now we only want to test the write throughput
|
||||
NumNamespaces: 1, |
||||
NumGroups: 1, |
||||
NumResourceTypes: 1, |
||||
} |
||||
tempDir := t.TempDir() |
||||
|
||||
// Create a new bleve backend
|
||||
backend, err := NewBleveBackend(BleveOptions{ |
||||
Root: tempDir, |
||||
}, tracing.NewNoopTracerService(), featuremgmt.WithFeatures(featuremgmt.FlagUnifiedStorageSearchPermissionFiltering), nil) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, backend) |
||||
|
||||
unitest.BenchmarkSearchBackend(t, backend, opts) |
||||
} |
||||
@ -0,0 +1,180 @@ |
||||
package test |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/grafana/pkg/storage/unified/resource" |
||||
"github.com/grafana/grafana/pkg/util/testutil" |
||||
) |
||||
|
||||
// Test names for the search backend test suite
|
||||
const ( |
||||
TestBuildIndex = "build index" |
||||
TestTotalDocs = "total docs" |
||||
TestResourceIndex = "resource index" |
||||
) |
||||
|
||||
// NewSearchBackendFunc is a function that creates a new SearchBackend instance
|
||||
type NewSearchBackendFunc func(ctx context.Context) resource.SearchBackend |
||||
|
||||
// RunSearchBackendTest runs the search backend test suite
|
||||
func RunSearchBackendTest(t *testing.T, newBackend NewSearchBackendFunc, opts *TestOptions) { |
||||
if testing.Short() { |
||||
t.Skip("skipping integration test") |
||||
} |
||||
|
||||
if opts == nil { |
||||
opts = &TestOptions{} |
||||
} |
||||
|
||||
if opts.NSPrefix == "" { |
||||
opts.NSPrefix = "test-" + time.Now().Format("20060102150405") |
||||
} |
||||
|
||||
t.Logf("Running tests with namespace prefix: %s", opts.NSPrefix) |
||||
|
||||
cases := []struct { |
||||
name string |
||||
fn func(*testing.T, resource.SearchBackend, string) |
||||
}{ |
||||
{TestBuildIndex, runTestSearchBackendBuildIndex}, |
||||
{TestTotalDocs, runTestSearchBackendTotalDocs}, |
||||
{TestResourceIndex, runTestResourceIndex}, |
||||
} |
||||
|
||||
for _, tc := range cases { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
tc.fn(t, newBackend(context.Background()), opts.NSPrefix) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func runTestSearchBackendBuildIndex(t *testing.T, backend resource.SearchBackend, nsPrefix string) { |
||||
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second)) |
||||
ns := resource.NamespacedResource{ |
||||
Namespace: nsPrefix + "-ns1", |
||||
Group: "group", |
||||
Resource: "resource", |
||||
} |
||||
|
||||
// Get the index should return nil if the index does not exist
|
||||
index, err := backend.GetIndex(ctx, ns) |
||||
require.NoError(t, err) |
||||
require.Nil(t, index) |
||||
|
||||
// Build the index
|
||||
index, err = backend.BuildIndex(ctx, ns, 0, 0, nil, func(index resource.ResourceIndex) (int64, error) { |
||||
// Write a test document
|
||||
err := index.BulkIndex(&resource.BulkIndexRequest{ |
||||
Items: []*resource.BulkIndexItem{ |
||||
{ |
||||
Action: resource.ActionIndex, |
||||
Doc: &resource.IndexableDocument{ |
||||
Key: &resource.ResourceKey{ |
||||
Namespace: ns.Namespace, |
||||
Group: ns.Group, |
||||
Resource: ns.Resource, |
||||
Name: "doc1", |
||||
}, |
||||
Title: "Document 1", |
||||
}, |
||||
}, |
||||
}, |
||||
}) |
||||
if err != nil { |
||||
return 0, err |
||||
} |
||||
return 1, nil |
||||
}) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, index) |
||||
|
||||
// Get the index should now return the index
|
||||
index, err = backend.GetIndex(ctx, ns) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, index) |
||||
} |
||||
|
||||
func runTestSearchBackendTotalDocs(t *testing.T, backend resource.SearchBackend, nsPrefix string) { |
||||
// Get total document count
|
||||
count := backend.TotalDocs() |
||||
require.GreaterOrEqual(t, count, int64(0)) |
||||
} |
||||
|
||||
func runTestResourceIndex(t *testing.T, backend resource.SearchBackend, nsPrefix string) { |
||||
ctx := testutil.NewTestContext(t, time.Now().Add(5*time.Second)) |
||||
ns := resource.NamespacedResource{ |
||||
Namespace: nsPrefix + "-ns1", |
||||
Group: "group", |
||||
Resource: "resource", |
||||
} |
||||
|
||||
// Build initial index with some test documents
|
||||
index, err := backend.BuildIndex(ctx, ns, 3, 0, nil, func(index resource.ResourceIndex) (int64, error) { |
||||
err := index.BulkIndex(&resource.BulkIndexRequest{ |
||||
Items: []*resource.BulkIndexItem{ |
||||
{ |
||||
Action: resource.ActionIndex, |
||||
Doc: &resource.IndexableDocument{ |
||||
Key: &resource.ResourceKey{ |
||||
Namespace: ns.Namespace, |
||||
Group: ns.Group, |
||||
Resource: ns.Resource, |
||||
Name: "doc1", |
||||
}, |
||||
Title: "Document 1", |
||||
Tags: []string{"tag1", "tag2"}, |
||||
Fields: map[string]interface{}{ |
||||
"field1": 1, |
||||
"field2": "value1", |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
Action: resource.ActionIndex, |
||||
Doc: &resource.IndexableDocument{ |
||||
Key: &resource.ResourceKey{ |
||||
Namespace: ns.Namespace, |
||||
Group: ns.Group, |
||||
Resource: ns.Resource, |
||||
Name: "doc2", |
||||
}, |
||||
Title: "Document 2", |
||||
Tags: []string{"tag2", "tag3"}, |
||||
Fields: map[string]interface{}{ |
||||
"field1": 2, |
||||
"field2": "value2", |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
}) |
||||
require.NoError(t, err) |
||||
return int64(2), nil |
||||
}) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, index) |
||||
|
||||
t.Run("Search", func(t *testing.T) { |
||||
req := &resource.ResourceSearchRequest{ |
||||
Options: &resource.ListOptions{ |
||||
Key: &resource.ResourceKey{ |
||||
Namespace: ns.Namespace, |
||||
Group: ns.Group, |
||||
Resource: ns.Resource, |
||||
}, |
||||
}, |
||||
Fields: []string{"title", "folder", "tags"}, |
||||
Query: "tag3", |
||||
Limit: 10, |
||||
} |
||||
resp, err := index.Search(ctx, nil, req, nil) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, resp) |
||||
require.Equal(t, int64(1), resp.TotalHits) // Only doc3 should have tag3 now
|
||||
}) |
||||
} |
||||
Loading…
Reference in new issue