mirror of https://github.com/grafana/loki
add a storage client for boltdb-shipper which would do all the object key management for storage operations (#4128)
* add a storage client for boltdb-shipper which would do all the object key management for storage operations * changes suggested from PR reviewpull/4180/head
Sandeep Sukhani
4 years ago
committed by
GitHub
parent
6ac96478a7
commit
8ef3d5fee9
@ -0,0 +1,96 @@ |
||||
package storage |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"path" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/grafana/loki/pkg/storage/chunk" |
||||
) |
||||
|
||||
const delimiter = "/" |
||||
|
||||
// Client is used to manage boltdb index files in object storage, when using boltdb-shipper.
|
||||
type Client interface { |
||||
ListTables(ctx context.Context) ([]string, error) |
||||
ListFiles(ctx context.Context, tableName string) ([]IndexFile, error) |
||||
GetFile(ctx context.Context, tableName, fileName string) (io.ReadCloser, error) |
||||
PutFile(ctx context.Context, tableName, fileName string, file io.ReadSeeker) error |
||||
DeleteFile(ctx context.Context, tableName, fileName string) error |
||||
IsFileNotFoundErr(err error) bool |
||||
Stop() |
||||
} |
||||
|
||||
type indexStorageClient struct { |
||||
objectClient chunk.ObjectClient |
||||
storagePrefix string |
||||
} |
||||
|
||||
type IndexFile struct { |
||||
Name string |
||||
ModifiedAt time.Time |
||||
} |
||||
|
||||
func NewIndexStorageClient(objectClient chunk.ObjectClient, storagePrefix string) Client { |
||||
return &indexStorageClient{objectClient: objectClient, storagePrefix: storagePrefix} |
||||
} |
||||
|
||||
func (s *indexStorageClient) ListTables(ctx context.Context) ([]string, error) { |
||||
_, tables, err := s.objectClient.List(ctx, s.storagePrefix, delimiter) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
tableNames := make([]string, 0, len(tables)) |
||||
for _, table := range tables { |
||||
tableNames = append(tableNames, path.Base(string(table))) |
||||
} |
||||
|
||||
return tableNames, nil |
||||
} |
||||
|
||||
func (s *indexStorageClient) ListFiles(ctx context.Context, tableName string) ([]IndexFile, error) { |
||||
// The forward slash here needs to stay because we are trying to list contents of a directory without which
|
||||
// we will get the name of the same directory back with hosted object stores.
|
||||
// This is due to the object stores not having a concept of directories.
|
||||
objects, _, err := s.objectClient.List(ctx, s.storagePrefix+tableName+delimiter, delimiter) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
files := make([]IndexFile, 0, len(objects)) |
||||
for _, object := range objects { |
||||
// The s3 client can also return the directory itself in the ListObjects.
|
||||
if strings.HasSuffix(object.Key, delimiter) { |
||||
continue |
||||
} |
||||
files = append(files, IndexFile{ |
||||
Name: path.Base(object.Key), |
||||
ModifiedAt: object.ModifiedAt, |
||||
}) |
||||
} |
||||
|
||||
return files, nil |
||||
} |
||||
|
||||
func (s *indexStorageClient) GetFile(ctx context.Context, tableName, fileName string) (io.ReadCloser, error) { |
||||
return s.objectClient.GetObject(ctx, s.storagePrefix+path.Join(tableName, fileName)) |
||||
} |
||||
|
||||
func (s *indexStorageClient) PutFile(ctx context.Context, tableName, fileName string, file io.ReadSeeker) error { |
||||
return s.objectClient.PutObject(ctx, s.storagePrefix+path.Join(tableName, fileName), file) |
||||
} |
||||
|
||||
func (s *indexStorageClient) DeleteFile(ctx context.Context, tableName, fileName string) error { |
||||
return s.objectClient.DeleteObject(ctx, s.storagePrefix+path.Join(tableName, fileName)) |
||||
} |
||||
|
||||
func (s *indexStorageClient) IsFileNotFoundErr(err error) bool { |
||||
return s.objectClient.IsObjectNotFoundErr(err) |
||||
} |
||||
|
||||
func (s *indexStorageClient) Stop() { |
||||
s.objectClient.Stop() |
||||
} |
||||
@ -0,0 +1,81 @@ |
||||
package storage |
||||
|
||||
import ( |
||||
"bytes" |
||||
"context" |
||||
"io/ioutil" |
||||
"os" |
||||
"path/filepath" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
|
||||
"github.com/grafana/loki/pkg/storage/chunk/local" |
||||
"github.com/grafana/loki/pkg/storage/chunk/util" |
||||
) |
||||
|
||||
func TestIndexStorageClient(t *testing.T) { |
||||
tempDir, err := ioutil.TempDir("", "test-index-storage-client") |
||||
require.NoError(t, err) |
||||
|
||||
defer func() { |
||||
require.NoError(t, os.RemoveAll(tempDir)) |
||||
}() |
||||
|
||||
storageKeyPrefix := "prefix/" |
||||
tablesToSetup := map[string][]string{ |
||||
"table1": {"a", "b"}, |
||||
"table2": {"b", "c", "d"}, |
||||
} |
||||
|
||||
objectClient, err := local.NewFSObjectClient(local.FSConfig{Directory: tempDir}) |
||||
require.NoError(t, err) |
||||
|
||||
for tableName, files := range tablesToSetup { |
||||
require.NoError(t, util.EnsureDirectory(filepath.Join(tempDir, storageKeyPrefix, tableName))) |
||||
for _, file := range files { |
||||
err := ioutil.WriteFile(filepath.Join(tempDir, storageKeyPrefix, tableName, file), []byte(tableName+file), 0666) |
||||
require.NoError(t, err) |
||||
} |
||||
} |
||||
|
||||
indexStorageClient := NewIndexStorageClient(objectClient, storageKeyPrefix) |
||||
|
||||
verifyFiles := func() { |
||||
tables, err := indexStorageClient.ListTables(context.Background()) |
||||
require.NoError(t, err) |
||||
require.Len(t, tables, len(tablesToSetup)) |
||||
for _, table := range tables { |
||||
expectedFiles, ok := tablesToSetup[table] |
||||
require.True(t, ok) |
||||
|
||||
filesInStorage, err := indexStorageClient.ListFiles(context.Background(), table) |
||||
require.NoError(t, err) |
||||
require.Len(t, filesInStorage, len(expectedFiles)) |
||||
|
||||
for i, fileInStorage := range filesInStorage { |
||||
require.Equal(t, expectedFiles[i], fileInStorage.Name) |
||||
readCloser, err := indexStorageClient.GetFile(context.Background(), table, fileInStorage.Name) |
||||
require.NoError(t, err) |
||||
|
||||
b, err := ioutil.ReadAll(readCloser) |
||||
require.NoError(t, readCloser.Close()) |
||||
require.NoError(t, err) |
||||
require.EqualValues(t, []byte(table+fileInStorage.Name), b) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// verify the files using indexStorageClient
|
||||
verifyFiles() |
||||
|
||||
// delete a file and verify them again
|
||||
require.NoError(t, indexStorageClient.DeleteFile(context.Background(), "table2", "d")) |
||||
tablesToSetup["table2"] = tablesToSetup["table2"][:2] |
||||
verifyFiles() |
||||
|
||||
// add a file and verify them again
|
||||
require.NoError(t, indexStorageClient.PutFile(context.Background(), "table2", "e", bytes.NewReader([]byte("table2"+"e")))) |
||||
tablesToSetup["table2"] = append(tablesToSetup["table2"], "e") |
||||
verifyFiles() |
||||
} |
||||
@ -1,55 +0,0 @@ |
||||
package util |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"strings" |
||||
|
||||
"github.com/grafana/loki/pkg/storage/chunk" |
||||
) |
||||
|
||||
type PrefixedObjectClient struct { |
||||
downstreamClient chunk.ObjectClient |
||||
prefix string |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) PutObject(ctx context.Context, objectKey string, object io.ReadSeeker) error { |
||||
return p.downstreamClient.PutObject(ctx, p.prefix+objectKey, object) |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) GetObject(ctx context.Context, objectKey string) (io.ReadCloser, error) { |
||||
return p.downstreamClient.GetObject(ctx, p.prefix+objectKey) |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) List(ctx context.Context, prefix, delimeter string) ([]chunk.StorageObject, []chunk.StorageCommonPrefix, error) { |
||||
objects, commonPrefixes, err := p.downstreamClient.List(ctx, p.prefix+prefix, delimeter) |
||||
if err != nil { |
||||
return nil, nil, err |
||||
} |
||||
|
||||
for i := range objects { |
||||
objects[i].Key = strings.TrimPrefix(objects[i].Key, p.prefix) |
||||
} |
||||
|
||||
for i := range commonPrefixes { |
||||
commonPrefixes[i] = chunk.StorageCommonPrefix(strings.TrimPrefix(string(commonPrefixes[i]), p.prefix)) |
||||
} |
||||
|
||||
return objects, commonPrefixes, nil |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) DeleteObject(ctx context.Context, objectKey string) error { |
||||
return p.downstreamClient.DeleteObject(ctx, p.prefix+objectKey) |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) IsObjectNotFoundErr(err error) bool { |
||||
return p.downstreamClient.IsObjectNotFoundErr(err) |
||||
} |
||||
|
||||
func (p PrefixedObjectClient) Stop() { |
||||
p.downstreamClient.Stop() |
||||
} |
||||
|
||||
func NewPrefixedObjectClient(downstreamClient chunk.ObjectClient, prefix string) chunk.ObjectClient { |
||||
return PrefixedObjectClient{downstreamClient: downstreamClient, prefix: prefix} |
||||
} |
||||
Loading…
Reference in new issue