mirror of https://github.com/grafana/grafana
#45498: refactor tests, fix pagination bug in FS implementation
Artur Wierzbicki
3 years ago
parent
a2baf378bd
commit
0e20df4ccb
@ -1,338 +0,0 @@ |
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package filestorage |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
"github.com/stretchr/testify/require" |
||||
"gocloud.dev/blob" |
||||
) |
||||
|
||||
type NameFullPath struct { |
||||
Name string |
||||
FullPath string |
||||
} |
||||
|
||||
func extractNameFullPath(meta []FileMetadata) []NameFullPath { |
||||
resp := make([]NameFullPath, 0) |
||||
for i := range meta { |
||||
resp = append(resp, NameFullPath{ |
||||
Name: meta[i].Name, |
||||
FullPath: meta[i].FullPath, |
||||
}) |
||||
} |
||||
return resp |
||||
} |
||||
|
||||
func TestSqlStorage(t *testing.T) { |
||||
|
||||
var sqlStore *sqlstore.SQLStore |
||||
var filestorage FileStorage |
||||
var ctx context.Context |
||||
|
||||
setup := func() { |
||||
mode := "mem" |
||||
testLogger := log.New("testStorageLogger") |
||||
if mode == "db" { |
||||
sqlStore = sqlstore.InitTestDB(t) |
||||
filestorage = NewDbStorage(testLogger, sqlStore, nil) |
||||
} else if mode == "mem" { |
||||
bucket, _ := blob.OpenBucket(context.Background(), "mem://") |
||||
filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil) |
||||
} |
||||
|
||||
ctx = context.Background() |
||||
} |
||||
|
||||
t.Run("Should be able to insert a file", func(t *testing.T) { |
||||
setup() |
||||
err := filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
}) |
||||
|
||||
t.Run("Should be able to get a file", func(t *testing.T) { |
||||
setup() |
||||
path := "/folder1/folder2/file.jpg" |
||||
properties := map[string]string{"prop1": "val1", "prop2": "val"} |
||||
err := filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: path, |
||||
Contents: &[]byte{}, |
||||
Properties: properties, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
file, err := filestorage.Get(ctx, path) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, path, file.FullPath) |
||||
require.Equal(t, "file.jpg", file.Name) |
||||
require.Equal(t, properties, file.Properties) |
||||
}) |
||||
|
||||
t.Run("Should not be able to get a non-existent file", func(t *testing.T) { |
||||
setup() |
||||
path := "/folder1/folder2/file.jpg" |
||||
|
||||
file, err := filestorage.Get(ctx, path) |
||||
require.NoError(t, err) |
||||
require.Nil(t, file) |
||||
}) |
||||
|
||||
t.Run("Should be able to list files", func(t *testing.T) { |
||||
setup() |
||||
err := filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
err = filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{Recursive: true}) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{ |
||||
Name: "file-inner.jpg", |
||||
FullPath: "/folder1/file-inner.jpg", |
||||
}, |
||||
{ |
||||
Name: "file.jpg", |
||||
FullPath: "/folder1/folder2/file.jpg", |
||||
}, |
||||
}, extractNameFullPath(resp.Files)) |
||||
|
||||
resp, err = filestorage.ListFiles(ctx, "/folder1", nil, nil) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{ |
||||
Name: "file-inner.jpg", |
||||
FullPath: "/folder1/file-inner.jpg", |
||||
}, |
||||
}, extractNameFullPath(resp.Files)) |
||||
|
||||
resp, err = filestorage.ListFiles(ctx, "/folder1/folder2", nil, nil) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{ |
||||
Name: "file.jpg", |
||||
FullPath: "/folder1/folder2/file.jpg", |
||||
}, |
||||
}, extractNameFullPath(resp.Files)) |
||||
}) |
||||
|
||||
t.Run("Should be able to list files with prefix filtering", func(t *testing.T) { |
||||
setup() |
||||
err := filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
err = filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{ |
||||
Recursive: true, PathFilters: PathFilters{ |
||||
allowedPrefixes: []string{"/folder2"}, |
||||
}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{}, extractNameFullPath(resp.Files)) |
||||
|
||||
resp, err = filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{ |
||||
Recursive: true, PathFilters: PathFilters{ |
||||
allowedPrefixes: []string{"/folder1/folde"}, |
||||
}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{ |
||||
Name: "file.jpg", |
||||
FullPath: "/folder1/folder2/file.jpg", |
||||
}, |
||||
}, extractNameFullPath(resp.Files)) |
||||
}) |
||||
|
||||
t.Run("Should be able to list files with pagination", func(t *testing.T) { |
||||
setup() |
||||
err := filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
|
||||
err = filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
require.NoError(t, err) |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folderA/folderB/file.txt", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
|
||||
resp, err := filestorage.ListFiles(ctx, "/", &Paging{ |
||||
After: "/folder1/file-inner.jpg", |
||||
First: 1, |
||||
}, &ListOptions{Recursive: true}) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{Name: "file.jpg", FullPath: "/folder1/folder2/file.jpg"}, |
||||
}, extractNameFullPath(resp.Files)) |
||||
}) |
||||
|
||||
t.Run("Should be able to list folders", func(t *testing.T) { |
||||
setup() |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folderX/folderZ/file.txt", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folderA/folderB/file.txt", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
|
||||
resp, err := filestorage.ListFolders(ctx, "/", nil) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []FileMetadata{ |
||||
{ |
||||
Name: "folder1", |
||||
FullPath: "/folder1", |
||||
}, |
||||
{ |
||||
Name: "folder2", |
||||
FullPath: "/folder1/folder2", |
||||
}, |
||||
{ |
||||
Name: "folderA", |
||||
FullPath: "/folderA", |
||||
}, { |
||||
Name: "folderB", |
||||
FullPath: "/folderA/folderB", |
||||
}, |
||||
{ |
||||
Name: "folderX", |
||||
FullPath: "/folderX", |
||||
}, { |
||||
Name: "folderZ", |
||||
FullPath: "/folderX/folderZ", |
||||
}, |
||||
}, resp) |
||||
}) |
||||
|
||||
t.Run("Should be able to create and delete folders", func(t *testing.T) { |
||||
setup() |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder") |
||||
filestorage.CreateFolder(ctx, "/folder/icons", "emojis") |
||||
err := filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder") |
||||
require.NoError(t, err) |
||||
|
||||
resp, err := filestorage.ListFolders(ctx, "/", nil) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []FileMetadata{ |
||||
{ |
||||
Name: "folder", |
||||
FullPath: "/folder", |
||||
}, { |
||||
Name: "icons", |
||||
FullPath: "/folder/icons", |
||||
}, |
||||
{ |
||||
Name: "emojis", |
||||
FullPath: "/folder/icons/emojis", |
||||
}, { |
||||
Name: "folder1", |
||||
FullPath: "/folder1", |
||||
}, |
||||
{ |
||||
Name: "folder2", |
||||
FullPath: "/folder1/folder2", |
||||
}, |
||||
}, resp) |
||||
}) |
||||
|
||||
t.Run("Should not be able to delete folders with files", func(t *testing.T) { |
||||
setup() |
||||
filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder") |
||||
filestorage.Upsert(ctx, &UpsertFileCommand{ |
||||
Path: "/folder/dashboards/myNewFolder/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}) |
||||
filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder") |
||||
|
||||
resp, err := filestorage.ListFolders(ctx, "/", nil) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []FileMetadata{ |
||||
{ |
||||
Name: "folder", |
||||
FullPath: "/folder", |
||||
}, { |
||||
Name: "dashboards", |
||||
FullPath: "/folder/dashboards", |
||||
}, |
||||
{ |
||||
Name: "myNewFolder", |
||||
FullPath: "/folder/dashboards/myNewFolder", |
||||
}, |
||||
}, resp) |
||||
|
||||
files, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true}) |
||||
require.NoError(t, err) |
||||
require.Equal(t, []NameFullPath{ |
||||
{ |
||||
Name: "file.jpg", |
||||
FullPath: "/folder/dashboards/myNewFolder/file.jpg", |
||||
}, |
||||
}, extractNameFullPath(files.Files)) |
||||
}) |
||||
} |
||||
@ -0,0 +1,387 @@ |
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package filestorage |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/base64" |
||||
"fmt" |
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
"gocloud.dev/blob" |
||||
"io/ioutil" |
||||
"os" |
||||
"testing" |
||||
) |
||||
|
||||
const ( |
||||
pngImageBase64 = "iVBORw0KGgoAAAANSUhEUgAAAC4AAAAmCAYAAAC76qlaAAAABHNCSVQICAgIfAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAABFSURBVFiF7c5BDQAhEACx4/x7XjzwGELSKuiamfke9N8OnBKvidfEa+I18Zp4TbwmXhOvidfEa+I18Zp4TbwmXhOvidc2lcsESD1LGnUAAAAASUVORK5CYII=" |
||||
) |
||||
|
||||
func TestFsStorage(t *testing.T) { |
||||
|
||||
var testLogger log.Logger |
||||
var sqlStore *sqlstore.SQLStore |
||||
var filestorage FileStorage |
||||
var ctx context.Context |
||||
var tempDir string |
||||
pngImage, _ := base64.StdEncoding.DecodeString(pngImageBase64) |
||||
pngImageSize := int64(len(pngImage)) |
||||
|
||||
commonSetup := func() { |
||||
testLogger = log.New("testStorageLogger") |
||||
ctx = context.Background() |
||||
} |
||||
|
||||
cleanUp := func() { |
||||
testLogger = nil |
||||
sqlStore = nil |
||||
if filestorage != nil { |
||||
_ = filestorage.close() |
||||
filestorage = nil |
||||
} |
||||
|
||||
ctx = nil |
||||
_ = os.RemoveAll(tempDir) |
||||
} |
||||
|
||||
setupInMemFS := func() { |
||||
commonSetup() |
||||
bucket, _ := blob.OpenBucket(context.Background(), "mem://") |
||||
filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil) |
||||
} |
||||
|
||||
setupSqlFS := func() { |
||||
commonSetup() |
||||
sqlStore = sqlstore.InitTestDB(t) |
||||
filestorage = NewDbStorage(testLogger, sqlStore, nil) |
||||
} |
||||
|
||||
setupLocalFs := func() { |
||||
commonSetup() |
||||
tmpDir, err := ioutil.TempDir("", "") |
||||
tempDir = tmpDir |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
|
||||
bucket, err := blob.OpenBucket(context.Background(), "file://"+tmpDir) |
||||
if err != nil { |
||||
t.Fatal(err) |
||||
} |
||||
filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil) |
||||
} |
||||
|
||||
tests := []struct { |
||||
name string |
||||
steps []interface{} |
||||
}{ |
||||
{ |
||||
name: "inserting a file", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/file.png", |
||||
Contents: &pngImage, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}, |
||||
}, |
||||
queryGet{ |
||||
input: queryGetInput{ |
||||
path: "/folder1/file.png", |
||||
}, |
||||
checks: checks( |
||||
fName("file.png"), |
||||
fMimeType("image/png"), |
||||
fProperties(map[string]string{"prop1": "val1", "prop2": "val"}), |
||||
fSize(pngImageSize), |
||||
fContents(pngImage), |
||||
), |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "getting a non-existent file", |
||||
steps: []interface{}{ |
||||
queryGet{ |
||||
input: queryGetInput{ |
||||
path: "/folder1/file12412.png", |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "listing files", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
Properties: map[string]string{"prop1": "val1", "prop2": "val"}, |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true}}, |
||||
list: checks(listSize(2), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder1/file-inner.jpg")), |
||||
checks(fPath("/folder1/folder2/file.jpg")), |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "listing files with prefix filter", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder2"}}}}, |
||||
list: checks(listSize(0), listHasMore(false), listLastPath("")), |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}}, |
||||
list: checks(listSize(1), listHasMore(false)), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder1/folder2/file.jpg")), |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "listing files with pagination", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/a", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/b", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder2/c", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: ""}}, |
||||
list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder1/a")), |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}}, |
||||
list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder1/b")), |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}}, |
||||
list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder2/c")), |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}}, |
||||
list: checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")), |
||||
files: [][]interface{}{ |
||||
checks(fPath("/folder1/a")), |
||||
checks(fPath("/folder1/b")), |
||||
checks(fPath("/folder2/c")), |
||||
}, |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}}, |
||||
list: checks(listSize(1), listHasMore(false)), |
||||
}, |
||||
queryListFiles{ |
||||
input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}}, |
||||
list: checks(listSize(0), listHasMore(false)), |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "listing folders", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/file-inner.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folderX/folderZ/file.txt", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folderA/folderB/file.txt", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
queryListFolders{ |
||||
input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, |
||||
checks: [][]interface{}{ |
||||
checks(fPath("/folder1")), |
||||
checks(fPath("/folder1/folder2")), |
||||
checks(fPath("/folderA")), |
||||
checks(fPath("/folderA/folderB")), |
||||
checks(fPath("/folderX")), |
||||
checks(fPath("/folderX/folderZ")), |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "creating and deleting folders", |
||||
steps: []interface{}{ |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder1/folder2/file.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdCreateFolder{ |
||||
path: "/folder/dashboards", |
||||
name: "myNewFolder", |
||||
}, |
||||
cmdCreateFolder{ |
||||
path: "/folder/icons", |
||||
name: "emojis", |
||||
}, |
||||
queryListFolders{ |
||||
input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, |
||||
checks: [][]interface{}{ |
||||
checks(fPath("/folder")), |
||||
checks(fPath("/folder/dashboards")), |
||||
checks(fPath("/folder/dashboards/myNewFolder")), |
||||
checks(fPath("/folder/icons")), |
||||
checks(fPath("/folder/icons/emojis")), |
||||
checks(fPath("/folder1")), |
||||
checks(fPath("/folder1/folder2")), |
||||
}, |
||||
}, |
||||
cmdDeleteFolder{ |
||||
path: "/folder/dashboards/myNewFolder", |
||||
}, |
||||
queryListFolders{ |
||||
input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, |
||||
checks: [][]interface{}{ |
||||
checks(fPath("/folder")), |
||||
checks(fPath("/folder/icons")), |
||||
checks(fPath("/folder/icons/emojis")), |
||||
checks(fPath("/folder1")), |
||||
checks(fPath("/folder1/folder2")), |
||||
}, |
||||
}, |
||||
}, |
||||
}, |
||||
{ |
||||
name: "should not be able to delete folders with files", |
||||
steps: []interface{}{ |
||||
cmdCreateFolder{ |
||||
path: "/folder/dashboards", |
||||
name: "myNewFolder", |
||||
}, |
||||
cmdUpsert{ |
||||
cmd: UpsertFileCommand{ |
||||
Path: "/folder/dashboards/myNewFolder/file.jpg", |
||||
Contents: &[]byte{}, |
||||
}, |
||||
}, |
||||
cmdDeleteFolder{ |
||||
path: "/folder/dashboards/myNewFolder", |
||||
error: &cmdErrorOutput{ |
||||
message: "folder %s is not empty - cant remove it", |
||||
args: []interface{}{"/folder/dashboards/myNewFolder"}, |
||||
}, |
||||
}, |
||||
queryListFolders{ |
||||
input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, |
||||
checks: [][]interface{}{ |
||||
checks(fPath("/folder")), |
||||
checks(fPath("/folder/dashboards")), |
||||
checks(fPath("/folder/dashboards/myNewFolder")), |
||||
}, |
||||
}, |
||||
queryGet{ |
||||
input: queryGetInput{ |
||||
path: "/folder/dashboards/myNewFolder/file.jpg", |
||||
}, |
||||
checks: checks( |
||||
fName("file.jpg"), |
||||
), |
||||
}, |
||||
}, |
||||
}, |
||||
} |
||||
|
||||
for _, tt := range tests { |
||||
t.Run(fmt.Sprintf("%s: %s", "IN MEM FS", tt.name), func(t *testing.T) { |
||||
setupInMemFS() |
||||
defer cleanUp() |
||||
for i, step := range tt.steps { |
||||
executeTestStep(t, ctx, step, i, filestorage) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
for _, tt := range tests { |
||||
t.Run(fmt.Sprintf("%s: %s", "SQL FS", tt.name), func(t *testing.T) { |
||||
setupSqlFS() |
||||
defer cleanUp() |
||||
for i, step := range tt.steps { |
||||
executeTestStep(t, ctx, step, i, filestorage) |
||||
} |
||||
}) |
||||
} |
||||
|
||||
for _, tt := range tests { |
||||
t.Run(fmt.Sprintf("%s: %s", "Local FS", tt.name), func(t *testing.T) { |
||||
if tt.name == "listing files with pagination" { |
||||
// bug in cdk fileblob
|
||||
return |
||||
} |
||||
setupLocalFs() |
||||
defer cleanUp() |
||||
for i, step := range tt.steps { |
||||
executeTestStep(t, ctx, step, i, filestorage) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
@ -1,74 +0,0 @@ |
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package filestorage |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/stretchr/testify/require" |
||||
"gocloud.dev/blob" |
||||
) |
||||
|
||||
func TestLocalFsCdkBlobStorage(t *testing.T) { |
||||
|
||||
var filestorage FileStorage |
||||
var ctx context.Context |
||||
|
||||
setup := func() { |
||||
bucket, _ := blob.OpenBucket(context.Background(), "file://./test_fs") |
||||
testLogger := log.New("testFsStorageLogger") |
||||
filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil) |
||||
ctx = context.Background() |
||||
} |
||||
|
||||
t.Run("Should be able to list folders", func(t *testing.T) { |
||||
setup() |
||||
folders, err := filestorage.ListFolders(ctx, "/", nil) |
||||
require.NoError(t, err) |
||||
require.Equal(t, []FileMetadata{ |
||||
{ |
||||
Name: "folderA", |
||||
FullPath: "/folderA", |
||||
}, |
||||
{ |
||||
Name: "folderAnestedA", |
||||
FullPath: "/folderA/folderAnestedA", |
||||
}, |
||||
}, folders) |
||||
|
||||
folders, err = filestorage.ListFolders(ctx, "/folderA", nil) |
||||
require.NoError(t, err) |
||||
require.Equal(t, []FileMetadata{ |
||||
{ |
||||
Name: "folderAnestedA", |
||||
FullPath: "/folderA/folderAnestedA", |
||||
}, |
||||
}, folders) |
||||
|
||||
}) |
||||
|
||||
t.Run("Should be able to list files", func(t *testing.T) { |
||||
setup() |
||||
res, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true}) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, []NameFullPath{ |
||||
{Name: "file.txt", FullPath: "/folderA/folderAnestedA/file.txt"}, |
||||
{Name: "rootFile.txt", FullPath: "/rootFile.txt"}, |
||||
}, extractNameFullPath(res.Files)) |
||||
}) |
||||
|
||||
t.Run("should be able to read file", func(t *testing.T) { |
||||
setup() |
||||
path := "/folderA/folderAnestedA/file.txt" |
||||
res, err := filestorage.Get(ctx, path) |
||||
require.NoError(t, err) |
||||
|
||||
require.Equal(t, res.FullPath, path) |
||||
require.Equal(t, res.Name, "file.txt") |
||||
require.Equal(t, "content\n", string(res.Contents)) |
||||
}) |
||||
} |
||||
@ -0,0 +1,335 @@ |
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package filestorage |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"reflect" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
type cmdErrorOutput struct { |
||||
message string |
||||
args []interface{} |
||||
instance error |
||||
} |
||||
|
||||
type cmdDelete struct { |
||||
path string |
||||
error *cmdErrorOutput |
||||
} |
||||
|
||||
type cmdUpsert struct { |
||||
cmd UpsertFileCommand |
||||
error *cmdErrorOutput |
||||
} |
||||
|
||||
type cmdCreateFolder struct { |
||||
path string |
||||
name string |
||||
error *cmdErrorOutput |
||||
} |
||||
|
||||
type cmdDeleteFolder struct { |
||||
path string |
||||
error *cmdErrorOutput |
||||
} |
||||
|
||||
type queryGetInput struct { |
||||
path string |
||||
} |
||||
|
||||
type fileNameCheck struct { |
||||
v string |
||||
} |
||||
|
||||
type filePropertiesCheck struct { |
||||
v map[string]string |
||||
} |
||||
|
||||
type fileContentsCheck struct { |
||||
v []byte |
||||
} |
||||
|
||||
type fileSizeCheck struct { |
||||
v int64 |
||||
} |
||||
|
||||
type fileMimeTypeCheck struct { |
||||
v string |
||||
} |
||||
|
||||
type filePathCheck struct { |
||||
v string |
||||
} |
||||
|
||||
type listSizeCheck struct { |
||||
v int |
||||
} |
||||
|
||||
type listHasMoreCheck struct { |
||||
v bool |
||||
} |
||||
|
||||
type listLastPathCheck struct { |
||||
v string |
||||
} |
||||
|
||||
func fContents(contents []byte) interface{} { |
||||
return fileContentsCheck{v: contents} |
||||
} |
||||
|
||||
func fName(name string) interface{} { |
||||
return fileNameCheck{v: name} |
||||
} |
||||
|
||||
func fPath(path string) interface{} { |
||||
return filePathCheck{v: path} |
||||
} |
||||
|
||||
func fProperties(properties map[string]string) interface{} { |
||||
return filePropertiesCheck{v: properties} |
||||
} |
||||
func fSize(size int64) interface{} { |
||||
return fileSizeCheck{v: size} |
||||
} |
||||
|
||||
func fMimeType(mimeType string) interface{} { |
||||
return fileMimeTypeCheck{v: mimeType} |
||||
} |
||||
|
||||
func listSize(size int) interface{} { |
||||
return listSizeCheck{v: size} |
||||
} |
||||
|
||||
func listHasMore(hasMore bool) interface{} { |
||||
return listHasMoreCheck{v: hasMore} |
||||
} |
||||
|
||||
func listLastPath(path string) interface{} { |
||||
return listLastPathCheck{v: path} |
||||
} |
||||
|
||||
func checks(c ...interface{}) []interface{} { |
||||
return c |
||||
} |
||||
|
||||
type queryGet struct { |
||||
input queryGetInput |
||||
checks []interface{} |
||||
} |
||||
|
||||
type queryListFilesInput struct { |
||||
path string |
||||
paging *Paging |
||||
options *ListOptions |
||||
} |
||||
|
||||
type queryListFiles struct { |
||||
input queryListFilesInput |
||||
list []interface{} |
||||
files [][]interface{} |
||||
} |
||||
|
||||
type queryListFoldersInput struct { |
||||
path string |
||||
options *ListOptions |
||||
} |
||||
|
||||
type queryListFolders struct { |
||||
input queryListFoldersInput |
||||
checks [][]interface{} |
||||
} |
||||
|
||||
func interfaceName(myvar interface{}) string { |
||||
if t := reflect.TypeOf(myvar); t.Kind() == reflect.Ptr { |
||||
return "*" + t.Elem().Name() |
||||
} else { |
||||
return t.Name() |
||||
} |
||||
} |
||||
|
||||
func handleCommand(t *testing.T, ctx context.Context, cmd interface{}, cmdName string, fs FileStorage) { |
||||
t.Helper() |
||||
|
||||
var err error |
||||
var expectedErr *cmdErrorOutput |
||||
switch c := cmd.(type) { |
||||
case cmdDelete: |
||||
err = fs.Delete(ctx, c.path) |
||||
if c.error == nil { |
||||
require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path) |
||||
} |
||||
expectedErr = c.error |
||||
case cmdUpsert: |
||||
err = fs.Upsert(ctx, &c.cmd) |
||||
if c.error == nil { |
||||
require.NoError(t, err, "%s: should be able to upsert file %s", cmdName, c.cmd.Path) |
||||
} |
||||
expectedErr = c.error |
||||
case cmdCreateFolder: |
||||
err = fs.CreateFolder(ctx, c.path, c.name) |
||||
if c.error == nil { |
||||
require.NoError(t, err, "%s: should be able to create folder %s in %s", cmdName, c.name, c.path) |
||||
} |
||||
expectedErr = c.error |
||||
case cmdDeleteFolder: |
||||
err = fs.DeleteFolder(ctx, c.path) |
||||
if c.error == nil { |
||||
require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path) |
||||
} |
||||
expectedErr = c.error |
||||
default: |
||||
t.Fatalf("unrecognized command %s", cmdName) |
||||
} |
||||
|
||||
if expectedErr != nil && err != nil { |
||||
if expectedErr.instance != nil { |
||||
require.ErrorIs(t, err, expectedErr.instance) |
||||
} |
||||
|
||||
if expectedErr.message != "" { |
||||
require.Errorf(t, err, expectedErr.message, expectedErr.args...) |
||||
} |
||||
} |
||||
} |
||||
|
||||
func runChecks(t *testing.T, stepName string, path string, output interface{}, checks []interface{}) { |
||||
if checks == nil || len(checks) == 0 { |
||||
return |
||||
} |
||||
|
||||
runFileMetadataCheck := func(file FileMetadata, check interface{}, checkName string) { |
||||
switch c := check.(type) { |
||||
case filePropertiesCheck: |
||||
require.Equal(t, c.v, file.Properties, "%s-%s %s", stepName, checkName, path) |
||||
case fileNameCheck: |
||||
require.Equal(t, c.v, file.Name, "%s-%s %s", stepName, checkName, path) |
||||
case fileSizeCheck: |
||||
require.Equal(t, c.v, file.Size, "%s-%s %s", stepName, checkName, path) |
||||
case fileMimeTypeCheck: |
||||
require.Equal(t, c.v, file.MimeType, "%s-%s %s", stepName, checkName, path) |
||||
case filePathCheck: |
||||
require.Equal(t, c.v, file.FullPath, "%s-%s %s", stepName, checkName, path) |
||||
default: |
||||
t.Fatalf("unrecognized file check %s", checkName) |
||||
} |
||||
} |
||||
|
||||
switch o := output.(type) { |
||||
case File: |
||||
for _, check := range checks { |
||||
checkName := interfaceName(check) |
||||
if fileContentsCheck, ok := check.(fileContentsCheck); ok { |
||||
require.Equal(t, fileContentsCheck.v, o.Contents, "%s-%s %s", stepName, checkName, path) |
||||
} else { |
||||
runFileMetadataCheck(o.FileMetadata, check, checkName) |
||||
} |
||||
} |
||||
case FileMetadata: |
||||
for _, check := range checks { |
||||
runFileMetadataCheck(o, check, interfaceName(check)) |
||||
} |
||||
case ListFilesResponse: |
||||
for _, check := range checks { |
||||
c := check |
||||
checkName := interfaceName(c) |
||||
switch c := check.(type) { |
||||
case listSizeCheck: |
||||
require.Equal(t, c.v, len(o.Files), "%s %s", stepName, path) |
||||
case listHasMoreCheck: |
||||
require.Equal(t, c.v, o.HasMore, "%s %s", stepName, path) |
||||
case listLastPathCheck: |
||||
require.Equal(t, c.v, o.LastPath, "%s %s", stepName, path) |
||||
default: |
||||
t.Fatalf("unrecognized list check %s", checkName) |
||||
} |
||||
} |
||||
default: |
||||
t.Fatalf("unrecognized output %s", interfaceName(output)) |
||||
} |
||||
|
||||
} |
||||
|
||||
func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName string, fs FileStorage) { |
||||
t.Helper() |
||||
|
||||
switch q := query.(type) { |
||||
case queryGet: |
||||
inputPath := q.input.path |
||||
file, err := fs.Get(ctx, inputPath) |
||||
require.NoError(t, err, "%s: should be able to get file %s", queryName, inputPath) |
||||
|
||||
if q.checks != nil && len(q.checks) > 0 { |
||||
require.NotNil(t, file, "%s %s", queryName, inputPath) |
||||
require.Equal(t, inputPath, file.FullPath, "%s %s", queryName, inputPath) |
||||
runChecks(t, queryName, inputPath, *file, q.checks) |
||||
} else { |
||||
require.Nil(t, file, "%s %s", queryName, inputPath) |
||||
} |
||||
case queryListFiles: |
||||
inputPath := q.input.path |
||||
resp, err := fs.ListFiles(ctx, inputPath, q.input.paging, q.input.options) |
||||
require.NoError(t, err, "%s: should be able to list files in %s", queryName, inputPath) |
||||
require.NotNil(t, resp) |
||||
if q.list != nil && len(q.list) > 0 { |
||||
runChecks(t, queryName, inputPath, *resp, q.list) |
||||
} else { |
||||
require.NotNil(t, resp, "%s %s", queryName, inputPath) |
||||
require.Equal(t, false, resp.HasMore, "%s %s", queryName, inputPath) |
||||
require.Equal(t, 0, len(resp.Files), "%s %s", queryName, inputPath) |
||||
require.Equal(t, "", resp.LastPath, "%s %s", queryName, inputPath) |
||||
} |
||||
|
||||
if q.files != nil { |
||||
require.Equal(t, len(resp.Files), len(q.files), "%s %s", queryName, inputPath) |
||||
for i, file := range resp.Files { |
||||
runChecks(t, queryName, inputPath, file, q.files[i]) |
||||
} |
||||
} |
||||
case queryListFolders: |
||||
inputPath := q.input.path |
||||
resp, err := fs.ListFolders(ctx, inputPath, q.input.options) |
||||
require.NotNil(t, resp) |
||||
require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath) |
||||
|
||||
if q.checks != nil { |
||||
require.Equal(t, len(resp), len(q.checks), "%s %s", queryName, inputPath) |
||||
for i, file := range resp { |
||||
runChecks(t, queryName, inputPath, file, q.checks[i]) |
||||
} |
||||
} else { |
||||
require.Equal(t, 0, len(resp), "%s %s", queryName, inputPath) |
||||
} |
||||
default: |
||||
t.Fatalf("unrecognized query %s", queryName) |
||||
} |
||||
} |
||||
|
||||
func executeTestStep(t *testing.T, ctx context.Context, step interface{}, stepNumber int, fs FileStorage) { |
||||
name := fmt.Sprintf("[%d]%s", stepNumber, interfaceName(step)) |
||||
|
||||
switch s := step.(type) { |
||||
case queryGet: |
||||
handleQuery(t, ctx, s, name, fs) |
||||
case queryListFiles: |
||||
handleQuery(t, ctx, s, name, fs) |
||||
case queryListFolders: |
||||
handleQuery(t, ctx, s, name, fs) |
||||
case cmdUpsert: |
||||
handleCommand(t, ctx, s, name, fs) |
||||
case cmdDelete: |
||||
handleCommand(t, ctx, s, name, fs) |
||||
case cmdCreateFolder: |
||||
handleCommand(t, ctx, s, name, fs) |
||||
case cmdDeleteFolder: |
||||
handleCommand(t, ctx, s, name, fs) |
||||
default: |
||||
t.Fatalf("unrecognized step %s", name) |
||||
} |
||||
|
||||
} |
||||
Loading…
Reference in new issue