diff --git a/pkg/infra/filestorage/api.go b/pkg/infra/filestorage/api.go index 1c419f7d4c0..99ad5aa7356 100644 --- a/pkg/infra/filestorage/api.go +++ b/pkg/infra/filestorage/api.go @@ -101,6 +101,8 @@ type FileStorage interface { CreateFolder(ctx context.Context, path string, name string) error DeleteFolder(ctx context.Context, path string) error + + close() error } // Get(ctx, "/myGit/dashboards/xyz123") diff --git a/pkg/infra/filestorage/cdk_blob_filestorage.go b/pkg/infra/filestorage/cdk_blob_filestorage.go index 3db1004b92c..b365b9277db 100644 --- a/pkg/infra/filestorage/cdk_blob_filestorage.go +++ b/pkg/infra/filestorage/cdk_blob_filestorage.go @@ -132,6 +132,8 @@ func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging if err == io.EOF { hasMore = false break + } else { + hasMore = true } if err != nil { @@ -218,6 +220,17 @@ func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging }, nil } +func (c cdkBlobStorage) fixInputPrefix(path string) string { + if path == Delimiter || path == "" { + return c.rootFolder + } + if strings.HasPrefix(path, Delimiter) { + path = fmt.Sprintf("%s%s", c.rootFolder, strings.TrimPrefix(path, Delimiter)) + } + + return path +} + func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string { if path == Delimiter || path == "" { return c.rootFolder @@ -236,8 +249,22 @@ func fixPath(path string) string { return newPath } +func (c cdkBlobStorage) convertListOptions(options *ListOptions) *ListOptions { + if options == nil || options.allowedPrefixes == nil || len(options.allowedPrefixes) == 0 { + return options + } + + newPrefixes := make([]string, len(options.allowedPrefixes)) + for i, prefix := range options.allowedPrefixes { + newPrefixes[i] = c.fixInputPrefix(prefix) + } + + options.PathFilters.allowedPrefixes = newPrefixes + return options +} + func (c cdkBlobStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) { - return c.listFiles(ctx, c.convertFolderPathToPrefix(folderPath), paging, options) + return c.listFiles(ctx, c.convertFolderPathToPrefix(folderPath), paging, c.convertListOptions(options)) } func (c cdkBlobStorage) listFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) { @@ -294,7 +321,7 @@ func (c cdkBlobStorage) listFolders(ctx context.Context, parentFolderPath string } func (c cdkBlobStorage) ListFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) { - folders, err := c.listFolders(ctx, c.convertFolderPathToPrefix(parentFolderPath), options) + folders, err := c.listFolders(ctx, c.convertFolderPathToPrefix(parentFolderPath), c.convertListOptions(options)) return folders, err } @@ -331,3 +358,7 @@ func (c cdkBlobStorage) DeleteFolder(ctx context.Context, folderPath string) err err = c.bucket.Delete(ctx, directoryMarkerPath) return err } + +func (c cdkBlobStorage) close() error { + return c.bucket.Close() +} diff --git a/pkg/infra/filestorage/db_filestorage.go b/pkg/infra/filestorage/db_filestorage.go index 82e13ffecfd..acf2e8f4a30 100644 --- a/pkg/infra/filestorage/db_filestorage.go +++ b/pkg/infra/filestorage/db_filestorage.go @@ -367,3 +367,7 @@ func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) erro return err } + +func (s dbFileStorage) close() error { + return nil +} diff --git a/pkg/infra/filestorage/filestorage.go b/pkg/infra/filestorage/filestorage.go index d6a9feedb49..57f1d7522ea 100644 --- a/pkg/infra/filestorage/filestorage.go +++ b/pkg/infra/filestorage/filestorage.go @@ -141,3 +141,7 @@ func (b service) CreateFolder(ctx context.Context, path string, folderName strin func (b service) DeleteFolder(ctx context.Context, path string) error { return errors.New("not available") } + +func (c service) close() error { + return c.grafanaDsStorage.close() +} diff --git a/pkg/infra/filestorage/filestorage_test.go b/pkg/infra/filestorage/filestorage_test.go deleted file mode 100644 index 513766f5467..00000000000 --- a/pkg/infra/filestorage/filestorage_test.go +++ /dev/null @@ -1,338 +0,0 @@ -//go:build integration -// +build integration - -package filestorage - -import ( - "context" - "testing" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/grafana/grafana/pkg/services/sqlstore" - "github.com/stretchr/testify/require" - "gocloud.dev/blob" -) - -type NameFullPath struct { - Name string - FullPath string -} - -func extractNameFullPath(meta []FileMetadata) []NameFullPath { - resp := make([]NameFullPath, 0) - for i := range meta { - resp = append(resp, NameFullPath{ - Name: meta[i].Name, - FullPath: meta[i].FullPath, - }) - } - return resp -} - -func TestSqlStorage(t *testing.T) { - - var sqlStore *sqlstore.SQLStore - var filestorage FileStorage - var ctx context.Context - - setup := func() { - mode := "mem" - testLogger := log.New("testStorageLogger") - if mode == "db" { - sqlStore = sqlstore.InitTestDB(t) - filestorage = NewDbStorage(testLogger, sqlStore, nil) - } else if mode == "mem" { - bucket, _ := blob.OpenBucket(context.Background(), "mem://") - filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil) - } - - ctx = context.Background() - } - - t.Run("Should be able to insert a file", func(t *testing.T) { - setup() - err := filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - }) - - t.Run("Should be able to get a file", func(t *testing.T) { - setup() - path := "/folder1/folder2/file.jpg" - properties := map[string]string{"prop1": "val1", "prop2": "val"} - err := filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: path, - Contents: &[]byte{}, - Properties: properties, - }) - require.NoError(t, err) - - file, err := filestorage.Get(ctx, path) - require.NoError(t, err) - - require.Equal(t, path, file.FullPath) - require.Equal(t, "file.jpg", file.Name) - require.Equal(t, properties, file.Properties) - }) - - t.Run("Should not be able to get a non-existent file", func(t *testing.T) { - setup() - path := "/folder1/folder2/file.jpg" - - file, err := filestorage.Get(ctx, path) - require.NoError(t, err) - require.Nil(t, file) - }) - - t.Run("Should be able to list files", func(t *testing.T) { - setup() - err := filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - - err = filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/file-inner.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - - resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{Recursive: true}) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - { - Name: "file-inner.jpg", - FullPath: "/folder1/file-inner.jpg", - }, - { - Name: "file.jpg", - FullPath: "/folder1/folder2/file.jpg", - }, - }, extractNameFullPath(resp.Files)) - - resp, err = filestorage.ListFiles(ctx, "/folder1", nil, nil) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - { - Name: "file-inner.jpg", - FullPath: "/folder1/file-inner.jpg", - }, - }, extractNameFullPath(resp.Files)) - - resp, err = filestorage.ListFiles(ctx, "/folder1/folder2", nil, nil) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - { - Name: "file.jpg", - FullPath: "/folder1/folder2/file.jpg", - }, - }, extractNameFullPath(resp.Files)) - }) - - t.Run("Should be able to list files with prefix filtering", func(t *testing.T) { - setup() - err := filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - - err = filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/file-inner.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - - resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{ - Recursive: true, PathFilters: PathFilters{ - allowedPrefixes: []string{"/folder2"}, - }, - }) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{}, extractNameFullPath(resp.Files)) - - resp, err = filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{ - Recursive: true, PathFilters: PathFilters{ - allowedPrefixes: []string{"/folder1/folde"}, - }, - }) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - { - Name: "file.jpg", - FullPath: "/folder1/folder2/file.jpg", - }, - }, extractNameFullPath(resp.Files)) - }) - - t.Run("Should be able to list files with pagination", func(t *testing.T) { - setup() - err := filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - - err = filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/file-inner.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - require.NoError(t, err) - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folderA/folderB/file.txt", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - - resp, err := filestorage.ListFiles(ctx, "/", &Paging{ - After: "/folder1/file-inner.jpg", - First: 1, - }, &ListOptions{Recursive: true}) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - {Name: "file.jpg", FullPath: "/folder1/folder2/file.jpg"}, - }, extractNameFullPath(resp.Files)) - }) - - t.Run("Should be able to list folders", func(t *testing.T) { - setup() - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/file-inner.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folderX/folderZ/file.txt", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folderA/folderB/file.txt", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - - resp, err := filestorage.ListFolders(ctx, "/", nil) - require.NoError(t, err) - - require.Equal(t, []FileMetadata{ - { - Name: "folder1", - FullPath: "/folder1", - }, - { - Name: "folder2", - FullPath: "/folder1/folder2", - }, - { - Name: "folderA", - FullPath: "/folderA", - }, { - Name: "folderB", - FullPath: "/folderA/folderB", - }, - { - Name: "folderX", - FullPath: "/folderX", - }, { - Name: "folderZ", - FullPath: "/folderX/folderZ", - }, - }, resp) - }) - - t.Run("Should be able to create and delete folders", func(t *testing.T) { - setup() - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder1/folder2/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder") - filestorage.CreateFolder(ctx, "/folder/icons", "emojis") - err := filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder") - require.NoError(t, err) - - resp, err := filestorage.ListFolders(ctx, "/", nil) - require.NoError(t, err) - - require.Equal(t, []FileMetadata{ - { - Name: "folder", - FullPath: "/folder", - }, { - Name: "icons", - FullPath: "/folder/icons", - }, - { - Name: "emojis", - FullPath: "/folder/icons/emojis", - }, { - Name: "folder1", - FullPath: "/folder1", - }, - { - Name: "folder2", - FullPath: "/folder1/folder2", - }, - }, resp) - }) - - t.Run("Should not be able to delete folders with files", func(t *testing.T) { - setup() - filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder") - filestorage.Upsert(ctx, &UpsertFileCommand{ - Path: "/folder/dashboards/myNewFolder/file.jpg", - Contents: &[]byte{}, - Properties: map[string]string{"prop1": "val1", "prop2": "val"}, - }) - filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder") - - resp, err := filestorage.ListFolders(ctx, "/", nil) - require.NoError(t, err) - - require.Equal(t, []FileMetadata{ - { - Name: "folder", - FullPath: "/folder", - }, { - Name: "dashboards", - FullPath: "/folder/dashboards", - }, - { - Name: "myNewFolder", - FullPath: "/folder/dashboards/myNewFolder", - }, - }, resp) - - files, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true}) - require.NoError(t, err) - require.Equal(t, []NameFullPath{ - { - Name: "file.jpg", - FullPath: "/folder/dashboards/myNewFolder/file.jpg", - }, - }, extractNameFullPath(files.Files)) - }) -} diff --git a/pkg/infra/filestorage/fs_integration_test.go b/pkg/infra/filestorage/fs_integration_test.go new file mode 100644 index 00000000000..2c18fa41b47 --- /dev/null +++ b/pkg/infra/filestorage/fs_integration_test.go @@ -0,0 +1,387 @@ +//go:build integration +// +build integration + +package filestorage + +import ( + "context" + "encoding/base64" + "fmt" + "github.com/grafana/grafana/pkg/infra/log" + "github.com/grafana/grafana/pkg/services/sqlstore" + "gocloud.dev/blob" + "io/ioutil" + "os" + "testing" +) + +const ( + pngImageBase64 = "iVBORw0KGgoAAAANSUhEUgAAAC4AAAAmCAYAAAC76qlaAAAABHNCSVQICAgIfAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAABFSURBVFiF7c5BDQAhEACx4/x7XjzwGELSKuiamfke9N8OnBKvidfEa+I18Zp4TbwmXhOvidfEa+I18Zp4TbwmXhOvidc2lcsESD1LGnUAAAAASUVORK5CYII=" +) + +func TestFsStorage(t *testing.T) { + + var testLogger log.Logger + var sqlStore *sqlstore.SQLStore + var filestorage FileStorage + var ctx context.Context + var tempDir string + pngImage, _ := base64.StdEncoding.DecodeString(pngImageBase64) + pngImageSize := int64(len(pngImage)) + + commonSetup := func() { + testLogger = log.New("testStorageLogger") + ctx = context.Background() + } + + cleanUp := func() { + testLogger = nil + sqlStore = nil + if filestorage != nil { + _ = filestorage.close() + filestorage = nil + } + + ctx = nil + _ = os.RemoveAll(tempDir) + } + + setupInMemFS := func() { + commonSetup() + bucket, _ := blob.OpenBucket(context.Background(), "mem://") + filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil) + } + + setupSqlFS := func() { + commonSetup() + sqlStore = sqlstore.InitTestDB(t) + filestorage = NewDbStorage(testLogger, sqlStore, nil) + } + + setupLocalFs := func() { + commonSetup() + tmpDir, err := ioutil.TempDir("", "") + tempDir = tmpDir + if err != nil { + t.Fatal(err) + } + + bucket, err := blob.OpenBucket(context.Background(), "file://"+tmpDir) + if err != nil { + t.Fatal(err) + } + filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil) + } + + tests := []struct { + name string + steps []interface{} + }{ + { + name: "inserting a file", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/file.png", + Contents: &pngImage, + Properties: map[string]string{"prop1": "val1", "prop2": "val"}, + }, + }, + queryGet{ + input: queryGetInput{ + path: "/folder1/file.png", + }, + checks: checks( + fName("file.png"), + fMimeType("image/png"), + fProperties(map[string]string{"prop1": "val1", "prop2": "val"}), + fSize(pngImageSize), + fContents(pngImage), + ), + }, + }, + }, + { + name: "getting a non-existent file", + steps: []interface{}{ + queryGet{ + input: queryGetInput{ + path: "/folder1/file12412.png", + }, + }, + }, + }, + { + name: "listing files", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/folder2/file.jpg", + Contents: &[]byte{}, + Properties: map[string]string{"prop1": "val1", "prop2": "val"}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/file-inner.jpg", + Contents: &[]byte{}, + Properties: map[string]string{"prop1": "val1", "prop2": "val"}, + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true}}, + list: checks(listSize(2), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")), + files: [][]interface{}{ + checks(fPath("/folder1/file-inner.jpg")), + checks(fPath("/folder1/folder2/file.jpg")), + }, + }, + }, + }, + { + name: "listing files with prefix filter", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/folder2/file.jpg", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/file-inner.jpg", + Contents: &[]byte{}, + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder2"}}}}, + list: checks(listSize(0), listHasMore(false), listLastPath("")), + }, + queryListFiles{ + input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}}, + list: checks(listSize(1), listHasMore(false)), + files: [][]interface{}{ + checks(fPath("/folder1/folder2/file.jpg")), + }, + }, + }, + }, + { + name: "listing files with pagination", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/a", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/b", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder2/c", + Contents: &[]byte{}, + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: ""}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")), + files: [][]interface{}{ + checks(fPath("/folder1/a")), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}}, + list: checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")), + files: [][]interface{}{ + checks(fPath("/folder1/b")), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}}, + list: checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")), + files: [][]interface{}{ + checks(fPath("/folder2/c")), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}}, + list: checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")), + files: [][]interface{}{ + checks(fPath("/folder1/a")), + checks(fPath("/folder1/b")), + checks(fPath("/folder2/c")), + }, + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}}, + list: checks(listSize(1), listHasMore(false)), + }, + queryListFiles{ + input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}}, + list: checks(listSize(0), listHasMore(false)), + }, + }, + }, + { + name: "listing folders", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/folder2/file.jpg", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/file-inner.jpg", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folderX/folderZ/file.txt", + Contents: &[]byte{}, + }, + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folderA/folderB/file.txt", + Contents: &[]byte{}, + }, + }, + queryListFolders{ + input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, + checks: [][]interface{}{ + checks(fPath("/folder1")), + checks(fPath("/folder1/folder2")), + checks(fPath("/folderA")), + checks(fPath("/folderA/folderB")), + checks(fPath("/folderX")), + checks(fPath("/folderX/folderZ")), + }, + }, + }, + }, + { + name: "creating and deleting folders", + steps: []interface{}{ + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder1/folder2/file.jpg", + Contents: &[]byte{}, + }, + }, + cmdCreateFolder{ + path: "/folder/dashboards", + name: "myNewFolder", + }, + cmdCreateFolder{ + path: "/folder/icons", + name: "emojis", + }, + queryListFolders{ + input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, + checks: [][]interface{}{ + checks(fPath("/folder")), + checks(fPath("/folder/dashboards")), + checks(fPath("/folder/dashboards/myNewFolder")), + checks(fPath("/folder/icons")), + checks(fPath("/folder/icons/emojis")), + checks(fPath("/folder1")), + checks(fPath("/folder1/folder2")), + }, + }, + cmdDeleteFolder{ + path: "/folder/dashboards/myNewFolder", + }, + queryListFolders{ + input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, + checks: [][]interface{}{ + checks(fPath("/folder")), + checks(fPath("/folder/icons")), + checks(fPath("/folder/icons/emojis")), + checks(fPath("/folder1")), + checks(fPath("/folder1/folder2")), + }, + }, + }, + }, + { + name: "should not be able to delete folders with files", + steps: []interface{}{ + cmdCreateFolder{ + path: "/folder/dashboards", + name: "myNewFolder", + }, + cmdUpsert{ + cmd: UpsertFileCommand{ + Path: "/folder/dashboards/myNewFolder/file.jpg", + Contents: &[]byte{}, + }, + }, + cmdDeleteFolder{ + path: "/folder/dashboards/myNewFolder", + error: &cmdErrorOutput{ + message: "folder %s is not empty - cant remove it", + args: []interface{}{"/folder/dashboards/myNewFolder"}, + }, + }, + queryListFolders{ + input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}}, + checks: [][]interface{}{ + checks(fPath("/folder")), + checks(fPath("/folder/dashboards")), + checks(fPath("/folder/dashboards/myNewFolder")), + }, + }, + queryGet{ + input: queryGetInput{ + path: "/folder/dashboards/myNewFolder/file.jpg", + }, + checks: checks( + fName("file.jpg"), + ), + }, + }, + }, + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s: %s", "IN MEM FS", tt.name), func(t *testing.T) { + setupInMemFS() + defer cleanUp() + for i, step := range tt.steps { + executeTestStep(t, ctx, step, i, filestorage) + } + }) + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s: %s", "SQL FS", tt.name), func(t *testing.T) { + setupSqlFS() + defer cleanUp() + for i, step := range tt.steps { + executeTestStep(t, ctx, step, i, filestorage) + } + }) + } + + for _, tt := range tests { + t.Run(fmt.Sprintf("%s: %s", "Local FS", tt.name), func(t *testing.T) { + if tt.name == "listing files with pagination" { + // bug in cdk fileblob + return + } + setupLocalFs() + defer cleanUp() + for i, step := range tt.steps { + executeTestStep(t, ctx, step, i, filestorage) + } + }) + } +} diff --git a/pkg/infra/filestorage/localfs_test.go b/pkg/infra/filestorage/localfs_test.go deleted file mode 100644 index d7b2542eed5..00000000000 --- a/pkg/infra/filestorage/localfs_test.go +++ /dev/null @@ -1,74 +0,0 @@ -//go:build integration -// +build integration - -package filestorage - -import ( - "context" - "testing" - - "github.com/grafana/grafana/pkg/infra/log" - "github.com/stretchr/testify/require" - "gocloud.dev/blob" -) - -func TestLocalFsCdkBlobStorage(t *testing.T) { - - var filestorage FileStorage - var ctx context.Context - - setup := func() { - bucket, _ := blob.OpenBucket(context.Background(), "file://./test_fs") - testLogger := log.New("testFsStorageLogger") - filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil) - ctx = context.Background() - } - - t.Run("Should be able to list folders", func(t *testing.T) { - setup() - folders, err := filestorage.ListFolders(ctx, "/", nil) - require.NoError(t, err) - require.Equal(t, []FileMetadata{ - { - Name: "folderA", - FullPath: "/folderA", - }, - { - Name: "folderAnestedA", - FullPath: "/folderA/folderAnestedA", - }, - }, folders) - - folders, err = filestorage.ListFolders(ctx, "/folderA", nil) - require.NoError(t, err) - require.Equal(t, []FileMetadata{ - { - Name: "folderAnestedA", - FullPath: "/folderA/folderAnestedA", - }, - }, folders) - - }) - - t.Run("Should be able to list files", func(t *testing.T) { - setup() - res, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true}) - require.NoError(t, err) - - require.Equal(t, []NameFullPath{ - {Name: "file.txt", FullPath: "/folderA/folderAnestedA/file.txt"}, - {Name: "rootFile.txt", FullPath: "/rootFile.txt"}, - }, extractNameFullPath(res.Files)) - }) - - t.Run("should be able to read file", func(t *testing.T) { - setup() - path := "/folderA/folderAnestedA/file.txt" - res, err := filestorage.Get(ctx, path) - require.NoError(t, err) - - require.Equal(t, res.FullPath, path) - require.Equal(t, res.Name, "file.txt") - require.Equal(t, "content\n", string(res.Contents)) - }) -} diff --git a/pkg/infra/filestorage/test_utils.go b/pkg/infra/filestorage/test_utils.go new file mode 100644 index 00000000000..3db6acc1fab --- /dev/null +++ b/pkg/infra/filestorage/test_utils.go @@ -0,0 +1,335 @@ +//go:build integration +// +build integration + +package filestorage + +import ( + "context" + "fmt" + "reflect" + "testing" + + "github.com/stretchr/testify/require" +) + +type cmdErrorOutput struct { + message string + args []interface{} + instance error +} + +type cmdDelete struct { + path string + error *cmdErrorOutput +} + +type cmdUpsert struct { + cmd UpsertFileCommand + error *cmdErrorOutput +} + +type cmdCreateFolder struct { + path string + name string + error *cmdErrorOutput +} + +type cmdDeleteFolder struct { + path string + error *cmdErrorOutput +} + +type queryGetInput struct { + path string +} + +type fileNameCheck struct { + v string +} + +type filePropertiesCheck struct { + v map[string]string +} + +type fileContentsCheck struct { + v []byte +} + +type fileSizeCheck struct { + v int64 +} + +type fileMimeTypeCheck struct { + v string +} + +type filePathCheck struct { + v string +} + +type listSizeCheck struct { + v int +} + +type listHasMoreCheck struct { + v bool +} + +type listLastPathCheck struct { + v string +} + +func fContents(contents []byte) interface{} { + return fileContentsCheck{v: contents} +} + +func fName(name string) interface{} { + return fileNameCheck{v: name} +} + +func fPath(path string) interface{} { + return filePathCheck{v: path} +} + +func fProperties(properties map[string]string) interface{} { + return filePropertiesCheck{v: properties} +} +func fSize(size int64) interface{} { + return fileSizeCheck{v: size} +} + +func fMimeType(mimeType string) interface{} { + return fileMimeTypeCheck{v: mimeType} +} + +func listSize(size int) interface{} { + return listSizeCheck{v: size} +} + +func listHasMore(hasMore bool) interface{} { + return listHasMoreCheck{v: hasMore} +} + +func listLastPath(path string) interface{} { + return listLastPathCheck{v: path} +} + +func checks(c ...interface{}) []interface{} { + return c +} + +type queryGet struct { + input queryGetInput + checks []interface{} +} + +type queryListFilesInput struct { + path string + paging *Paging + options *ListOptions +} + +type queryListFiles struct { + input queryListFilesInput + list []interface{} + files [][]interface{} +} + +type queryListFoldersInput struct { + path string + options *ListOptions +} + +type queryListFolders struct { + input queryListFoldersInput + checks [][]interface{} +} + +func interfaceName(myvar interface{}) string { + if t := reflect.TypeOf(myvar); t.Kind() == reflect.Ptr { + return "*" + t.Elem().Name() + } else { + return t.Name() + } +} + +func handleCommand(t *testing.T, ctx context.Context, cmd interface{}, cmdName string, fs FileStorage) { + t.Helper() + + var err error + var expectedErr *cmdErrorOutput + switch c := cmd.(type) { + case cmdDelete: + err = fs.Delete(ctx, c.path) + if c.error == nil { + require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path) + } + expectedErr = c.error + case cmdUpsert: + err = fs.Upsert(ctx, &c.cmd) + if c.error == nil { + require.NoError(t, err, "%s: should be able to upsert file %s", cmdName, c.cmd.Path) + } + expectedErr = c.error + case cmdCreateFolder: + err = fs.CreateFolder(ctx, c.path, c.name) + if c.error == nil { + require.NoError(t, err, "%s: should be able to create folder %s in %s", cmdName, c.name, c.path) + } + expectedErr = c.error + case cmdDeleteFolder: + err = fs.DeleteFolder(ctx, c.path) + if c.error == nil { + require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path) + } + expectedErr = c.error + default: + t.Fatalf("unrecognized command %s", cmdName) + } + + if expectedErr != nil && err != nil { + if expectedErr.instance != nil { + require.ErrorIs(t, err, expectedErr.instance) + } + + if expectedErr.message != "" { + require.Errorf(t, err, expectedErr.message, expectedErr.args...) + } + } +} + +func runChecks(t *testing.T, stepName string, path string, output interface{}, checks []interface{}) { + if checks == nil || len(checks) == 0 { + return + } + + runFileMetadataCheck := func(file FileMetadata, check interface{}, checkName string) { + switch c := check.(type) { + case filePropertiesCheck: + require.Equal(t, c.v, file.Properties, "%s-%s %s", stepName, checkName, path) + case fileNameCheck: + require.Equal(t, c.v, file.Name, "%s-%s %s", stepName, checkName, path) + case fileSizeCheck: + require.Equal(t, c.v, file.Size, "%s-%s %s", stepName, checkName, path) + case fileMimeTypeCheck: + require.Equal(t, c.v, file.MimeType, "%s-%s %s", stepName, checkName, path) + case filePathCheck: + require.Equal(t, c.v, file.FullPath, "%s-%s %s", stepName, checkName, path) + default: + t.Fatalf("unrecognized file check %s", checkName) + } + } + + switch o := output.(type) { + case File: + for _, check := range checks { + checkName := interfaceName(check) + if fileContentsCheck, ok := check.(fileContentsCheck); ok { + require.Equal(t, fileContentsCheck.v, o.Contents, "%s-%s %s", stepName, checkName, path) + } else { + runFileMetadataCheck(o.FileMetadata, check, checkName) + } + } + case FileMetadata: + for _, check := range checks { + runFileMetadataCheck(o, check, interfaceName(check)) + } + case ListFilesResponse: + for _, check := range checks { + c := check + checkName := interfaceName(c) + switch c := check.(type) { + case listSizeCheck: + require.Equal(t, c.v, len(o.Files), "%s %s", stepName, path) + case listHasMoreCheck: + require.Equal(t, c.v, o.HasMore, "%s %s", stepName, path) + case listLastPathCheck: + require.Equal(t, c.v, o.LastPath, "%s %s", stepName, path) + default: + t.Fatalf("unrecognized list check %s", checkName) + } + } + default: + t.Fatalf("unrecognized output %s", interfaceName(output)) + } + +} + +func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName string, fs FileStorage) { + t.Helper() + + switch q := query.(type) { + case queryGet: + inputPath := q.input.path + file, err := fs.Get(ctx, inputPath) + require.NoError(t, err, "%s: should be able to get file %s", queryName, inputPath) + + if q.checks != nil && len(q.checks) > 0 { + require.NotNil(t, file, "%s %s", queryName, inputPath) + require.Equal(t, inputPath, file.FullPath, "%s %s", queryName, inputPath) + runChecks(t, queryName, inputPath, *file, q.checks) + } else { + require.Nil(t, file, "%s %s", queryName, inputPath) + } + case queryListFiles: + inputPath := q.input.path + resp, err := fs.ListFiles(ctx, inputPath, q.input.paging, q.input.options) + require.NoError(t, err, "%s: should be able to list files in %s", queryName, inputPath) + require.NotNil(t, resp) + if q.list != nil && len(q.list) > 0 { + runChecks(t, queryName, inputPath, *resp, q.list) + } else { + require.NotNil(t, resp, "%s %s", queryName, inputPath) + require.Equal(t, false, resp.HasMore, "%s %s", queryName, inputPath) + require.Equal(t, 0, len(resp.Files), "%s %s", queryName, inputPath) + require.Equal(t, "", resp.LastPath, "%s %s", queryName, inputPath) + } + + if q.files != nil { + require.Equal(t, len(resp.Files), len(q.files), "%s %s", queryName, inputPath) + for i, file := range resp.Files { + runChecks(t, queryName, inputPath, file, q.files[i]) + } + } + case queryListFolders: + inputPath := q.input.path + resp, err := fs.ListFolders(ctx, inputPath, q.input.options) + require.NotNil(t, resp) + require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath) + + if q.checks != nil { + require.Equal(t, len(resp), len(q.checks), "%s %s", queryName, inputPath) + for i, file := range resp { + runChecks(t, queryName, inputPath, file, q.checks[i]) + } + } else { + require.Equal(t, 0, len(resp), "%s %s", queryName, inputPath) + } + default: + t.Fatalf("unrecognized query %s", queryName) + } +} + +func executeTestStep(t *testing.T, ctx context.Context, step interface{}, stepNumber int, fs FileStorage) { + name := fmt.Sprintf("[%d]%s", stepNumber, interfaceName(step)) + + switch s := step.(type) { + case queryGet: + handleQuery(t, ctx, s, name, fs) + case queryListFiles: + handleQuery(t, ctx, s, name, fs) + case queryListFolders: + handleQuery(t, ctx, s, name, fs) + case cmdUpsert: + handleCommand(t, ctx, s, name, fs) + case cmdDelete: + handleCommand(t, ctx, s, name, fs) + case cmdCreateFolder: + handleCommand(t, ctx, s, name, fs) + case cmdDeleteFolder: + handleCommand(t, ctx, s, name, fs) + default: + t.Fatalf("unrecognized step %s", name) + } + +} diff --git a/pkg/infra/filestorage/wrapper.go b/pkg/infra/filestorage/wrapper.go index 605b61188d5..e9309c5b169 100644 --- a/pkg/infra/filestorage/wrapper.go +++ b/pkg/infra/filestorage/wrapper.go @@ -208,3 +208,7 @@ func (b wrapper) DeleteFolder(ctx context.Context, path string) error { return b.wrapped.DeleteFolder(ctx, path) } + +func (c wrapper) close() error { + return c.wrapped.close() +}