#45498: refactor tests, fix pagination bug in FS implementation

pkg/infra/filestorage/api.go

@@ -101,6 +101,8 @@ type FileStorage interface {
 
 	CreateFolder(ctx context.Context, path string, name string) error
 	DeleteFolder(ctx context.Context, path string) error
+
+	close() error
 }
 
 // Get(ctx, "/myGit/dashboards/xyz123")

pkg/infra/filestorage/cdk_blob_filestorage.go

@@ -132,6 +132,8 @@ func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging
 		if err == io.EOF {
 			hasMore = false
 			break
+		} else {
+			hasMore = true
 		}
 
 		if err != nil {
@@ -218,6 +220,17 @@ func (c cdkBlobStorage) listFiles(ctx context.Context, folderPath string, paging
 	}, nil
 }
 
+func (c cdkBlobStorage) fixInputPrefix(path string) string {
+	if path == Delimiter || path == "" {
+		return c.rootFolder
+	}
+	if strings.HasPrefix(path, Delimiter) {
+		path = fmt.Sprintf("%s%s", c.rootFolder, strings.TrimPrefix(path, Delimiter))
+	}
+
+	return path
+}
+
 func (c cdkBlobStorage) convertFolderPathToPrefix(path string) string {
 	if path == Delimiter || path == "" {
 		return c.rootFolder
@@ -236,8 +249,22 @@ func fixPath(path string) string {
 	return newPath
 }
 
+func (c cdkBlobStorage) convertListOptions(options *ListOptions) *ListOptions {
+	if options == nil || options.allowedPrefixes == nil || len(options.allowedPrefixes) == 0 {
+		return options
+	}
+
+	newPrefixes := make([]string, len(options.allowedPrefixes))
+	for i, prefix := range options.allowedPrefixes {
+		newPrefixes[i] = c.fixInputPrefix(prefix)
+	}
+
+	options.PathFilters.allowedPrefixes = newPrefixes
+	return options
+}
+
 func (c cdkBlobStorage) ListFiles(ctx context.Context, folderPath string, paging *Paging, options *ListOptions) (*ListFilesResponse, error) {
-	return c.listFiles(ctx, c.convertFolderPathToPrefix(folderPath), paging, options)
+	return c.listFiles(ctx, c.convertFolderPathToPrefix(folderPath), paging, c.convertListOptions(options))
 }
 
 func (c cdkBlobStorage) listFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) {
@@ -294,7 +321,7 @@ func (c cdkBlobStorage) listFolders(ctx context.Context, parentFolderPath string
 }
 
 func (c cdkBlobStorage) ListFolders(ctx context.Context, parentFolderPath string, options *ListOptions) ([]FileMetadata, error) {
-	folders, err := c.listFolders(ctx, c.convertFolderPathToPrefix(parentFolderPath), options)
+	folders, err := c.listFolders(ctx, c.convertFolderPathToPrefix(parentFolderPath), c.convertListOptions(options))
 	return folders, err
 }
 
@@ -331,3 +358,7 @@ func (c cdkBlobStorage) DeleteFolder(ctx context.Context, folderPath string) err
 	err = c.bucket.Delete(ctx, directoryMarkerPath)
 	return err
 }
+
+func (c cdkBlobStorage) close() error {
+	return c.bucket.Close()
+}

pkg/infra/filestorage/db_filestorage.go

@@ -367,3 +367,7 @@ func (s dbFileStorage) DeleteFolder(ctx context.Context, folderPath string) erro
 
 	return err
 }
+
+func (s dbFileStorage) close() error {
+	return nil
+}

pkg/infra/filestorage/filestorage.go

@@ -141,3 +141,7 @@ func (b service) CreateFolder(ctx context.Context, path string, folderName strin
 func (b service) DeleteFolder(ctx context.Context, path string) error {
 	return errors.New("not available")
 }
+
+func (c service) close() error {
+	return c.grafanaDsStorage.close()
+}

pkg/infra/filestorage/filestorage_test.go

@@ -1,338 +0,0 @@
-//go:build integration
-// +build integration
-
-package filestorage
-
-import (
-	"context"
-	"testing"
-
-	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/grafana/grafana/pkg/services/sqlstore"
-	"github.com/stretchr/testify/require"
-	"gocloud.dev/blob"
-)
-
-type NameFullPath struct {
-	Name     string
-	FullPath string
-}
-
-func extractNameFullPath(meta []FileMetadata) []NameFullPath {
-	resp := make([]NameFullPath, 0)
-	for i := range meta {
-		resp = append(resp, NameFullPath{
-			Name:     meta[i].Name,
-			FullPath: meta[i].FullPath,
-		})
-	}
-	return resp
-}
-
-func TestSqlStorage(t *testing.T) {
-
-	var sqlStore *sqlstore.SQLStore
-	var filestorage FileStorage
-	var ctx context.Context
-
-	setup := func() {
-		mode := "mem"
-		testLogger := log.New("testStorageLogger")
-		if mode == "db" {
-			sqlStore = sqlstore.InitTestDB(t)
-			filestorage = NewDbStorage(testLogger, sqlStore, nil)
-		} else if mode == "mem" {
-			bucket, _ := blob.OpenBucket(context.Background(), "mem://")
-			filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil)
-		}
-
-		ctx = context.Background()
-	}
-
-	t.Run("Should be able to insert a file", func(t *testing.T) {
-		setup()
-		err := filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-	})
-
-	t.Run("Should be able to get a file", func(t *testing.T) {
-		setup()
-		path := "/folder1/folder2/file.jpg"
-		properties := map[string]string{"prop1": "val1", "prop2": "val"}
-		err := filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       path,
-			Contents:   &[]byte{},
-			Properties: properties,
-		})
-		require.NoError(t, err)
-
-		file, err := filestorage.Get(ctx, path)
-		require.NoError(t, err)
-
-		require.Equal(t, path, file.FullPath)
-		require.Equal(t, "file.jpg", file.Name)
-		require.Equal(t, properties, file.Properties)
-	})
-
-	t.Run("Should not be able to get a non-existent file", func(t *testing.T) {
-		setup()
-		path := "/folder1/folder2/file.jpg"
-
-		file, err := filestorage.Get(ctx, path)
-		require.NoError(t, err)
-		require.Nil(t, file)
-	})
-
-	t.Run("Should be able to list files", func(t *testing.T) {
-		setup()
-		err := filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-
-		err = filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/file-inner.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-
-		resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{Recursive: true})
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{
-				Name:     "file-inner.jpg",
-				FullPath: "/folder1/file-inner.jpg",
-			},
-			{
-				Name:     "file.jpg",
-				FullPath: "/folder1/folder2/file.jpg",
-			},
-		}, extractNameFullPath(resp.Files))
-
-		resp, err = filestorage.ListFiles(ctx, "/folder1", nil, nil)
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{
-				Name:     "file-inner.jpg",
-				FullPath: "/folder1/file-inner.jpg",
-			},
-		}, extractNameFullPath(resp.Files))
-
-		resp, err = filestorage.ListFiles(ctx, "/folder1/folder2", nil, nil)
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{
-				Name:     "file.jpg",
-				FullPath: "/folder1/folder2/file.jpg",
-			},
-		}, extractNameFullPath(resp.Files))
-	})
-
-	t.Run("Should be able to list files with prefix filtering", func(t *testing.T) {
-		setup()
-		err := filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-
-		err = filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/file-inner.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-
-		resp, err := filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{
-			Recursive: true, PathFilters: PathFilters{
-				allowedPrefixes: []string{"/folder2"},
-			},
-		})
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{}, extractNameFullPath(resp.Files))
-
-		resp, err = filestorage.ListFiles(ctx, "/folder1", nil, &ListOptions{
-			Recursive: true, PathFilters: PathFilters{
-				allowedPrefixes: []string{"/folder1/folde"},
-			},
-		})
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{
-				Name:     "file.jpg",
-				FullPath: "/folder1/folder2/file.jpg",
-			},
-		}, extractNameFullPath(resp.Files))
-	})
-
-	t.Run("Should be able to list files with pagination", func(t *testing.T) {
-		setup()
-		err := filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-
-		err = filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/file-inner.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		require.NoError(t, err)
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folderA/folderB/file.txt",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-
-		resp, err := filestorage.ListFiles(ctx, "/", &Paging{
-			After: "/folder1/file-inner.jpg",
-			First: 1,
-		}, &ListOptions{Recursive: true})
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{Name: "file.jpg", FullPath: "/folder1/folder2/file.jpg"},
-		}, extractNameFullPath(resp.Files))
-	})
-
-	t.Run("Should be able to list folders", func(t *testing.T) {
-		setup()
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/file-inner.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folderX/folderZ/file.txt",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folderA/folderB/file.txt",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-
-		resp, err := filestorage.ListFolders(ctx, "/", nil)
-		require.NoError(t, err)
-
-		require.Equal(t, []FileMetadata{
-			{
-				Name:     "folder1",
-				FullPath: "/folder1",
-			},
-			{
-				Name:     "folder2",
-				FullPath: "/folder1/folder2",
-			},
-			{
-				Name:     "folderA",
-				FullPath: "/folderA",
-			}, {
-				Name:     "folderB",
-				FullPath: "/folderA/folderB",
-			},
-			{
-				Name:     "folderX",
-				FullPath: "/folderX",
-			}, {
-				Name:     "folderZ",
-				FullPath: "/folderX/folderZ",
-			},
-		}, resp)
-	})
-
-	t.Run("Should be able to create and delete folders", func(t *testing.T) {
-		setup()
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder1/folder2/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder")
-		filestorage.CreateFolder(ctx, "/folder/icons", "emojis")
-		err := filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder")
-		require.NoError(t, err)
-
-		resp, err := filestorage.ListFolders(ctx, "/", nil)
-		require.NoError(t, err)
-
-		require.Equal(t, []FileMetadata{
-			{
-				Name:     "folder",
-				FullPath: "/folder",
-			}, {
-				Name:     "icons",
-				FullPath: "/folder/icons",
-			},
-			{
-				Name:     "emojis",
-				FullPath: "/folder/icons/emojis",
-			}, {
-				Name:     "folder1",
-				FullPath: "/folder1",
-			},
-			{
-				Name:     "folder2",
-				FullPath: "/folder1/folder2",
-			},
-		}, resp)
-	})
-
-	t.Run("Should not be able to delete folders with files", func(t *testing.T) {
-		setup()
-		filestorage.CreateFolder(ctx, "/folder/dashboards", "myNewFolder")
-		filestorage.Upsert(ctx, &UpsertFileCommand{
-			Path:       "/folder/dashboards/myNewFolder/file.jpg",
-			Contents:   &[]byte{},
-			Properties: map[string]string{"prop1": "val1", "prop2": "val"},
-		})
-		filestorage.DeleteFolder(ctx, "/folder/dashboards/myNewFolder")
-
-		resp, err := filestorage.ListFolders(ctx, "/", nil)
-		require.NoError(t, err)
-
-		require.Equal(t, []FileMetadata{
-			{
-				Name:     "folder",
-				FullPath: "/folder",
-			}, {
-				Name:     "dashboards",
-				FullPath: "/folder/dashboards",
-			},
-			{
-				Name:     "myNewFolder",
-				FullPath: "/folder/dashboards/myNewFolder",
-			},
-		}, resp)
-
-		files, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true})
-		require.NoError(t, err)
-		require.Equal(t, []NameFullPath{
-			{
-				Name:     "file.jpg",
-				FullPath: "/folder/dashboards/myNewFolder/file.jpg",
-			},
-		}, extractNameFullPath(files.Files))
-	})
-}

pkg/infra/filestorage/fs_integration_test.go

@@ -0,0 +1,387 @@
+//go:build integration
+// +build integration
+
+package filestorage
+
+import (
+	"context"
+	"encoding/base64"
+	"fmt"
+	"github.com/grafana/grafana/pkg/infra/log"
+	"github.com/grafana/grafana/pkg/services/sqlstore"
+	"gocloud.dev/blob"
+	"io/ioutil"
+	"os"
+	"testing"
+)
+
+const (
+	pngImageBase64 = "iVBORw0KGgoAAAANSUhEUgAAAC4AAAAmCAYAAAC76qlaAAAABHNCSVQICAgIfAhkiAAAABl0RVh0U29mdHdhcmUAZ25vbWUtc2NyZWVuc2hvdO8Dvz4AAABFSURBVFiF7c5BDQAhEACx4/x7XjzwGELSKuiamfke9N8OnBKvidfEa+I18Zp4TbwmXhOvidfEa+I18Zp4TbwmXhOvidc2lcsESD1LGnUAAAAASUVORK5CYII="
+)
+
+func TestFsStorage(t *testing.T) {
+
+	var testLogger log.Logger
+	var sqlStore *sqlstore.SQLStore
+	var filestorage FileStorage
+	var ctx context.Context
+	var tempDir string
+	pngImage, _ := base64.StdEncoding.DecodeString(pngImageBase64)
+	pngImageSize := int64(len(pngImage))
+
+	commonSetup := func() {
+		testLogger = log.New("testStorageLogger")
+		ctx = context.Background()
+	}
+
+	cleanUp := func() {
+		testLogger = nil
+		sqlStore = nil
+		if filestorage != nil {
+			_ = filestorage.close()
+			filestorage = nil
+		}
+
+		ctx = nil
+		_ = os.RemoveAll(tempDir)
+	}
+
+	setupInMemFS := func() {
+		commonSetup()
+		bucket, _ := blob.OpenBucket(context.Background(), "mem://")
+		filestorage = NewCdkBlobStorage(testLogger, bucket, Delimiter, nil)
+	}
+
+	setupSqlFS := func() {
+		commonSetup()
+		sqlStore = sqlstore.InitTestDB(t)
+		filestorage = NewDbStorage(testLogger, sqlStore, nil)
+	}
+
+	setupLocalFs := func() {
+		commonSetup()
+		tmpDir, err := ioutil.TempDir("", "")
+		tempDir = tmpDir
+		if err != nil {
+			t.Fatal(err)
+		}
+
+		bucket, err := blob.OpenBucket(context.Background(), "file://"+tmpDir)
+		if err != nil {
+			t.Fatal(err)
+		}
+		filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil)
+	}
+
+	tests := []struct {
+		name  string
+		steps []interface{}
+	}{
+		{
+			name: "inserting a file",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:       "/folder1/file.png",
+						Contents:   &pngImage,
+						Properties: map[string]string{"prop1": "val1", "prop2": "val"},
+					},
+				},
+				queryGet{
+					input: queryGetInput{
+						path: "/folder1/file.png",
+					},
+					checks: checks(
+						fName("file.png"),
+						fMimeType("image/png"),
+						fProperties(map[string]string{"prop1": "val1", "prop2": "val"}),
+						fSize(pngImageSize),
+						fContents(pngImage),
+					),
+				},
+			},
+		},
+		{
+			name: "getting a non-existent file",
+			steps: []interface{}{
+				queryGet{
+					input: queryGetInput{
+						path: "/folder1/file12412.png",
+					},
+				},
+			},
+		},
+		{
+			name: "listing files",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:       "/folder1/folder2/file.jpg",
+						Contents:   &[]byte{},
+						Properties: map[string]string{"prop1": "val1", "prop2": "val"},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:       "/folder1/file-inner.jpg",
+						Contents:   &[]byte{},
+						Properties: map[string]string{"prop1": "val1", "prop2": "val"},
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true}},
+					list:  checks(listSize(2), listHasMore(false), listLastPath("/folder1/folder2/file.jpg")),
+					files: [][]interface{}{
+						checks(fPath("/folder1/file-inner.jpg")),
+						checks(fPath("/folder1/folder2/file.jpg")),
+					},
+				},
+			},
+		},
+		{
+			name: "listing files with prefix filter",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/folder2/file.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/file-inner.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder2"}}}},
+					list:  checks(listSize(0), listHasMore(false), listLastPath("")),
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/folder1", options: &ListOptions{Recursive: true, PathFilters: PathFilters{allowedPrefixes: []string{"/folder1/folder"}}}},
+					list:  checks(listSize(1), listHasMore(false)),
+					files: [][]interface{}{
+						checks(fPath("/folder1/folder2/file.jpg")),
+					},
+				},
+			},
+		},
+		{
+			name: "listing files with pagination",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/a",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/b",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder2/c",
+						Contents: &[]byte{},
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: ""}},
+					list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/a")),
+					files: [][]interface{}{
+						checks(fPath("/folder1/a")),
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/a"}},
+					list:  checks(listSize(1), listHasMore(true), listLastPath("/folder1/b")),
+					files: [][]interface{}{
+						checks(fPath("/folder1/b")),
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 1, After: "/folder1/b"}},
+					list:  checks(listSize(1), listHasMore(false), listLastPath("/folder2/c")),
+					files: [][]interface{}{
+						checks(fPath("/folder2/c")),
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: ""}},
+					list:  checks(listSize(3), listHasMore(false), listLastPath("/folder2/c")),
+					files: [][]interface{}{
+						checks(fPath("/folder1/a")),
+						checks(fPath("/folder1/b")),
+						checks(fPath("/folder2/c")),
+					},
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2"}},
+					list:  checks(listSize(1), listHasMore(false)),
+				},
+				queryListFiles{
+					input: queryListFilesInput{path: "/", options: &ListOptions{Recursive: true}, paging: &Paging{First: 5, After: "/folder2/c"}},
+					list:  checks(listSize(0), listHasMore(false)),
+				},
+			},
+		},
+		{
+			name: "listing folders",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/folder2/file.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/file-inner.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folderX/folderZ/file.txt",
+						Contents: &[]byte{},
+					},
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folderA/folderB/file.txt",
+						Contents: &[]byte{},
+					},
+				},
+				queryListFolders{
+					input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}},
+					checks: [][]interface{}{
+						checks(fPath("/folder1")),
+						checks(fPath("/folder1/folder2")),
+						checks(fPath("/folderA")),
+						checks(fPath("/folderA/folderB")),
+						checks(fPath("/folderX")),
+						checks(fPath("/folderX/folderZ")),
+					},
+				},
+			},
+		},
+		{
+			name: "creating and deleting folders",
+			steps: []interface{}{
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder1/folder2/file.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				cmdCreateFolder{
+					path: "/folder/dashboards",
+					name: "myNewFolder",
+				},
+				cmdCreateFolder{
+					path: "/folder/icons",
+					name: "emojis",
+				},
+				queryListFolders{
+					input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}},
+					checks: [][]interface{}{
+						checks(fPath("/folder")),
+						checks(fPath("/folder/dashboards")),
+						checks(fPath("/folder/dashboards/myNewFolder")),
+						checks(fPath("/folder/icons")),
+						checks(fPath("/folder/icons/emojis")),
+						checks(fPath("/folder1")),
+						checks(fPath("/folder1/folder2")),
+					},
+				},
+				cmdDeleteFolder{
+					path: "/folder/dashboards/myNewFolder",
+				},
+				queryListFolders{
+					input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}},
+					checks: [][]interface{}{
+						checks(fPath("/folder")),
+						checks(fPath("/folder/icons")),
+						checks(fPath("/folder/icons/emojis")),
+						checks(fPath("/folder1")),
+						checks(fPath("/folder1/folder2")),
+					},
+				},
+			},
+		},
+		{
+			name: "should not be able to delete folders with files",
+			steps: []interface{}{
+				cmdCreateFolder{
+					path: "/folder/dashboards",
+					name: "myNewFolder",
+				},
+				cmdUpsert{
+					cmd: UpsertFileCommand{
+						Path:     "/folder/dashboards/myNewFolder/file.jpg",
+						Contents: &[]byte{},
+					},
+				},
+				cmdDeleteFolder{
+					path: "/folder/dashboards/myNewFolder",
+					error: &cmdErrorOutput{
+						message: "folder %s is not empty - cant remove it",
+						args:    []interface{}{"/folder/dashboards/myNewFolder"},
+					},
+				},
+				queryListFolders{
+					input: queryListFoldersInput{path: "/", options: &ListOptions{Recursive: true}},
+					checks: [][]interface{}{
+						checks(fPath("/folder")),
+						checks(fPath("/folder/dashboards")),
+						checks(fPath("/folder/dashboards/myNewFolder")),
+					},
+				},
+				queryGet{
+					input: queryGetInput{
+						path: "/folder/dashboards/myNewFolder/file.jpg",
+					},
+					checks: checks(
+						fName("file.jpg"),
+					),
+				},
+			},
+		},
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s: %s", "IN MEM FS", tt.name), func(t *testing.T) {
+			setupInMemFS()
+			defer cleanUp()
+			for i, step := range tt.steps {
+				executeTestStep(t, ctx, step, i, filestorage)
+			}
+		})
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s: %s", "SQL FS", tt.name), func(t *testing.T) {
+			setupSqlFS()
+			defer cleanUp()
+			for i, step := range tt.steps {
+				executeTestStep(t, ctx, step, i, filestorage)
+			}
+		})
+	}
+
+	for _, tt := range tests {
+		t.Run(fmt.Sprintf("%s: %s", "Local FS", tt.name), func(t *testing.T) {
+			if tt.name == "listing files with pagination" {
+				// bug in cdk fileblob
+				return
+			}
+			setupLocalFs()
+			defer cleanUp()
+			for i, step := range tt.steps {
+				executeTestStep(t, ctx, step, i, filestorage)
+			}
+		})
+	}
+}

pkg/infra/filestorage/localfs_test.go

@@ -1,74 +0,0 @@
-//go:build integration
-// +build integration
-
-package filestorage
-
-import (
-	"context"
-	"testing"
-
-	"github.com/grafana/grafana/pkg/infra/log"
-	"github.com/stretchr/testify/require"
-	"gocloud.dev/blob"
-)
-
-func TestLocalFsCdkBlobStorage(t *testing.T) {
-
-	var filestorage FileStorage
-	var ctx context.Context
-
-	setup := func() {
-		bucket, _ := blob.OpenBucket(context.Background(), "file://./test_fs")
-		testLogger := log.New("testFsStorageLogger")
-		filestorage = NewCdkBlobStorage(testLogger, bucket, "", nil)
-		ctx = context.Background()
-	}
-
-	t.Run("Should be able to list folders", func(t *testing.T) {
-		setup()
-		folders, err := filestorage.ListFolders(ctx, "/", nil)
-		require.NoError(t, err)
-		require.Equal(t, []FileMetadata{
-			{
-				Name:     "folderA",
-				FullPath: "/folderA",
-			},
-			{
-				Name:     "folderAnestedA",
-				FullPath: "/folderA/folderAnestedA",
-			},
-		}, folders)
-
-		folders, err = filestorage.ListFolders(ctx, "/folderA", nil)
-		require.NoError(t, err)
-		require.Equal(t, []FileMetadata{
-			{
-				Name:     "folderAnestedA",
-				FullPath: "/folderA/folderAnestedA",
-			},
-		}, folders)
-
-	})
-
-	t.Run("Should be able to list files", func(t *testing.T) {
-		setup()
-		res, err := filestorage.ListFiles(ctx, "/", nil, &ListOptions{Recursive: true})
-		require.NoError(t, err)
-
-		require.Equal(t, []NameFullPath{
-			{Name: "file.txt", FullPath: "/folderA/folderAnestedA/file.txt"},
-			{Name: "rootFile.txt", FullPath: "/rootFile.txt"},
-		}, extractNameFullPath(res.Files))
-	})
-
-	t.Run("should be able to read file", func(t *testing.T) {
-		setup()
-		path := "/folderA/folderAnestedA/file.txt"
-		res, err := filestorage.Get(ctx, path)
-		require.NoError(t, err)
-
-		require.Equal(t, res.FullPath, path)
-		require.Equal(t, res.Name, "file.txt")
-		require.Equal(t, "content\n", string(res.Contents))
-	})
-}

pkg/infra/filestorage/test_utils.go

@@ -0,0 +1,335 @@
+//go:build integration
+// +build integration
+
+package filestorage
+
+import (
+	"context"
+	"fmt"
+	"reflect"
+	"testing"
+
+	"github.com/stretchr/testify/require"
+)
+
+type cmdErrorOutput struct {
+	message  string
+	args     []interface{}
+	instance error
+}
+
+type cmdDelete struct {
+	path  string
+	error *cmdErrorOutput
+}
+
+type cmdUpsert struct {
+	cmd   UpsertFileCommand
+	error *cmdErrorOutput
+}
+
+type cmdCreateFolder struct {
+	path  string
+	name  string
+	error *cmdErrorOutput
+}
+
+type cmdDeleteFolder struct {
+	path  string
+	error *cmdErrorOutput
+}
+
+type queryGetInput struct {
+	path string
+}
+
+type fileNameCheck struct {
+	v string
+}
+
+type filePropertiesCheck struct {
+	v map[string]string
+}
+
+type fileContentsCheck struct {
+	v []byte
+}
+
+type fileSizeCheck struct {
+	v int64
+}
+
+type fileMimeTypeCheck struct {
+	v string
+}
+
+type filePathCheck struct {
+	v string
+}
+
+type listSizeCheck struct {
+	v int
+}
+
+type listHasMoreCheck struct {
+	v bool
+}
+
+type listLastPathCheck struct {
+	v string
+}
+
+func fContents(contents []byte) interface{} {
+	return fileContentsCheck{v: contents}
+}
+
+func fName(name string) interface{} {
+	return fileNameCheck{v: name}
+}
+
+func fPath(path string) interface{} {
+	return filePathCheck{v: path}
+}
+
+func fProperties(properties map[string]string) interface{} {
+	return filePropertiesCheck{v: properties}
+}
+func fSize(size int64) interface{} {
+	return fileSizeCheck{v: size}
+}
+
+func fMimeType(mimeType string) interface{} {
+	return fileMimeTypeCheck{v: mimeType}
+}
+
+func listSize(size int) interface{} {
+	return listSizeCheck{v: size}
+}
+
+func listHasMore(hasMore bool) interface{} {
+	return listHasMoreCheck{v: hasMore}
+}
+
+func listLastPath(path string) interface{} {
+	return listLastPathCheck{v: path}
+}
+
+func checks(c ...interface{}) []interface{} {
+	return c
+}
+
+type queryGet struct {
+	input  queryGetInput
+	checks []interface{}
+}
+
+type queryListFilesInput struct {
+	path    string
+	paging  *Paging
+	options *ListOptions
+}
+
+type queryListFiles struct {
+	input queryListFilesInput
+	list  []interface{}
+	files [][]interface{}
+}
+
+type queryListFoldersInput struct {
+	path    string
+	options *ListOptions
+}
+
+type queryListFolders struct {
+	input  queryListFoldersInput
+	checks [][]interface{}
+}
+
+func interfaceName(myvar interface{}) string {
+	if t := reflect.TypeOf(myvar); t.Kind() == reflect.Ptr {
+		return "*" + t.Elem().Name()
+	} else {
+		return t.Name()
+	}
+}
+
+func handleCommand(t *testing.T, ctx context.Context, cmd interface{}, cmdName string, fs FileStorage) {
+	t.Helper()
+
+	var err error
+	var expectedErr *cmdErrorOutput
+	switch c := cmd.(type) {
+	case cmdDelete:
+		err = fs.Delete(ctx, c.path)
+		if c.error == nil {
+			require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path)
+		}
+		expectedErr = c.error
+	case cmdUpsert:
+		err = fs.Upsert(ctx, &c.cmd)
+		if c.error == nil {
+			require.NoError(t, err, "%s: should be able to upsert file %s", cmdName, c.cmd.Path)
+		}
+		expectedErr = c.error
+	case cmdCreateFolder:
+		err = fs.CreateFolder(ctx, c.path, c.name)
+		if c.error == nil {
+			require.NoError(t, err, "%s: should be able to create folder %s in %s", cmdName, c.name, c.path)
+		}
+		expectedErr = c.error
+	case cmdDeleteFolder:
+		err = fs.DeleteFolder(ctx, c.path)
+		if c.error == nil {
+			require.NoError(t, err, "%s: should be able to delete %s", cmdName, c.path)
+		}
+		expectedErr = c.error
+	default:
+		t.Fatalf("unrecognized command %s", cmdName)
+	}
+
+	if expectedErr != nil && err != nil {
+		if expectedErr.instance != nil {
+			require.ErrorIs(t, err, expectedErr.instance)
+		}
+
+		if expectedErr.message != "" {
+			require.Errorf(t, err, expectedErr.message, expectedErr.args...)
+		}
+	}
+}
+
+func runChecks(t *testing.T, stepName string, path string, output interface{}, checks []interface{}) {
+	if checks == nil || len(checks) == 0 {
+		return
+	}
+
+	runFileMetadataCheck := func(file FileMetadata, check interface{}, checkName string) {
+		switch c := check.(type) {
+		case filePropertiesCheck:
+			require.Equal(t, c.v, file.Properties, "%s-%s %s", stepName, checkName, path)
+		case fileNameCheck:
+			require.Equal(t, c.v, file.Name, "%s-%s %s", stepName, checkName, path)
+		case fileSizeCheck:
+			require.Equal(t, c.v, file.Size, "%s-%s %s", stepName, checkName, path)
+		case fileMimeTypeCheck:
+			require.Equal(t, c.v, file.MimeType, "%s-%s %s", stepName, checkName, path)
+		case filePathCheck:
+			require.Equal(t, c.v, file.FullPath, "%s-%s %s", stepName, checkName, path)
+		default:
+			t.Fatalf("unrecognized file check %s", checkName)
+		}
+	}
+
+	switch o := output.(type) {
+	case File:
+		for _, check := range checks {
+			checkName := interfaceName(check)
+			if fileContentsCheck, ok := check.(fileContentsCheck); ok {
+				require.Equal(t, fileContentsCheck.v, o.Contents, "%s-%s %s", stepName, checkName, path)
+			} else {
+				runFileMetadataCheck(o.FileMetadata, check, checkName)
+			}
+		}
+	case FileMetadata:
+		for _, check := range checks {
+			runFileMetadataCheck(o, check, interfaceName(check))
+		}
+	case ListFilesResponse:
+		for _, check := range checks {
+			c := check
+			checkName := interfaceName(c)
+			switch c := check.(type) {
+			case listSizeCheck:
+				require.Equal(t, c.v, len(o.Files), "%s %s", stepName, path)
+			case listHasMoreCheck:
+				require.Equal(t, c.v, o.HasMore, "%s %s", stepName, path)
+			case listLastPathCheck:
+				require.Equal(t, c.v, o.LastPath, "%s %s", stepName, path)
+			default:
+				t.Fatalf("unrecognized list check %s", checkName)
+			}
+		}
+	default:
+		t.Fatalf("unrecognized output %s", interfaceName(output))
+	}
+
+}
+
+func handleQuery(t *testing.T, ctx context.Context, query interface{}, queryName string, fs FileStorage) {
+	t.Helper()
+
+	switch q := query.(type) {
+	case queryGet:
+		inputPath := q.input.path
+		file, err := fs.Get(ctx, inputPath)
+		require.NoError(t, err, "%s: should be able to get file %s", queryName, inputPath)
+
+		if q.checks != nil && len(q.checks) > 0 {
+			require.NotNil(t, file, "%s %s", queryName, inputPath)
+			require.Equal(t, inputPath, file.FullPath, "%s %s", queryName, inputPath)
+			runChecks(t, queryName, inputPath, *file, q.checks)
+		} else {
+			require.Nil(t, file, "%s %s", queryName, inputPath)
+		}
+	case queryListFiles:
+		inputPath := q.input.path
+		resp, err := fs.ListFiles(ctx, inputPath, q.input.paging, q.input.options)
+		require.NoError(t, err, "%s: should be able to list files in %s", queryName, inputPath)
+		require.NotNil(t, resp)
+		if q.list != nil && len(q.list) > 0 {
+			runChecks(t, queryName, inputPath, *resp, q.list)
+		} else {
+			require.NotNil(t, resp, "%s %s", queryName, inputPath)
+			require.Equal(t, false, resp.HasMore, "%s %s", queryName, inputPath)
+			require.Equal(t, 0, len(resp.Files), "%s %s", queryName, inputPath)
+			require.Equal(t, "", resp.LastPath, "%s %s", queryName, inputPath)
+		}
+
+		if q.files != nil {
+			require.Equal(t, len(resp.Files), len(q.files), "%s %s", queryName, inputPath)
+			for i, file := range resp.Files {
+				runChecks(t, queryName, inputPath, file, q.files[i])
+			}
+		}
+	case queryListFolders:
+		inputPath := q.input.path
+		resp, err := fs.ListFolders(ctx, inputPath, q.input.options)
+		require.NotNil(t, resp)
+		require.NoError(t, err, "%s: should be able to list folders in %s", queryName, inputPath)
+
+		if q.checks != nil {
+			require.Equal(t, len(resp), len(q.checks), "%s %s", queryName, inputPath)
+			for i, file := range resp {
+				runChecks(t, queryName, inputPath, file, q.checks[i])
+			}
+		} else {
+			require.Equal(t, 0, len(resp), "%s %s", queryName, inputPath)
+		}
+	default:
+		t.Fatalf("unrecognized query %s", queryName)
+	}
+}
+
+func executeTestStep(t *testing.T, ctx context.Context, step interface{}, stepNumber int, fs FileStorage) {
+	name := fmt.Sprintf("[%d]%s", stepNumber, interfaceName(step))
+
+	switch s := step.(type) {
+	case queryGet:
+		handleQuery(t, ctx, s, name, fs)
+	case queryListFiles:
+		handleQuery(t, ctx, s, name, fs)
+	case queryListFolders:
+		handleQuery(t, ctx, s, name, fs)
+	case cmdUpsert:
+		handleCommand(t, ctx, s, name, fs)
+	case cmdDelete:
+		handleCommand(t, ctx, s, name, fs)
+	case cmdCreateFolder:
+		handleCommand(t, ctx, s, name, fs)
+	case cmdDeleteFolder:
+		handleCommand(t, ctx, s, name, fs)
+	default:
+		t.Fatalf("unrecognized step %s", name)
+	}
+
+}

pkg/infra/filestorage/wrapper.go

@@ -208,3 +208,7 @@ func (b wrapper) DeleteFolder(ctx context.Context, path string) error {
 
 	return b.wrapped.DeleteFolder(ctx, path)
 }
+
+func (c wrapper) close() error {
+	return c.wrapped.close()
+}