grafana/pkg/services/store/service_test.go

package store

import (
	"bytes"
	"context"
	"fmt"
	"os"
	"path/filepath"
	"strings"
	"testing"

	"github.com/grafana/grafana-plugin-sdk-go/experimental"
	"github.com/stretchr/testify/mock"
	"github.com/stretchr/testify/require"

	"github.com/grafana/grafana/pkg/infra/db"
	"github.com/grafana/grafana/pkg/infra/filestorage"
	"github.com/grafana/grafana/pkg/services/accesscontrol"
	"github.com/grafana/grafana/pkg/services/quota/quotatest"
	"github.com/grafana/grafana/pkg/services/user"
	"github.com/grafana/grafana/pkg/setting"
	"github.com/grafana/grafana/pkg/tests/testsuite"
	testdatasource "github.com/grafana/grafana/pkg/tsdb/grafana-testdata-datasource"
)

var (
	cfg = &setting.Cfg{
		Storage: setting.StorageSettings{
			AllowUnsanitizedSvgUpload: true,
		},
	}

	htmlBytes, _        = os.ReadFile("testdata/page.html")
	jpgBytes, _         = os.ReadFile("testdata/image.jpg")
	svgBytes, _         = os.ReadFile("testdata/image.svg")
	dummyUser           = &user.SignedInUser{OrgID: 1}
	globalUser          = &user.SignedInUser{OrgID: 0}
	allowAllAuthService = newStaticStorageAuthService(func(ctx context.Context, user *user.SignedInUser, storageName string) map[string]filestorage.PathFilter {
		return map[string]filestorage.PathFilter{
			ActionFilesDelete: allowAllPathFilter,
			ActionFilesWrite:  allowAllPathFilter,
			ActionFilesRead:   allowAllPathFilter,
		}
	})
	denyAllAuthService = newStaticStorageAuthService(func(ctx context.Context, user *user.SignedInUser, storageName string) map[string]filestorage.PathFilter {
		return map[string]filestorage.PathFilter{
			ActionFilesDelete: denyAllPathFilter,
			ActionFilesWrite:  denyAllPathFilter,
			ActionFilesRead:   denyAllPathFilter,
		}
	})
	publicRoot, _            = filepath.Abs("../../../public")
	publicStaticFilesStorage = newDiskStorage(
		RootStorageMeta{
			Builtin:  true,
			ReadOnly: true,
		}, RootStorageConfig{
			Prefix: "public",
			Name:   "Public static files",
			Disk: &StorageLocalDiskConfig{
				Path: publicRoot,
				Roots: []string{
					"/img/icons/",
					"/img/bg/",
					"/gazetteer/",
					"/maps/",
					"/upload/",
				},
			}})
)

func TestMain(m *testing.M) {
	testsuite.Run(m)
}

func TestListFiles(t *testing.T) {
	roots := []storageRuntime{publicStaticFilesStorage}

	store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
		return make([]storageRuntime, 0)
	}, allowAllAuthService, cfg, nil)
	frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
	require.NoError(t, err)

	experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata.golden", frame.Frame, true)

	file, err := store.Read(context.Background(), dummyUser, "public/maps/countries.geojson")
	require.NoError(t, err)
	require.NotNil(t, file)

	t.Skip("Skipping golden JSON frame test as it is flaky")
	testDsFrame, err := testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name)
	require.NoError(t, err)
	experimental.CheckGoldenJSONFrame(t, "testdata", "public_testdata_js_libraries.golden", testDsFrame, true)
}

func TestListFilesWithoutPermissions(t *testing.T) {
	roots := []storageRuntime{publicStaticFilesStorage}

	store := newStandardStorageService(db.InitTestDB(t), roots, func(orgId int64) []storageRuntime {
		return make([]storageRuntime, 0)
	}, denyAllAuthService, cfg, nil)
	frame, err := store.List(context.Background(), dummyUser, "public/maps", 0)
	require.NoError(t, err)
	rowLen, err := frame.RowLen()
	require.NoError(t, err)
	require.Equal(t, 0, rowLen)
}

func setupUploadStore(t *testing.T, authService storageAuthService) (StorageService, *filestorage.MockFileStorage, string) {
	t.Helper()
	storageName := "resources"
	mockStorage := &filestorage.MockFileStorage{}
	sqlStorage := newSQLStorage(RootStorageMeta{}, storageName, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)
	sqlStorage.store = mockStorage

	if authService == nil {
		authService = allowAllAuthService
	}
	store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage}, func(orgId int64) []storageRuntime {
		return make([]storageRuntime, 0)
	}, authService, cfg, nil)
	store.cfg = &GlobalStorageConfig{
		AllowUnsanitizedSvgUpload: true,
	}
	store.quotaService = quotatest.New(false, nil)

	return store, mockStorage, storageName
}

func TestShouldUploadWhenNoFileAlreadyExists(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	fileName := "/myFile.jpg"
	mockStorage.On("Get", mock.Anything, fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
	mockStorage.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
		Path:     fileName,
		MimeType: "image/jpeg",
		Contents: jpgBytes,
	}).Return(nil)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   jpgBytes,
		Path:       storageName + fileName,
	})
	require.NoError(t, err)
}

func TestShouldFailUploadWithoutAccess(t *testing.T) {
	service, _, storageName := setupUploadStore(t, denyAllAuthService)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   jpgBytes,
		Path:       storageName + "/myFile.jpg",
	})
	require.ErrorIs(t, err, ErrAccessDenied)
}

func TestShouldFailUploadWhenFileAlreadyExists(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	mockStorage.On("Get", mock.Anything, "/myFile.jpg", &filestorage.GetFileOptions{WithContents: false}).Return(&filestorage.File{Contents: make([]byte, 0)}, true, nil)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   jpgBytes,
		Path:       storageName + "/myFile.jpg",
	})
	require.ErrorIs(t, err, ErrFileAlreadyExists)
}

func TestShouldDelegateFileDeletion(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	mockStorage.On("Delete", mock.Anything, "/myFile.jpg").Return(nil)

	err := service.Delete(context.Background(), dummyUser, storageName+"/myFile.jpg")
	require.NoError(t, err)
}

func TestShouldDelegateFolderCreation(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	mockStorage.On("CreateFolder", mock.Anything, "/nestedFolder/mostNestedFolder").Return(nil)

	err := service.CreateFolder(context.Background(), dummyUser, &CreateFolderCmd{Path: storageName + "/nestedFolder/mostNestedFolder"})
	require.NoError(t, err)
}

func TestShouldDelegateFolderDeletion(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)
	cmds := []*DeleteFolderCmd{
		{
			Path:  storageName,
			Force: false,
		},
		{
			Path:  storageName,
			Force: true,
		}}

	ctx := context.Background()

	for _, cmd := range cmds {
		mockStorage.On("DeleteFolder", ctx, "/", &filestorage.DeleteFolderOptions{
			Force:        cmd.Force,
			AccessFilter: allowAllPathFilter,
		}).Once().Return(nil)
		err := service.DeleteFolder(ctx, dummyUser, cmd)
		require.NoError(t, err)
	}
}

func TestShouldUploadSvg(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	fileName := "/myFile.svg"
	mockStorage.On("Get", mock.Anything, fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
	mockStorage.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
		Path:     fileName,
		MimeType: "image/svg+xml",
		Contents: svgBytes,
	}).Return(nil)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   svgBytes,
		Path:       storageName + fileName,
	})
	require.NoError(t, err)
}

func TestShouldNotUploadHtmlDisguisedAsSvg(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	fileName := "/myFile.svg"
	mockStorage.On("Get", mock.Anything, fileName).Return(nil, nil)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   htmlBytes,
		Path:       storageName + fileName,
	})
	require.ErrorIs(t, err, ErrValidationFailed)
}

func TestShouldNotUploadJpgDisguisedAsSvg(t *testing.T) {
	service, mockStorage, storageName := setupUploadStore(t, nil)

	fileName := "/myFile.svg"
	mockStorage.On("Get", mock.Anything, fileName).Return(nil, nil)

	err := service.Upload(context.Background(), dummyUser, &UploadRequest{
		EntityType: EntityTypeImage,
		Contents:   jpgBytes,
		Path:       storageName + fileName,
	})
	require.ErrorIs(t, err, ErrValidationFailed)
}

func TestSetupWithNonUniqueStoragePrefixes(t *testing.T) {
	prefix := "resources"
	sqlStorage := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)
	sqlStorage2 := newSQLStorage(RootStorageMeta{}, prefix, "Testing upload", "dummy descr", &StorageSQLConfig{}, db.InitTestDB(t), 1, false)

	defer func() {
		if r := recover(); r == nil {
			t.Errorf("The setup should have panicked")
		}
	}()

	newStandardStorageService(db.InitTestDB(t), []storageRuntime{sqlStorage, sqlStorage2}, func(orgId int64) []storageRuntime {
		return make([]storageRuntime, 0)
	}, allowAllAuthService, cfg, nil)
}

func TestContentRootWithNestedStorage(t *testing.T) {
	globalOrgID := int64(accesscontrol.GlobalOrgID)
	testDB := db.InitTestDB(t)
	orgedUser := &user.SignedInUser{OrgID: 1}

	t.Helper()
	mockContentFSApi := &filestorage.MockFileStorage{}
	contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Content root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, false)
	contentStorage.store = mockContentFSApi

	nestedRoot := "nested"
	mockNestedFSApi := &filestorage.MockFileStorage{}
	nestedStorage := newSQLStorage(RootStorageMeta{}, nestedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, true)
	nestedStorage.store = mockNestedFSApi

	nestedOrgedRoot := "nestedOrged"
	mockNestedOrgedFSApi := &filestorage.MockFileStorage{}
	nestedOrgedStorage := newSQLStorage(RootStorageMeta{}, nestedOrgedRoot, "Nested root", "dummy descr", &StorageSQLConfig{}, testDB, globalOrgID, true)
	nestedOrgedStorage.store = mockNestedOrgedFSApi

	store := newStandardStorageService(db.InitTestDB(t), []storageRuntime{contentStorage, nestedStorage}, func(orgId int64) []storageRuntime {
		return []storageRuntime{nestedOrgedStorage, contentStorage}
	}, allowAllAuthService, cfg, nil)
	store.cfg = &GlobalStorageConfig{
		AllowUnsanitizedSvgUpload: true,
	}
	store.quotaService = quotatest.New(false, nil)
	fileName := "file.jpg"

	tests := []struct {
		user         *user.SignedInUser
		name         string
		mockNestedFS *filestorage.MockFileStorage
		nestedRoot   string
	}{
		{
			user:         globalUser,
			name:         "global user, global nested storage",
			mockNestedFS: mockNestedFSApi,
			nestedRoot:   nestedRoot,
		},
		{
			user:         orgedUser,
			name:         "non-global user, global nested storage",
			mockNestedFS: mockNestedFSApi,
			nestedRoot:   nestedRoot,
		},
		{
			user:         orgedUser,
			name:         "non-global user, non-global nested storage",
			mockNestedFS: mockNestedOrgedFSApi,
			nestedRoot:   nestedOrgedRoot,
		},
	}

	for _, test := range tests {
		t.Run(test.name+": Uploading a file under a /content/nested/.. should delegate to the nested storage", func(t *testing.T) {
			test.mockNestedFS.On("Get", mock.Anything, filestorage.Delimiter+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
			test.mockNestedFS.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
				Path:     filestorage.Delimiter + fileName,
				MimeType: "image/jpeg",
				Contents: jpgBytes,
			}).Return(nil)
			mockContentFSApi.AssertNotCalled(t, "Get")
			mockContentFSApi.AssertNotCalled(t, "Upsert")

			err := store.Upload(context.Background(), test.user, &UploadRequest{
				EntityType: EntityTypeImage,
				Contents:   jpgBytes,
				Path:       strings.Join([]string{RootContent, test.nestedRoot, fileName}, filestorage.Delimiter),
			})
			require.NoError(t, err)
		})

		t.Run(test.name+": Creating a /content/nested folder should fail", func(t *testing.T) {
			mockContentFSApi.AssertNotCalled(t, "CreateFolder")

			err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: RootContent + "/" + test.nestedRoot})
			require.ErrorIs(t, err, ErrValidationFailed)
		})

		t.Run(test.name+": Deleting a /content/nested folder should fail", func(t *testing.T) {
			mockContentFSApi.AssertNotCalled(t, "DeleteFolder")

			err := store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: RootContent + "/" + test.nestedRoot})
			require.ErrorIs(t, err, ErrValidationFailed)
		})

		t.Run(test.name+": Listing /content/nested should delegate to the nested root", func(t *testing.T) {
			mockContentFSApi.AssertNotCalled(t, "List")
			test.mockNestedFS.On(
				"List",
				mock.Anything,
				"/",
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			_, err := store.List(context.Background(), test.user, RootContent+"/"+test.nestedRoot, 0)
			require.NoError(t, err)
		})

		t.Run(test.name+": Listing a folder inside /content/nested/.. should delegate to the nested root", func(t *testing.T) {
			mockContentFSApi.AssertNotCalled(t, "List")
			test.mockNestedFS.On(
				"List",
				mock.Anything,
				"/folder1/folder2",
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot, "folder1", "folder2"}, "/"), 0)
			require.NoError(t, err)
		})

		t.Run(test.name+": Listing outside of the nested storages should delegate to the content root", func(t *testing.T) {
			test.mockNestedFS.AssertNotCalled(t, "List")

			mockContentFSApi.On(
				"List",
				mock.Anything,
				"/not-nested-content",
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			mockContentFSApi.On(
				"List",
				mock.Anything,
				"/a/b/c",
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			mockContentFSApi.On(
				"List",
				mock.Anything,
				fmt.Sprintf("/%sa", test.nestedRoot),
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			mockContentFSApi.On(
				"List",
				mock.Anything,
				fmt.Sprintf("/%sa/b", test.nestedRoot),
				mock.Anything,
				mock.Anything,
			).Return(&filestorage.ListResponse{
				Files: []*filestorage.File{},
			}, nil)

			_, err := store.List(context.Background(), test.user, strings.Join([]string{RootContent, "not-nested-content"}, "/"), 0)
			require.NoError(t, err)

			_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, "a", "b", "c"}, "/"), 0)
			require.NoError(t, err)

			_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a"}, "/"), 0)
			require.NoError(t, err)

			_, err = store.List(context.Background(), test.user, strings.Join([]string{RootContent, test.nestedRoot + "a", "b"}, "/"), 0)
			require.NoError(t, err)
		})

		t.Run(test.name+": Uploading files outside of the nested storages should delegate to the content root", func(t *testing.T) {
			test.mockNestedFS.AssertNotCalled(t, "Get")
			test.mockNestedFS.AssertNotCalled(t, "Upsert")

			// file at the root of the content root - /content/myFile.jpg
			fileName := "myFile.jpg"
			mockContentFSApi.On("Get", mock.Anything, "/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
			mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
				Path:     "/" + fileName,
				MimeType: "image/jpeg",
				Contents: jpgBytes,
			}).Return(nil)

			err := store.Upload(context.Background(), dummyUser, &UploadRequest{
				EntityType: EntityTypeImage,
				Contents:   jpgBytes,
				Path:       strings.Join([]string{RootContent, fileName}, "/"),
			})
			require.NoError(t, err)

			// file in the folder belonging to the content root storage - /content/nested/a/myFile.jpg
			mockContentFSApi.On("Get", mock.Anything, "/a/"+fileName, &filestorage.GetFileOptions{WithContents: false}).Return(nil, false, nil)
			mockContentFSApi.On("Upsert", mock.Anything, &filestorage.UpsertFileCommand{
				Path:     "/a/" + fileName,
				MimeType: "image/jpeg",
				Contents: jpgBytes,
			}).Return(nil)

			err = store.Upload(context.Background(), dummyUser, &UploadRequest{
				EntityType: EntityTypeImage,
				Contents:   jpgBytes,
				Path:       strings.Join([]string{RootContent, "a", fileName}, "/"),
			})
			require.NoError(t, err)
		})

		t.Run(test.name+": Creating folders under /content/nested/.. should delegate to the nested roots", func(t *testing.T) {
			mockContentFSApi.AssertNotCalled(t, "CreateFolder")
			mockContentFSApi.AssertNotCalled(t, "DeleteFolder")

			test.mockNestedFS.On("CreateFolder", mock.Anything, "/folder").Return(nil)

			path := strings.Join([]string{RootContent, test.nestedRoot, "folder"}, "/")
			err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
			require.NoError(t, err)

			test.mockNestedFS.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)

			err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
			require.NoError(t, err)
		})

		t.Run(test.name+": Creating folders under outside of the nested storages should delegate to the content root", func(t *testing.T) {
			test.mockNestedFS.AssertNotCalled(t, "CreateFolder")
			test.mockNestedFS.AssertNotCalled(t, "DeleteFolder")

			mockContentFSApi.On("CreateFolder", mock.Anything, "/folder").Return(nil)

			path := strings.Join([]string{RootContent, "folder"}, "/")
			err := store.CreateFolder(context.Background(), test.user, &CreateFolderCmd{Path: path})
			require.NoError(t, err)

			mockContentFSApi.On("DeleteFolder", mock.Anything, "/folder", mock.Anything).Return(nil)

			err = store.DeleteFolder(context.Background(), test.user, &DeleteFolderCmd{Path: path})
			require.NoError(t, err)
		})
	}
}

func TestShadowingExistingFolderByNestedContentRoot(t *testing.T) {
	db := db.InitTestDB(t)
	ctx := context.Background()
	nestedStorage := newSQLStorage(RootStorageMeta{}, "nested", "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, true)
	contentStorage := newSQLStorage(RootStorageMeta{}, RootContent, "Testing upload", "dummy descr", &StorageSQLConfig{}, db, accesscontrol.GlobalOrgID, false)

	_, err := contentStorage.Write(ctx, &WriteValueRequest{
		User:       globalUser,
		Path:       "/nested/abc.jpg",
		EntityType: EntityTypeImage,
		Body:       jpgBytes,
	})
	require.NoError(t, err)

	store := newStandardStorageService(db, []storageRuntime{nestedStorage, contentStorage}, func(orgId int64) []storageRuntime { return make([]storageRuntime, 0) }, allowAllAuthService, cfg, nil)
	store.cfg = &GlobalStorageConfig{
		AllowUnsanitizedSvgUpload: true,
	}

	resp, err := store.List(ctx, globalUser, "content/nested", 0)
	require.NoError(t, err)
	require.NotNil(t, resp)

	rowLen, err := resp.Frame.RowLen()
	require.NoError(t, err)
	require.Equal(t, 0, rowLen) // nested storage is empty

	resp, err = store.List(ctx, globalUser, "content", 0)
	require.NoError(t, err)
	require.NotNil(t, resp)

	rowLen, err = resp.Frame.RowLen()
	require.NoError(t, err)
	require.Equal(t, 1, rowLen) // just a single "nested" folder
}