apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Storage: refactor readonly support (#52127)

Browse Source
pull/52176/head
Ryan McKinley 4 years ago committed by GitHub
parent 074bcf8599
commit ab6cf9e94d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
5 changed files with 129 additions and 120 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      pkg/services/store/entity_events.go
  2. 152
      pkg/services/store/http.go
  3. 72
      pkg/services/store/service.go
  4. 16
      pkg/services/store/tree.go
  5. 8
      pkg/services/store/validate.go

1
pkg/services/store/entity_events.go
Unescape View File

@ -26,6 +26,7 @@ const (
EntityTypeDashboard EntityType = "dashboard"
EntityTypeFolder EntityType = "folder"
EntityTypeImage EntityType = "image"
EntityTypeJSON EntityType = "json"
)
// CreateDatabaseEntityId creates entityId for entities stored in the existing SQL tables

152
pkg/services/store/http.go
Unescape View File

@ -15,8 +15,6 @@ import (
"github.com/grafana/grafana/pkg/web"
)
var errFileTooBig = response.Error(400, "Please limit file uploaded under 1MB", errors.New("file is too big"))
// HTTPStorageService passes raw HTTP requests to a well typed storage service
type HTTPStorageService interface {
List(c *models.ReqContext) response.Response
@ -57,87 +55,91 @@ func UploadErrorToStatusCode(err error) int {
}
func (s *httpStorage) Upload(c *models.ReqContext) response.Response {
// 32 MB is the default used by FormFile()
if err := c.Req.ParseMultipartForm(32 << 20); err != nil {
return response.Error(400, "error in parsing form", err)
type rspInfo struct {
Message string `json:"message,omitempty"`
Path string `json:"path,omitempty"`
Count int `json:"count,omitempty"`
Bytes int `json:"bytes,omitempty"`
Error bool `json:"err,omitempty"`
}
rsp := &rspInfo{Message: "uploaded"}
c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE)
if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil {
msg := fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
return response.Error(400, msg, err)
}
files := c.Req.MultipartForm.File["file"]
if len(files) != 1 {
return response.JSON(400, map[string]interface{}{
"message": "please upload files one at a time",
"err": true,
})
}
folder, ok := getMultipartFormValue(c.Req, "folder")
if !ok || folder == "" {
return response.JSON(400, map[string]interface{}{
"message": "please specify the upload folder",
"err": true,
})
}
overwriteExistingFile, _ := getMultipartFormValue(c.Req, "overwriteExistingFile")
fileHeader := files[0]
if fileHeader.Size > MAX_UPLOAD_SIZE {
return errFileTooBig
}
// restrict file size based on file size
// open each file to copy contents
file, err := fileHeader.Open()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
err = file.Close()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
if (len(data)) > MAX_UPLOAD_SIZE {
return errFileTooBig
}
path := folder + "/" + fileHeader.Filename
mimeType := http.DetectContentType(data)
err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{
Contents: data,
MimeType: mimeType,
EntityType: EntityTypeImage,
Path: path,
OverwriteExistingFile: overwriteExistingFile == "true",
})
if err != nil {
return response.Error(UploadErrorToStatusCode(err), err.Error(), err)
}
return response.JSON(200, map[string]interface{}{
"message": "Uploaded successfully",
"path": path,
"file": fileHeader.Filename,
"err": true,
})
rsp.Message = fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE))
rsp.Error = true
return response.JSON(400, rsp)
}
message := getMultipartFormValue(c.Req, "message")
overwriteExistingFile := getMultipartFormValue(c.Req, "overwriteExistingFile") != "false" // must explicitly overwrite
folder := getMultipartFormValue(c.Req, "folder")
for k, fileHeaders := range c.Req.MultipartForm.File {
path := getMultipartFormValue(c.Req, k+".path") // match the path with a file
if len(fileHeaders) > 1 {
path = ""
}
if path == "" && folder == "" {
rsp.Message = "please specify the upload folder or full path"
rsp.Error = true
return response.JSON(400, rsp)
}
for _, fileHeader := range fileHeaders {
// restrict file size based on file size
// open each file to copy contents
file, err := fileHeader.Open()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
err = file.Close()
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
data, err := ioutil.ReadAll(file)
if err != nil {
return response.Error(500, "Internal Server Error", err)
}
if path == "" {
path = folder + "/" + fileHeader.Filename
}
entityType := EntityTypeJSON
mimeType := http.DetectContentType(data)
if strings.HasPrefix(mimeType, "image") {
entityType = EntityTypeImage
}
err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{
Contents: data,
MimeType: mimeType,
EntityType: entityType,
Path: path,
OverwriteExistingFile: overwriteExistingFile,
Properties: map[string]string{
"message": message, // the commit/changelog entry
},
})
if err != nil {
return response.Error(UploadErrorToStatusCode(err), err.Error(), err)
}
rsp.Count++
rsp.Bytes += len(data)
rsp.Path = path
}
}
return response.JSON(200, rsp)
}
func getMultipartFormValue(req *http.Request, key string) (string, bool) {
func getMultipartFormValue(req *http.Request, key string) string {
v, ok := req.MultipartForm.Value[key]
if !ok || len(v) != 1 {
return "", false
return ""
}
return v[0], ok
return v[0]
}
func (s *httpStorage) Read(c *models.ReqContext) response.Response {

72
pkg/services/store/service.go
Unescape View File

@ -4,7 +4,8 @@ import (
"context"
"errors"
"fmt"
"strings"
"os"
"path/filepath"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/infra/log"
@ -25,9 +26,8 @@ var ErrValidationFailed = errors.New("request validation failed")
var ErrFileAlreadyExists = errors.New("file exists")
const RootPublicStatic = "public-static"
const RootResources = "resources"
const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB
const MAX_UPLOAD_SIZE = 1 * 1024 * 1024 // 3MB
type DeleteFolderCmd struct {
Path string `json:"path"`
@ -85,11 +85,26 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles,
setDescription("Access files from the static public files"),
}
// Development dashboards
if setting.Env != setting.Prod {
devenv := filepath.Join(cfg.StaticRootPath, "..", "devenv")
if _, err := os.Stat(devenv); !os.IsNotExist(err) {
// path/to/whatever exists
s := newDiskStorage("devenv", "Development Environment", &StorageLocalDiskConfig{
Path: devenv,
Roots: []string{
"/dev-dashboards/",
},
}).setReadOnly(false).setDescription("Explore files within the developer environment directly")
globalRoots = append(globalRoots, s)
}
}
initializeOrgStorages := func(orgId int64) []storageRuntime {
storages := make([]storageRuntime, 0)
if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) {
storages = append(storages,
newSQLStorage(RootResources,
newSQLStorage("resources",
"Resources",
&StorageSQLConfig{orgId: orgId}, sql).
setBuiltin(true).
@ -155,22 +170,16 @@ type UploadRequest struct {
OverwriteExistingFile bool
}
func storageSupportsMutatingOperations(path string) bool {
// TODO: this is temporary - make it rbac-driven
return strings.HasPrefix(path, RootResources+"/") || path == RootResources
}
func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error {
upload, _ := s.tree.getRoot(getOrgId(user), RootResources)
upload, storagePath := s.tree.getRoot(getOrgId(user), req.Path)
if upload == nil {
return ErrUploadFeatureDisabled
}
if !storageSupportsMutatingOperations(req.Path) {
if upload.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(req.Path, RootResources)
validationResult := s.validateUploadRequest(ctx, user, req, storagePath)
if !validationResult.ok {
grafanaStorageLogger.Warn("file upload validation failed", "filetype", req.MimeType, "path", req.Path, "reason", validationResult.reason)
@ -186,7 +195,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
grafanaStorageLogger.Info("uploading a file", "filetype", req.MimeType, "path", req.Path)
if !req.OverwriteExistingFile {
file, err := upload.Get(ctx, storagePath)
file, err := upload.Store().Get(ctx, storagePath)
if err != nil {
grafanaStorageLogger.Error("failed while checking file existence", "err", err, "path", req.Path)
return ErrUploadInternalError
@ -197,7 +206,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
}
}
if err := upload.Upsert(ctx, upsertCommand); err != nil {
if err := upload.Store().Upsert(ctx, upsertCommand); err != nil {
grafanaStorageLogger.Error("failed while uploading the file", "err", err, "path", req.Path)
return ErrUploadInternalError
}
@ -206,34 +215,32 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed
}
func (s *standardStorageService) DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error {
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
if !storageSupportsMutatingOperations(cmd.Path) {
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(cmd.Path, RootResources)
if storagePath == "" {
storagePath = filestorage.Delimiter
}
return resources.DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
return root.Store().DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true})
}
func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error {
if !storageSupportsMutatingOperations(cmd.Path) {
return ErrUnsupportedStorage
root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
return fmt.Errorf("resources storage is not enabled")
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(cmd.Path, RootResources)
err := resources.CreateFolder(ctx, storagePath)
err := root.Store().CreateFolder(ctx, storagePath)
if err != nil {
return err
}
@ -241,17 +248,16 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.
}
func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error {
if !storageSupportsMutatingOperations(path) {
return ErrUnsupportedStorage
root, storagePath := s.tree.getRoot(getOrgId(user), path)
if root == nil {
return fmt.Errorf("resources storage is not enabled")
}
resources, _ := s.tree.getRoot(getOrgId(user), RootResources)
if resources == nil {
return fmt.Errorf("resources storage is not enabled")
if root.Meta().ReadOnly {
return ErrUnsupportedStorage
}
storagePath := strings.TrimPrefix(path, RootResources)
err := resources.Delete(ctx, storagePath)
err := root.Store().Delete(ctx, storagePath)
if err != nil {
return err
}

16
pkg/services/store/tree.go
Unescape View File

@ -11,7 +11,7 @@ import (
type nestedTree struct {
rootsByOrgId map[int64][]storageRuntime
lookup map[int64]map[string]filestorage.FileStorage
lookup map[int64]map[string]storageRuntime
orgInitMutex sync.Mutex
initializeOrgStorages func(orgId int64) []storageRuntime
@ -21,10 +21,10 @@ var (
_ storageTree = (*nestedTree)(nil)
)
func asNameToFileStorageMap(storages []storageRuntime) map[string]filestorage.FileStorage {
lookup := make(map[string]filestorage.FileStorage)
func asNameToFileStorageMap(storages []storageRuntime) map[string]storageRuntime {
lookup := make(map[string]storageRuntime)
for _, storage := range storages {
lookup[storage.Meta().Config.Prefix] = storage.Store()
lookup[storage.Meta().Config.Prefix] = storage
}
return lookup
}
@ -33,7 +33,7 @@ func (t *nestedTree) init() {
t.orgInitMutex.Lock()
defer t.orgInitMutex.Unlock()
t.lookup = make(map[int64]map[string]filestorage.FileStorage, len(t.rootsByOrgId))
t.lookup = make(map[int64]map[string]storageRuntime, len(t.rootsByOrgId))
for orgId, storages := range t.rootsByOrgId {
t.lookup[orgId] = asNameToFileStorageMap(storages)
@ -50,7 +50,7 @@ func (t *nestedTree) assureOrgIsInitialized(orgId int64) {
}
}
func (t *nestedTree) getRoot(orgId int64, path string) (filestorage.FileStorage, string) {
func (t *nestedTree) getRoot(orgId int64, path string) (storageRuntime, string) {
t.assureOrgIsInitialized(orgId)
if path == "" {
@ -82,7 +82,7 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi
if root == nil {
return nil, nil // not found (or not ready)
}
return root.Get(ctx, path)
return root.Store().Get(ctx, path)
}
func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (*StorageListFrame, error) {
@ -146,7 +146,7 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (
return nil, nil // not found (or not ready)
}
listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{
listResponse, err := root.Store().List(ctx, path, nil, &filestorage.ListOptions{
Recursive: false,
WithFolders: true,
WithFiles: true,

8
pkg/services/store/validate.go
Unescape View File

@ -74,15 +74,15 @@ func (s *standardStorageService) validateUploadRequest(ctx context.Context, user
}
switch req.EntityType {
case EntityTypeJSON:
fallthrough
case EntityTypeFolder:
fallthrough
case EntityTypeDashboard:
// TODO: add proper validation
var something interface{}
if err := json.Unmarshal(req.Contents, &something); err != nil {
return fail(err.Error())
if !json.Valid(req.Contents) {
return fail("invalid json")
}
return success()
case EntityTypeImage:
return s.validateImage(ctx, user, req)

Write Preview
Loading…
Cancel
Save