diff --git a/pkg/services/store/entity_events.go b/pkg/services/store/entity_events.go index 1698b8ffc1f..1c4055908d4 100644 --- a/pkg/services/store/entity_events.go +++ b/pkg/services/store/entity_events.go @@ -26,6 +26,7 @@ const ( EntityTypeDashboard EntityType = "dashboard" EntityTypeFolder EntityType = "folder" EntityTypeImage EntityType = "image" + EntityTypeJSON EntityType = "json" ) // CreateDatabaseEntityId creates entityId for entities stored in the existing SQL tables diff --git a/pkg/services/store/http.go b/pkg/services/store/http.go index 912d75f1cf2..43f2f69c8c3 100644 --- a/pkg/services/store/http.go +++ b/pkg/services/store/http.go @@ -15,8 +15,6 @@ import ( "github.com/grafana/grafana/pkg/web" ) -var errFileTooBig = response.Error(400, "Please limit file uploaded under 1MB", errors.New("file is too big")) - // HTTPStorageService passes raw HTTP requests to a well typed storage service type HTTPStorageService interface { List(c *models.ReqContext) response.Response @@ -57,87 +55,91 @@ func UploadErrorToStatusCode(err error) int { } func (s *httpStorage) Upload(c *models.ReqContext) response.Response { - // 32 MB is the default used by FormFile() - if err := c.Req.ParseMultipartForm(32 << 20); err != nil { - return response.Error(400, "error in parsing form", err) + type rspInfo struct { + Message string `json:"message,omitempty"` + Path string `json:"path,omitempty"` + Count int `json:"count,omitempty"` + Bytes int `json:"bytes,omitempty"` + Error bool `json:"err,omitempty"` } + rsp := &rspInfo{Message: "uploaded"} + c.Req.Body = http.MaxBytesReader(c.Resp, c.Req.Body, MAX_UPLOAD_SIZE) if err := c.Req.ParseMultipartForm(MAX_UPLOAD_SIZE); err != nil { - msg := fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE)) - return response.Error(400, msg, err) - } - - files := c.Req.MultipartForm.File["file"] - if len(files) != 1 { - return response.JSON(400, map[string]interface{}{ - "message": "please upload files one at a time", - "err": true, - }) - } - - folder, ok := getMultipartFormValue(c.Req, "folder") - if !ok || folder == "" { - return response.JSON(400, map[string]interface{}{ - "message": "please specify the upload folder", - "err": true, - }) - } - overwriteExistingFile, _ := getMultipartFormValue(c.Req, "overwriteExistingFile") - - fileHeader := files[0] - if fileHeader.Size > MAX_UPLOAD_SIZE { - return errFileTooBig - } - - // restrict file size based on file size - // open each file to copy contents - file, err := fileHeader.Open() - if err != nil { - return response.Error(500, "Internal Server Error", err) - } - err = file.Close() - if err != nil { - return response.Error(500, "Internal Server Error", err) - } - data, err := ioutil.ReadAll(file) - if err != nil { - return response.Error(500, "Internal Server Error", err) - } - - if (len(data)) > MAX_UPLOAD_SIZE { - return errFileTooBig - } - - path := folder + "/" + fileHeader.Filename - - mimeType := http.DetectContentType(data) - - err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{ - Contents: data, - MimeType: mimeType, - EntityType: EntityTypeImage, - Path: path, - OverwriteExistingFile: overwriteExistingFile == "true", - }) - - if err != nil { - return response.Error(UploadErrorToStatusCode(err), err.Error(), err) - } - - return response.JSON(200, map[string]interface{}{ - "message": "Uploaded successfully", - "path": path, - "file": fileHeader.Filename, - "err": true, - }) + rsp.Message = fmt.Sprintf("Please limit file uploaded under %s", util.ByteCountSI(MAX_UPLOAD_SIZE)) + rsp.Error = true + return response.JSON(400, rsp) + } + message := getMultipartFormValue(c.Req, "message") + overwriteExistingFile := getMultipartFormValue(c.Req, "overwriteExistingFile") != "false" // must explicitly overwrite + folder := getMultipartFormValue(c.Req, "folder") + + for k, fileHeaders := range c.Req.MultipartForm.File { + path := getMultipartFormValue(c.Req, k+".path") // match the path with a file + if len(fileHeaders) > 1 { + path = "" + } + if path == "" && folder == "" { + rsp.Message = "please specify the upload folder or full path" + rsp.Error = true + return response.JSON(400, rsp) + } + + for _, fileHeader := range fileHeaders { + // restrict file size based on file size + // open each file to copy contents + file, err := fileHeader.Open() + if err != nil { + return response.Error(500, "Internal Server Error", err) + } + err = file.Close() + if err != nil { + return response.Error(500, "Internal Server Error", err) + } + data, err := ioutil.ReadAll(file) + if err != nil { + return response.Error(500, "Internal Server Error", err) + } + + if path == "" { + path = folder + "/" + fileHeader.Filename + } + + entityType := EntityTypeJSON + mimeType := http.DetectContentType(data) + if strings.HasPrefix(mimeType, "image") { + entityType = EntityTypeImage + } + + err = s.store.Upload(c.Req.Context(), c.SignedInUser, &UploadRequest{ + Contents: data, + MimeType: mimeType, + EntityType: entityType, + Path: path, + OverwriteExistingFile: overwriteExistingFile, + Properties: map[string]string{ + "message": message, // the commit/changelog entry + }, + }) + + if err != nil { + return response.Error(UploadErrorToStatusCode(err), err.Error(), err) + } + rsp.Count++ + rsp.Bytes += len(data) + rsp.Path = path + } + } + + return response.JSON(200, rsp) } -func getMultipartFormValue(req *http.Request, key string) (string, bool) { +func getMultipartFormValue(req *http.Request, key string) string { v, ok := req.MultipartForm.Value[key] if !ok || len(v) != 1 { - return "", false + return "" } - return v[0], ok + return v[0] } func (s *httpStorage) Read(c *models.ReqContext) response.Response { diff --git a/pkg/services/store/service.go b/pkg/services/store/service.go index e096e8c89af..3713843f11f 100644 --- a/pkg/services/store/service.go +++ b/pkg/services/store/service.go @@ -4,7 +4,8 @@ import ( "context" "errors" "fmt" - "strings" + "os" + "path/filepath" "github.com/grafana/grafana/pkg/infra/filestorage" "github.com/grafana/grafana/pkg/infra/log" @@ -25,9 +26,8 @@ var ErrValidationFailed = errors.New("request validation failed") var ErrFileAlreadyExists = errors.New("file exists") const RootPublicStatic = "public-static" -const RootResources = "resources" -const MAX_UPLOAD_SIZE = 3 * 1024 * 1024 // 3MB +const MAX_UPLOAD_SIZE = 1 * 1024 * 1024 // 3MB type DeleteFolderCmd struct { Path string `json:"path"` @@ -85,11 +85,26 @@ func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, setDescription("Access files from the static public files"), } + // Development dashboards + if setting.Env != setting.Prod { + devenv := filepath.Join(cfg.StaticRootPath, "..", "devenv") + if _, err := os.Stat(devenv); !os.IsNotExist(err) { + // path/to/whatever exists + s := newDiskStorage("devenv", "Development Environment", &StorageLocalDiskConfig{ + Path: devenv, + Roots: []string{ + "/dev-dashboards/", + }, + }).setReadOnly(false).setDescription("Explore files within the developer environment directly") + globalRoots = append(globalRoots, s) + } + } + initializeOrgStorages := func(orgId int64) []storageRuntime { storages := make([]storageRuntime, 0) if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) { storages = append(storages, - newSQLStorage(RootResources, + newSQLStorage("resources", "Resources", &StorageSQLConfig{orgId: orgId}, sql). setBuiltin(true). @@ -155,22 +170,16 @@ type UploadRequest struct { OverwriteExistingFile bool } -func storageSupportsMutatingOperations(path string) bool { - // TODO: this is temporary - make it rbac-driven - return strings.HasPrefix(path, RootResources+"/") || path == RootResources -} - func (s *standardStorageService) Upload(ctx context.Context, user *models.SignedInUser, req *UploadRequest) error { - upload, _ := s.tree.getRoot(getOrgId(user), RootResources) + upload, storagePath := s.tree.getRoot(getOrgId(user), req.Path) if upload == nil { return ErrUploadFeatureDisabled } - if !storageSupportsMutatingOperations(req.Path) { + if upload.Meta().ReadOnly { return ErrUnsupportedStorage } - storagePath := strings.TrimPrefix(req.Path, RootResources) validationResult := s.validateUploadRequest(ctx, user, req, storagePath) if !validationResult.ok { grafanaStorageLogger.Warn("file upload validation failed", "filetype", req.MimeType, "path", req.Path, "reason", validationResult.reason) @@ -186,7 +195,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed grafanaStorageLogger.Info("uploading a file", "filetype", req.MimeType, "path", req.Path) if !req.OverwriteExistingFile { - file, err := upload.Get(ctx, storagePath) + file, err := upload.Store().Get(ctx, storagePath) if err != nil { grafanaStorageLogger.Error("failed while checking file existence", "err", err, "path", req.Path) return ErrUploadInternalError @@ -197,7 +206,7 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed } } - if err := upload.Upsert(ctx, upsertCommand); err != nil { + if err := upload.Store().Upsert(ctx, upsertCommand); err != nil { grafanaStorageLogger.Error("failed while uploading the file", "err", err, "path", req.Path) return ErrUploadInternalError } @@ -206,34 +215,32 @@ func (s *standardStorageService) Upload(ctx context.Context, user *models.Signed } func (s *standardStorageService) DeleteFolder(ctx context.Context, user *models.SignedInUser, cmd *DeleteFolderCmd) error { - resources, _ := s.tree.getRoot(getOrgId(user), RootResources) - if resources == nil { + root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path) + if root == nil { return fmt.Errorf("resources storage is not enabled") } - if !storageSupportsMutatingOperations(cmd.Path) { + if root.Meta().ReadOnly { return ErrUnsupportedStorage } - storagePath := strings.TrimPrefix(cmd.Path, RootResources) if storagePath == "" { storagePath = filestorage.Delimiter } - return resources.DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true}) + return root.Store().DeleteFolder(ctx, storagePath, &filestorage.DeleteFolderOptions{Force: true}) } func (s *standardStorageService) CreateFolder(ctx context.Context, user *models.SignedInUser, cmd *CreateFolderCmd) error { - if !storageSupportsMutatingOperations(cmd.Path) { - return ErrUnsupportedStorage + root, storagePath := s.tree.getRoot(getOrgId(user), cmd.Path) + if root == nil { + return fmt.Errorf("resources storage is not enabled") } - resources, _ := s.tree.getRoot(getOrgId(user), RootResources) - if resources == nil { - return fmt.Errorf("resources storage is not enabled") + if root.Meta().ReadOnly { + return ErrUnsupportedStorage } - storagePath := strings.TrimPrefix(cmd.Path, RootResources) - err := resources.CreateFolder(ctx, storagePath) + err := root.Store().CreateFolder(ctx, storagePath) if err != nil { return err } @@ -241,17 +248,16 @@ func (s *standardStorageService) CreateFolder(ctx context.Context, user *models. } func (s *standardStorageService) Delete(ctx context.Context, user *models.SignedInUser, path string) error { - if !storageSupportsMutatingOperations(path) { - return ErrUnsupportedStorage + root, storagePath := s.tree.getRoot(getOrgId(user), path) + if root == nil { + return fmt.Errorf("resources storage is not enabled") } - resources, _ := s.tree.getRoot(getOrgId(user), RootResources) - if resources == nil { - return fmt.Errorf("resources storage is not enabled") + if root.Meta().ReadOnly { + return ErrUnsupportedStorage } - storagePath := strings.TrimPrefix(path, RootResources) - err := resources.Delete(ctx, storagePath) + err := root.Store().Delete(ctx, storagePath) if err != nil { return err } diff --git a/pkg/services/store/tree.go b/pkg/services/store/tree.go index 24f32377dbd..5748fe5adc3 100644 --- a/pkg/services/store/tree.go +++ b/pkg/services/store/tree.go @@ -11,7 +11,7 @@ import ( type nestedTree struct { rootsByOrgId map[int64][]storageRuntime - lookup map[int64]map[string]filestorage.FileStorage + lookup map[int64]map[string]storageRuntime orgInitMutex sync.Mutex initializeOrgStorages func(orgId int64) []storageRuntime @@ -21,10 +21,10 @@ var ( _ storageTree = (*nestedTree)(nil) ) -func asNameToFileStorageMap(storages []storageRuntime) map[string]filestorage.FileStorage { - lookup := make(map[string]filestorage.FileStorage) +func asNameToFileStorageMap(storages []storageRuntime) map[string]storageRuntime { + lookup := make(map[string]storageRuntime) for _, storage := range storages { - lookup[storage.Meta().Config.Prefix] = storage.Store() + lookup[storage.Meta().Config.Prefix] = storage } return lookup } @@ -33,7 +33,7 @@ func (t *nestedTree) init() { t.orgInitMutex.Lock() defer t.orgInitMutex.Unlock() - t.lookup = make(map[int64]map[string]filestorage.FileStorage, len(t.rootsByOrgId)) + t.lookup = make(map[int64]map[string]storageRuntime, len(t.rootsByOrgId)) for orgId, storages := range t.rootsByOrgId { t.lookup[orgId] = asNameToFileStorageMap(storages) @@ -50,7 +50,7 @@ func (t *nestedTree) assureOrgIsInitialized(orgId int64) { } } -func (t *nestedTree) getRoot(orgId int64, path string) (filestorage.FileStorage, string) { +func (t *nestedTree) getRoot(orgId int64, path string) (storageRuntime, string) { t.assureOrgIsInitialized(orgId) if path == "" { @@ -82,7 +82,7 @@ func (t *nestedTree) GetFile(ctx context.Context, orgId int64, path string) (*fi if root == nil { return nil, nil // not found (or not ready) } - return root.Get(ctx, path) + return root.Store().Get(ctx, path) } func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) (*StorageListFrame, error) { @@ -146,7 +146,7 @@ func (t *nestedTree) ListFolder(ctx context.Context, orgId int64, path string) ( return nil, nil // not found (or not ready) } - listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{ + listResponse, err := root.Store().List(ctx, path, nil, &filestorage.ListOptions{ Recursive: false, WithFolders: true, WithFiles: true, diff --git a/pkg/services/store/validate.go b/pkg/services/store/validate.go index 7ad7cf391bf..cc0c2f011dc 100644 --- a/pkg/services/store/validate.go +++ b/pkg/services/store/validate.go @@ -74,15 +74,15 @@ func (s *standardStorageService) validateUploadRequest(ctx context.Context, user } switch req.EntityType { + case EntityTypeJSON: + fallthrough case EntityTypeFolder: fallthrough case EntityTypeDashboard: // TODO: add proper validation - var something interface{} - if err := json.Unmarshal(req.Contents, &something); err != nil { - return fail(err.Error()) + if !json.Valid(req.Contents) { + return fail("invalid json") } - return success() case EntityTypeImage: return s.validateImage(ctx, user, req)