mirror of https://github.com/grafana/grafana
Storage: add basic storage service (#46604)
Ryan McKinley
4 years ago
committed by
GitHub
parent
4df7bf5ab2
commit
1cfb9a4a19
@ -0,0 +1,53 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
type RootStorageConfig struct { |
||||||
|
Type string `json:"type"` |
||||||
|
Prefix string `json:"prefix"` |
||||||
|
Name string `json:"name"` |
||||||
|
|
||||||
|
// Depending on type, these will be configured
|
||||||
|
Disk *StorageLocalDiskConfig `json:"disk,omitempty"` |
||||||
|
Git *StorageGitConfig `json:"git,omitempty"` |
||||||
|
SQL *StorageSQLConfig `json:"sql,omitempty"` |
||||||
|
S3 *StorageS3Config `json:"s3,omitempty"` |
||||||
|
GCS *StorageGCSConfig `json:"gcs,omitempty"` |
||||||
|
} |
||||||
|
|
||||||
|
type StorageLocalDiskConfig struct { |
||||||
|
Path string `json:"path"` |
||||||
|
Roots []string `json:"roots,omitempty"` // null is everything
|
||||||
|
} |
||||||
|
|
||||||
|
type StorageGitConfig struct { |
||||||
|
Remote string `json:"remote"` |
||||||
|
Branch string `json:"branch"` |
||||||
|
Root string `json:"root"` // subfolder within the remote
|
||||||
|
|
||||||
|
// Pull interval?
|
||||||
|
// Requires pull request?
|
||||||
|
RequirePullRequest bool `json:"requirePullRequest"` |
||||||
|
|
||||||
|
// SECURE JSON :grimicing:
|
||||||
|
AccessToken string `json:"accessToken,omitempty"` // Simplest auth method for github
|
||||||
|
} |
||||||
|
|
||||||
|
type StorageSQLConfig struct { |
||||||
|
// no custom settings
|
||||||
|
} |
||||||
|
|
||||||
|
type StorageS3Config struct { |
||||||
|
Bucket string `json:"bucket"` |
||||||
|
Folder string `json:"folder"` |
||||||
|
|
||||||
|
// SECURE!!!
|
||||||
|
AccessKey string `json:"accessKey"` |
||||||
|
SecretKey string `json:"secretKey"` |
||||||
|
Region string `json:"region"` |
||||||
|
} |
||||||
|
|
||||||
|
type StorageGCSConfig struct { |
||||||
|
Bucket string `json:"bucket"` |
||||||
|
Folder string `json:"folder"` |
||||||
|
|
||||||
|
CredentialsFile string `json:"credentialsFile"` |
||||||
|
} |
||||||
@ -0,0 +1,71 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"github.com/grafana/grafana/pkg/api/response" |
||||||
|
"github.com/grafana/grafana/pkg/models" |
||||||
|
"github.com/grafana/grafana/pkg/web" |
||||||
|
) |
||||||
|
|
||||||
|
// HTTPStorageService passes raw HTTP requests to a well typed storage service
|
||||||
|
type HTTPStorageService interface { |
||||||
|
List(c *models.ReqContext) response.Response |
||||||
|
Read(c *models.ReqContext) response.Response |
||||||
|
Delete(c *models.ReqContext) response.Response |
||||||
|
Upload(c *models.ReqContext) response.Response |
||||||
|
} |
||||||
|
|
||||||
|
type httpStorage struct { |
||||||
|
store StorageService |
||||||
|
} |
||||||
|
|
||||||
|
func ProvideHTTPService(store StorageService) HTTPStorageService { |
||||||
|
return &httpStorage{ |
||||||
|
store: store, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func (s *httpStorage) Upload(c *models.ReqContext) response.Response { |
||||||
|
action := "Upload" |
||||||
|
scope, path := getPathAndScope(c) |
||||||
|
|
||||||
|
return response.JSON(200, map[string]string{ |
||||||
|
"action": action, |
||||||
|
"scope": scope, |
||||||
|
"path": path, |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func (s *httpStorage) Read(c *models.ReqContext) response.Response { |
||||||
|
action := "Read" |
||||||
|
scope, path := getPathAndScope(c) |
||||||
|
|
||||||
|
return response.JSON(200, map[string]string{ |
||||||
|
"action": action, |
||||||
|
"scope": scope, |
||||||
|
"path": path, |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func (s *httpStorage) Delete(c *models.ReqContext) response.Response { |
||||||
|
action := "Delete" |
||||||
|
scope, path := getPathAndScope(c) |
||||||
|
|
||||||
|
return response.JSON(200, map[string]string{ |
||||||
|
"action": action, |
||||||
|
"scope": scope, |
||||||
|
"path": path, |
||||||
|
}) |
||||||
|
} |
||||||
|
|
||||||
|
func (s *httpStorage) List(c *models.ReqContext) response.Response { |
||||||
|
params := web.Params(c.Req) |
||||||
|
path := params["*"] |
||||||
|
frame, err := s.store.List(c.Req.Context(), c.SignedInUser, path) |
||||||
|
if err != nil { |
||||||
|
return response.Error(400, "error reading path", err) |
||||||
|
} |
||||||
|
if frame == nil { |
||||||
|
return response.Error(404, "not found", nil) |
||||||
|
} |
||||||
|
return response.JSONStreaming(200, frame) |
||||||
|
} |
||||||
@ -0,0 +1,88 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"os" |
||||||
|
"path/filepath" |
||||||
|
|
||||||
|
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||||
|
"github.com/grafana/grafana/pkg/infra/filestorage" |
||||||
|
"github.com/grafana/grafana/pkg/infra/log" |
||||||
|
"github.com/grafana/grafana/pkg/models" |
||||||
|
"github.com/grafana/grafana/pkg/registry" |
||||||
|
"github.com/grafana/grafana/pkg/services/featuremgmt" |
||||||
|
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||||
|
"github.com/grafana/grafana/pkg/setting" |
||||||
|
) |
||||||
|
|
||||||
|
var grafanaStorageLogger = log.New("grafanaStorageLogger") |
||||||
|
|
||||||
|
const RootPublicStatic = "public-static" |
||||||
|
|
||||||
|
type StorageService interface { |
||||||
|
registry.BackgroundService |
||||||
|
|
||||||
|
// List folder contents
|
||||||
|
List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) |
||||||
|
|
||||||
|
// Read raw file contents out of the store
|
||||||
|
Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) |
||||||
|
} |
||||||
|
|
||||||
|
type standardStorageService struct { |
||||||
|
sql *sqlstore.SQLStore |
||||||
|
tree *nestedTree |
||||||
|
} |
||||||
|
|
||||||
|
func ProvideService(sql *sqlstore.SQLStore, features featuremgmt.FeatureToggles, cfg *setting.Cfg) StorageService { |
||||||
|
roots := []storageRuntime{ |
||||||
|
newDiskStorage(RootPublicStatic, "Public static files", &StorageLocalDiskConfig{ |
||||||
|
Path: cfg.StaticRootPath, |
||||||
|
Roots: []string{ |
||||||
|
"/testdata/", |
||||||
|
// "/img/icons/",
|
||||||
|
// "/img/bg/",
|
||||||
|
"/img/", |
||||||
|
"/gazetteer/", |
||||||
|
"/maps/", |
||||||
|
}, |
||||||
|
}).setReadOnly(true).setBuiltin(true), |
||||||
|
} |
||||||
|
|
||||||
|
storage := filepath.Join(cfg.DataPath, "storage") |
||||||
|
_ = os.MkdirAll(storage, 0700) |
||||||
|
|
||||||
|
if features.IsEnabled(featuremgmt.FlagStorageLocalUpload) { |
||||||
|
roots = append(roots, newDiskStorage("upload", "Local file upload", &StorageLocalDiskConfig{ |
||||||
|
Path: filepath.Join(storage, "upload"), |
||||||
|
})) |
||||||
|
} |
||||||
|
s := newStandardStorageService(roots) |
||||||
|
s.sql = sql |
||||||
|
return s |
||||||
|
} |
||||||
|
|
||||||
|
func newStandardStorageService(roots []storageRuntime) *standardStorageService { |
||||||
|
res := &nestedTree{ |
||||||
|
roots: roots, |
||||||
|
} |
||||||
|
res.init() |
||||||
|
return &standardStorageService{ |
||||||
|
tree: res, |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func (s *standardStorageService) Run(ctx context.Context) error { |
||||||
|
grafanaStorageLogger.Info("storage starting") |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func (s *standardStorageService) List(ctx context.Context, user *models.SignedInUser, path string) (*data.Frame, error) { |
||||||
|
// apply access control here
|
||||||
|
return s.tree.ListFolder(ctx, path) |
||||||
|
} |
||||||
|
|
||||||
|
func (s *standardStorageService) Read(ctx context.Context, user *models.SignedInUser, path string) (*filestorage.File, error) { |
||||||
|
// TODO: permission check!
|
||||||
|
return s.tree.GetFile(ctx, path) |
||||||
|
} |
||||||
@ -0,0 +1,47 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"bytes" |
||||||
|
"context" |
||||||
|
"path" |
||||||
|
"path/filepath" |
||||||
|
"testing" |
||||||
|
|
||||||
|
"github.com/grafana/grafana-plugin-sdk-go/experimental" |
||||||
|
"github.com/grafana/grafana/pkg/tsdb/testdatasource" |
||||||
|
"github.com/stretchr/testify/require" |
||||||
|
) |
||||||
|
|
||||||
|
func TestListFiles(t *testing.T) { |
||||||
|
publicRoot, err := filepath.Abs("../../../public") |
||||||
|
require.NoError(t, err) |
||||||
|
roots := []storageRuntime{ |
||||||
|
newDiskStorage("public", "Public static files", &StorageLocalDiskConfig{ |
||||||
|
Path: publicRoot, |
||||||
|
Roots: []string{ |
||||||
|
"/testdata/", |
||||||
|
"/img/icons/", |
||||||
|
"/img/bg/", |
||||||
|
"/gazetteer/", |
||||||
|
"/maps/", |
||||||
|
"/upload/", |
||||||
|
}, |
||||||
|
}).setReadOnly(true).setBuiltin(true), |
||||||
|
} |
||||||
|
|
||||||
|
store := newStandardStorageService(roots) |
||||||
|
frame, err := store.List(context.Background(), nil, "public/testdata") |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata.golden.txt"), frame, true) |
||||||
|
require.NoError(t, err) |
||||||
|
|
||||||
|
file, err := store.Read(context.Background(), nil, "public/testdata/js_libraries.csv") |
||||||
|
require.NoError(t, err) |
||||||
|
require.NotNil(t, file) |
||||||
|
|
||||||
|
frame, err = testdatasource.LoadCsvContent(bytes.NewReader(file.Contents), file.Name) |
||||||
|
require.NoError(t, err) |
||||||
|
err = experimental.CheckGoldenFrame(path.Join("testdata", "public_testdata_js_libraries.golden.txt"), frame, true) |
||||||
|
require.NoError(t, err) |
||||||
|
} |
||||||
@ -0,0 +1,91 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"fmt" |
||||||
|
"strings" |
||||||
|
|
||||||
|
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||||
|
"github.com/grafana/grafana/pkg/infra/filestorage" |
||||||
|
"gocloud.dev/blob" |
||||||
|
) |
||||||
|
|
||||||
|
const rootStorageTypeDisk = "disk" |
||||||
|
|
||||||
|
type rootStorageDisk struct { |
||||||
|
baseStorageRuntime |
||||||
|
|
||||||
|
settings *StorageLocalDiskConfig |
||||||
|
} |
||||||
|
|
||||||
|
func newDiskStorage(prefix string, name string, cfg *StorageLocalDiskConfig) *rootStorageDisk { |
||||||
|
if cfg == nil { |
||||||
|
cfg = &StorageLocalDiskConfig{} |
||||||
|
} |
||||||
|
|
||||||
|
meta := RootStorageMeta{ |
||||||
|
Config: RootStorageConfig{ |
||||||
|
Type: rootStorageTypeDisk, |
||||||
|
Prefix: prefix, |
||||||
|
Name: name, |
||||||
|
Disk: cfg, |
||||||
|
}, |
||||||
|
} |
||||||
|
if prefix == "" { |
||||||
|
meta.Notice = append(meta.Notice, data.Notice{ |
||||||
|
Severity: data.NoticeSeverityError, |
||||||
|
Text: "Missing prefix", |
||||||
|
}) |
||||||
|
} |
||||||
|
if cfg.Path == "" { |
||||||
|
meta.Notice = append(meta.Notice, data.Notice{ |
||||||
|
Severity: data.NoticeSeverityError, |
||||||
|
Text: "Missing path configuration", |
||||||
|
}) |
||||||
|
} |
||||||
|
s := &rootStorageDisk{} |
||||||
|
|
||||||
|
if meta.Notice == nil { |
||||||
|
path := fmt.Sprintf("file://%s", cfg.Path) |
||||||
|
bucket, err := blob.OpenBucket(context.Background(), path) |
||||||
|
if err != nil { |
||||||
|
grafanaStorageLogger.Warn("error loading storage", "prefix", prefix, "err", err) |
||||||
|
meta.Notice = append(meta.Notice, data.Notice{ |
||||||
|
Severity: data.NoticeSeverityError, |
||||||
|
Text: "Failed to initialize storage", |
||||||
|
}) |
||||||
|
} else { |
||||||
|
s.store = filestorage.NewCdkBlobStorage(grafanaStorageLogger, |
||||||
|
bucket, "", |
||||||
|
filestorage.NewPathFilters(cfg.Roots, nil, nil, nil)) |
||||||
|
|
||||||
|
meta.Ready = true // exists!
|
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
s.meta = meta |
||||||
|
s.settings = cfg |
||||||
|
return s |
||||||
|
} |
||||||
|
|
||||||
|
func (s *rootStorageDisk) Sync() error { |
||||||
|
return nil // already in sync
|
||||||
|
} |
||||||
|
|
||||||
|
// with local disk user metadata and messages are lost
|
||||||
|
func (s *rootStorageDisk) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { |
||||||
|
byteAray := []byte(cmd.Body) |
||||||
|
|
||||||
|
path := cmd.Path |
||||||
|
if !strings.HasPrefix(path, filestorage.Delimiter) { |
||||||
|
path = filestorage.Delimiter + path |
||||||
|
} |
||||||
|
err := s.store.Upsert(ctx, &filestorage.UpsertFileCommand{ |
||||||
|
Path: path, |
||||||
|
Contents: &byteAray, |
||||||
|
}) |
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
return &WriteValueResponse{Code: 200}, nil |
||||||
|
} |
||||||
@ -0,0 +1,27 @@ |
|||||||
|
🌟 This was machine generated. Do not edit. 🌟 |
||||||
|
|
||||||
|
Frame[0] { |
||||||
|
"type": "directory-listing", |
||||||
|
"custom": { |
||||||
|
"HasMore": false |
||||||
|
} |
||||||
|
} |
||||||
|
Name: |
||||||
|
Dimensions: 3 Fields by 7 Rows |
||||||
|
+--------------------------+-------------------------+---------------+ |
||||||
|
| Name: name | Name: mediaType | Name: size | |
||||||
|
| Labels: | Labels: | Labels: | |
||||||
|
| Type: []string | Type: []string | Type: []int64 | |
||||||
|
+--------------------------+-------------------------+---------------+ |
||||||
|
| browser_marketshare.csv | text/csv; charset=utf-8 | 355 | |
||||||
|
| flight_info_by_state.csv | text/csv; charset=utf-8 | 681 | |
||||||
|
| gdp_per_capita.csv | text/csv; charset=utf-8 | 4116 | |
||||||
|
| js_libraries.csv | text/csv; charset=utf-8 | 179 | |
||||||
|
| ohlc_dogecoin.csv | text/csv; charset=utf-8 | 191804 | |
||||||
|
| population_by_state.csv | text/csv; charset=utf-8 | 138 | |
||||||
|
| weight_height.csv | text/csv; charset=utf-8 | 418121 | |
||||||
|
+--------------------------+-------------------------+---------------+ |
||||||
|
|
||||||
|
|
||||||
|
====== TEST DATA RESPONSE (arrow base64) ====== |
||||||
|
FRAME=QVJST1cxAAD/////WAIAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEEAAoADAAAAAgABAAKAAAACAAAAKgAAAADAAAATAAAACgAAAAEAAAAMP7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAABQ/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAAHD+//8IAAAAQAAAADcAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsImN1c3RvbSI6eyJIYXNNb3JlIjpmYWxzZX19AAQAAABtZXRhAAAAAAMAAAAYAQAApAAAAAQAAAAG////FAAAAHAAAAB4AAAAAAAAAnwAAAACAAAALAAAAAQAAAD4/v//CAAAABAAAAAEAAAAc2l6ZQAAAAAEAAAAbmFtZQAAAAAc////CAAAABwAAAAQAAAAeyJ1bml0IjoiYnl0ZXMifQAAAAAGAAAAY29uZmlnAAAAAAAACAAMAAgABwAIAAAAAAAAAUAAAAAEAAAAc2l6ZQAAAACi////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAJD///8IAAAAFAAAAAkAAABtZWRpYVR5cGUAAAAEAAAAbmFtZQAAAAAAAAAAjP///wkAAABtZWRpYVR5cGUAEgAYABQAAAATAAwAAAAIAAQAEgAAABQAAABEAAAASAAAAAAAAAVEAAAAAQAAAAwAAAAIAAwACAAEAAgAAAAIAAAAEAAAAAQAAABuYW1lAAAAAAQAAABuYW1lAAAAAAAAAAAEAAQABAAAAAQAAABuYW1lAAAAAP////8IAQAAFAAAAAAAAAAMABYAFAATAAwABAAMAAAAsAEAAAAAAAAUAAAAAAAAAwQACgAYAAwACAAEAAoAAAAUAAAAmAAAAAcAAAAAAAAAAAAAAAgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAgAAAAAAAAACAAAAAAAAAAigAAAAAAAACwAAAAAAAAAAAAAAAAAAAAsAAAAAAAAAAgAAAAAAAAANAAAAAAAAAAoQAAAAAAAAB4AQAAAAAAAAAAAAAAAAAAeAEAAAAAAAA4AAAAAAAAAAAAAAADAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAHAAAAAAAAAAAAAAAAAAAAAAAAABcAAAAvAAAAQQAAAFEAAABiAAAAeQAAAIoAAABicm93c2VyX21hcmtldHNoYXJlLmNzdmZsaWdodF9pbmZvX2J5X3N0YXRlLmNzdmdkcF9wZXJfY2FwaXRhLmNzdmpzX2xpYnJhcmllcy5jc3ZvaGxjX2RvZ2Vjb2luLmNzdnBvcHVsYXRpb25fYnlfc3RhdGUuY3N2d2VpZ2h0X2hlaWdodC5jc3YAAAAAAAAAAAAAFwAAAC4AAABFAAAAXAAAAHMAAACKAAAAoQAAAHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04dGV4dC9jc3Y7IGNoYXJzZXQ9dXRmLTh0ZXh0L2NzdjsgY2hhcnNldD11dGYtOHRleHQvY3N2OyBjaGFyc2V0PXV0Zi04AAAAAAAAAGMBAAAAAAAAqQIAAAAAAAAUEAAAAAAAALMAAAAAAAAAPO0CAAAAAACKAAAAAAAAAElhBgAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAQAAQAAAGgCAAAAAAAAEAEAAAAAAACwAQAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAAqAAAAAMAAABMAAAAKAAAAAQAAAAw/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAFD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAcP7//wgAAABAAAAANwAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwiY3VzdG9tIjp7Ikhhc01vcmUiOmZhbHNlfX0ABAAAAG1ldGEAAAAAAwAAABgBAACkAAAABAAAAAb///8UAAAAcAAAAHgAAAAAAAACfAAAAAIAAAAsAAAABAAAAPj+//8IAAAAEAAAAAQAAABzaXplAAAAAAQAAABuYW1lAAAAABz///8IAAAAHAAAABAAAAB7InVuaXQiOiJieXRlcyJ9AAAAAAYAAABjb25maWcAAAAAAAAIAAwACAAHAAgAAAAAAAABQAAAAAQAAABzaXplAAAAAKL///8UAAAAQAAAAEAAAAAAAAAFPAAAAAEAAAAEAAAAkP///wgAAAAUAAAACQAAAG1lZGlhVHlwZQAAAAQAAABuYW1lAAAAAAAAAACM////CQAAAG1lZGlhVHlwZQASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAAiAIAAEFSUk9XMQ== |
||||||
@ -0,0 +1,106 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
|
||||||
|
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||||
|
"github.com/grafana/grafana/pkg/infra/filestorage" |
||||||
|
) |
||||||
|
|
||||||
|
type nestedTree struct { |
||||||
|
roots []storageRuntime |
||||||
|
lookup map[string]filestorage.FileStorage |
||||||
|
} |
||||||
|
|
||||||
|
var ( |
||||||
|
_ storageTree = (*nestedTree)(nil) |
||||||
|
) |
||||||
|
|
||||||
|
func (t *nestedTree) init() { |
||||||
|
t.lookup = make(map[string]filestorage.FileStorage, len(t.roots)) |
||||||
|
for _, root := range t.roots { |
||||||
|
t.lookup[root.Meta().Config.Prefix] = root.Store() |
||||||
|
} |
||||||
|
} |
||||||
|
|
||||||
|
func (t *nestedTree) getRoot(path string) (filestorage.FileStorage, string) { |
||||||
|
if path == "" { |
||||||
|
return nil, "" |
||||||
|
} |
||||||
|
|
||||||
|
rootKey, path := splitFirstSegment(path) |
||||||
|
root, ok := t.lookup[rootKey] |
||||||
|
if !ok || root == nil { |
||||||
|
return nil, path // not found or not ready
|
||||||
|
} |
||||||
|
return root, filestorage.Delimiter + path |
||||||
|
} |
||||||
|
|
||||||
|
func (t *nestedTree) GetFile(ctx context.Context, path string) (*filestorage.File, error) { |
||||||
|
if path == "" { |
||||||
|
return nil, nil // not found
|
||||||
|
} |
||||||
|
|
||||||
|
root, path := t.getRoot(path) |
||||||
|
if root == nil { |
||||||
|
return nil, nil // not found (or not ready)
|
||||||
|
} |
||||||
|
return root.Get(ctx, path) |
||||||
|
} |
||||||
|
|
||||||
|
func (t *nestedTree) ListFolder(ctx context.Context, path string) (*data.Frame, error) { |
||||||
|
if path == "" || path == "/" { |
||||||
|
count := len(t.roots) |
||||||
|
title := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||||
|
names := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||||
|
mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||||
|
title.Name = "title" |
||||||
|
names.Name = "name" |
||||||
|
mtype.Name = "mediaType" |
||||||
|
for i, f := range t.roots { |
||||||
|
names.Set(i, f.Meta().Config.Prefix) |
||||||
|
title.Set(i, f.Meta().Config.Name) |
||||||
|
mtype.Set(i, "directory") |
||||||
|
} |
||||||
|
frame := data.NewFrame("", names, title, mtype) |
||||||
|
frame.SetMeta(&data.FrameMeta{ |
||||||
|
Type: data.FrameTypeDirectoryListing, |
||||||
|
}) |
||||||
|
return frame, nil |
||||||
|
} |
||||||
|
|
||||||
|
root, path := t.getRoot(path) |
||||||
|
if root == nil { |
||||||
|
return nil, nil // not found (or not ready)
|
||||||
|
} |
||||||
|
|
||||||
|
listResponse, err := root.List(ctx, path, nil, &filestorage.ListOptions{Recursive: false, WithFolders: true, WithFiles: true}) |
||||||
|
|
||||||
|
if err != nil { |
||||||
|
return nil, err |
||||||
|
} |
||||||
|
|
||||||
|
count := len(listResponse.Files) |
||||||
|
names := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||||
|
mtype := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||||
|
fsize := data.NewFieldFromFieldType(data.FieldTypeInt64, count) |
||||||
|
names.Name = "name" |
||||||
|
mtype.Name = "mediaType" |
||||||
|
fsize.Name = "size" |
||||||
|
fsize.Config = &data.FieldConfig{ |
||||||
|
Unit: "bytes", |
||||||
|
} |
||||||
|
for i, f := range listResponse.Files { |
||||||
|
names.Set(i, f.Name) |
||||||
|
mtype.Set(i, f.MimeType) |
||||||
|
fsize.Set(i, f.Size) |
||||||
|
} |
||||||
|
frame := data.NewFrame("", names, mtype, fsize) |
||||||
|
frame.SetMeta(&data.FrameMeta{ |
||||||
|
Type: data.FrameTypeDirectoryListing, |
||||||
|
Custom: map[string]interface{}{ |
||||||
|
"HasMore": listResponse.HasMore, |
||||||
|
}, |
||||||
|
}) |
||||||
|
return frame, nil |
||||||
|
} |
||||||
@ -0,0 +1,92 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"context" |
||||||
|
"encoding/json" |
||||||
|
|
||||||
|
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||||
|
"github.com/grafana/grafana/pkg/infra/filestorage" |
||||||
|
"github.com/grafana/grafana/pkg/models" |
||||||
|
) |
||||||
|
|
||||||
|
type WriteValueRequest struct { |
||||||
|
Path string |
||||||
|
User *models.SignedInUser |
||||||
|
Body json.RawMessage `json:"body,omitempty"` |
||||||
|
Message string `json:"message,omitempty"` |
||||||
|
Title string `json:"title,omitempty"` // For PRs
|
||||||
|
Action string `json:"action,omitempty"` // pr | save
|
||||||
|
} |
||||||
|
|
||||||
|
type WriteValueResponse struct { |
||||||
|
Code int `json:"code,omitempty"` |
||||||
|
Message string `json:"message,omitempty"` |
||||||
|
URL string `json:"url,omitempty"` |
||||||
|
Hash string `json:"hash,omitempty"` |
||||||
|
Branch string `json:"branch,omitempty"` |
||||||
|
Pending bool `json:"pending,omitempty"` |
||||||
|
Size int64 `json:"size,omitempty"` |
||||||
|
} |
||||||
|
|
||||||
|
type storageTree interface { |
||||||
|
GetFile(ctx context.Context, path string) (*filestorage.File, error) |
||||||
|
ListFolder(ctx context.Context, path string) (*data.Frame, error) |
||||||
|
} |
||||||
|
|
||||||
|
//-------------------------------------------
|
||||||
|
// INTERNAL
|
||||||
|
//-------------------------------------------
|
||||||
|
|
||||||
|
type storageRuntime interface { |
||||||
|
Meta() RootStorageMeta |
||||||
|
|
||||||
|
Store() filestorage.FileStorage |
||||||
|
|
||||||
|
Sync() error |
||||||
|
|
||||||
|
// Different storage knows how to handle comments and tracking
|
||||||
|
Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) |
||||||
|
} |
||||||
|
|
||||||
|
type baseStorageRuntime struct { |
||||||
|
meta RootStorageMeta |
||||||
|
store filestorage.FileStorage |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) Meta() RootStorageMeta { |
||||||
|
return t.meta |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) Store() filestorage.FileStorage { |
||||||
|
return t.store |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) Sync() error { |
||||||
|
return nil |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) Write(ctx context.Context, cmd *WriteValueRequest) (*WriteValueResponse, error) { |
||||||
|
return &WriteValueResponse{ |
||||||
|
Code: 500, |
||||||
|
Message: "unsupportted operation (base)", |
||||||
|
}, nil |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) setReadOnly(val bool) *baseStorageRuntime { |
||||||
|
t.meta.ReadOnly = val |
||||||
|
return t |
||||||
|
} |
||||||
|
|
||||||
|
func (t *baseStorageRuntime) setBuiltin(val bool) *baseStorageRuntime { |
||||||
|
t.meta.Builtin = val |
||||||
|
return t |
||||||
|
} |
||||||
|
|
||||||
|
type RootStorageMeta struct { |
||||||
|
ReadOnly bool `json:"editable,omitempty"` |
||||||
|
Builtin bool `json:"builtin,omitempty"` |
||||||
|
Ready bool `json:"ready"` // can connect
|
||||||
|
Notice []data.Notice `json:"notice,omitempty"` |
||||||
|
|
||||||
|
Config RootStorageConfig `json:"config"` |
||||||
|
} |
||||||
@ -0,0 +1,31 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"path/filepath" |
||||||
|
"strings" |
||||||
|
|
||||||
|
"github.com/grafana/grafana/pkg/models" |
||||||
|
"github.com/grafana/grafana/pkg/web" |
||||||
|
) |
||||||
|
|
||||||
|
func splitFirstSegment(path string) (string, string) { |
||||||
|
idx := strings.Index(path, "/") |
||||||
|
if idx == 0 { |
||||||
|
path = path[1:] |
||||||
|
idx = strings.Index(path, "/") |
||||||
|
} |
||||||
|
|
||||||
|
if idx > 0 { |
||||||
|
return path[:idx], path[idx+1:] |
||||||
|
} |
||||||
|
return path, "" |
||||||
|
} |
||||||
|
|
||||||
|
func getPathAndScope(c *models.ReqContext) (string, string) { |
||||||
|
params := web.Params(c.Req) |
||||||
|
path := params["*"] |
||||||
|
if path == "" { |
||||||
|
return "", "" |
||||||
|
} |
||||||
|
return splitFirstSegment(filepath.Clean(path)) |
||||||
|
} |
||||||
@ -0,0 +1,25 @@ |
|||||||
|
package store |
||||||
|
|
||||||
|
import ( |
||||||
|
"testing" |
||||||
|
|
||||||
|
"github.com/stretchr/testify/require" |
||||||
|
) |
||||||
|
|
||||||
|
func TestUtils(t *testing.T) { |
||||||
|
a, b := splitFirstSegment("") |
||||||
|
require.Equal(t, "", a) |
||||||
|
require.Equal(t, "", b) |
||||||
|
|
||||||
|
a, b = splitFirstSegment("hello") |
||||||
|
require.Equal(t, "hello", a) |
||||||
|
require.Equal(t, "", b) |
||||||
|
|
||||||
|
a, b = splitFirstSegment("hello/world") |
||||||
|
require.Equal(t, "hello", a) |
||||||
|
require.Equal(t, "world", b) |
||||||
|
|
||||||
|
a, b = splitFirstSegment("/hello/world") // strip leading slash
|
||||||
|
require.Equal(t, "hello", a) |
||||||
|
require.Equal(t, "world", b) |
||||||
|
} |
||||||
@ -1,43 +0,0 @@ |
|||||||
package grafanads |
|
||||||
|
|
||||||
import ( |
|
||||||
"encoding/json" |
|
||||||
"path" |
|
||||||
"testing" |
|
||||||
|
|
||||||
"github.com/grafana/grafana/pkg/services/searchV2" |
|
||||||
"github.com/grafana/grafana/pkg/setting" |
|
||||||
|
|
||||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
|
||||||
"github.com/grafana/grafana-plugin-sdk-go/experimental" |
|
||||||
"github.com/stretchr/testify/require" |
|
||||||
) |
|
||||||
|
|
||||||
func asJSON(v interface{}) json.RawMessage { |
|
||||||
b, _ := json.Marshal(v) |
|
||||||
return b |
|
||||||
} |
|
||||||
|
|
||||||
func TestReadFolderListing(t *testing.T) { |
|
||||||
ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService()) |
|
||||||
dr := ds.doListQuery(backend.DataQuery{ |
|
||||||
QueryType: "x", |
|
||||||
JSON: asJSON(listQueryModel{ |
|
||||||
Path: "testdata", |
|
||||||
}), |
|
||||||
}) |
|
||||||
err := experimental.CheckGoldenDataResponse(path.Join("testdata", "list.golden.txt"), &dr, true) |
|
||||||
require.NoError(t, err) |
|
||||||
} |
|
||||||
|
|
||||||
func TestReadCSVFile(t *testing.T) { |
|
||||||
ds := newService(&setting.Cfg{StaticRootPath: "../../../public"}, searchV2.NewStubSearchService()) |
|
||||||
dr := ds.doReadQuery(backend.DataQuery{ |
|
||||||
QueryType: "x", |
|
||||||
JSON: asJSON(readQueryModel{ |
|
||||||
Path: "testdata/js_libraries.csv", |
|
||||||
}), |
|
||||||
}) |
|
||||||
err := experimental.CheckGoldenDataResponse(path.Join("testdata", "jslib.golden.txt"), &dr, true) |
|
||||||
require.NoError(t, err) |
|
||||||
} |
|
||||||
@ -1,25 +0,0 @@ |
|||||||
🌟 This was machine generated. Do not edit. 🌟 |
|
||||||
|
|
||||||
Frame[0] { |
|
||||||
"type": "directory-listing", |
|
||||||
"pathSeparator": "/" |
|
||||||
} |
|
||||||
Name: |
|
||||||
Dimensions: 2 Fields by 7 Rows |
|
||||||
+--------------------------+------------------+ |
|
||||||
| Name: name | Name: media-type | |
|
||||||
| Labels: | Labels: | |
|
||||||
| Type: []string | Type: []string | |
|
||||||
+--------------------------+------------------+ |
|
||||||
| browser_marketshare.csv | | |
|
||||||
| flight_info_by_state.csv | | |
|
||||||
| gdp_per_capita.csv | | |
|
||||||
| js_libraries.csv | | |
|
||||||
| ohlc_dogecoin.csv | | |
|
||||||
| population_by_state.csv | | |
|
||||||
| weight_height.csv | | |
|
||||||
+--------------------------+------------------+ |
|
||||||
|
|
||||||
|
|
||||||
====== TEST DATA RESPONSE (arrow base64) ====== |
|
||||||
FRAME=QVJST1cxAAD/////uAEAABAAAAAAAAoADgAMAAsABAAKAAAAFAAAAAAAAAEDAAoADAAAAAgABAAKAAAACAAAAKQAAAADAAAATAAAACgAAAAEAAAA0P7//wgAAAAMAAAAAAAAAAAAAAAFAAAAcmVmSWQAAADw/v//CAAAAAwAAAAAAAAAAAAAAAQAAABuYW1lAAAAABD///8IAAAAPAAAADAAAAB7InR5cGUiOiJkaXJlY3RvcnktbGlzdGluZyIsInBhdGhTZXBhcmF0b3IiOiIvIn0AAAAABAAAAG1ldGEAAAAAAgAAAHwAAAAEAAAAnv///xQAAABAAAAAQAAAAAAAAAU8AAAAAQAAAAQAAACM////CAAAABQAAAAKAAAAbWVkaWEtdHlwZQAABAAAAG5hbWUAAAAAAAAAAIj///8KAAAAbWVkaWEtdHlwZQAAAAASABgAFAAAABMADAAAAAgABAASAAAAFAAAAEQAAABIAAAAAAAABUQAAAABAAAADAAAAAgADAAIAAQACAAAAAgAAAAQAAAABAAAAG5hbWUAAAAABAAAAG5hbWUAAAAAAAAAAAQABAAEAAAABAAAAG5hbWUAAAAA/////9gAAAAUAAAAAAAAAAwAFgAUABMADAAEAAwAAADQAAAAAAAAABQAAAAAAAADAwAKABgADAAIAAQACgAAABQAAAB4AAAABwAAAAAAAAAAAAAABgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAACAAAAAAAAAAIAAAAAAAAACQAAAAAAAAALAAAAAAAAAAAAAAAAAAAACwAAAAAAAAACAAAAAAAAAA0AAAAAAAAAAAAAAAAAAAAAAAAAACAAAABwAAAAAAAAAAAAAAAAAAAAcAAAAAAAAAAAAAAAAAAAAAAAAAFwAAAC8AAABBAAAAUQAAAGIAAAB5AAAAigAAAGJyb3dzZXJfbWFya2V0c2hhcmUuY3N2ZmxpZ2h0X2luZm9fYnlfc3RhdGUuY3N2Z2RwX3Blcl9jYXBpdGEuY3N2anNfbGlicmFyaWVzLmNzdm9obGNfZG9nZWNvaW4uY3N2cG9wdWxhdGlvbl9ieV9zdGF0ZS5jc3Z3ZWlnaHRfaGVpZ2h0LmNzdgAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEAAAAAwAFAASAAwACAAEAAwAAAAQAAAALAAAADwAAAAAAAMAAQAAAMgBAAAAAAAA4AAAAAAAAADQAAAAAAAAAAAAAAAAAAAAAAAAAAAACgAMAAAACAAEAAoAAAAIAAAApAAAAAMAAABMAAAAKAAAAAQAAADQ/v//CAAAAAwAAAAAAAAAAAAAAAUAAAByZWZJZAAAAPD+//8IAAAADAAAAAAAAAAAAAAABAAAAG5hbWUAAAAAEP///wgAAAA8AAAAMAAAAHsidHlwZSI6ImRpcmVjdG9yeS1saXN0aW5nIiwicGF0aFNlcGFyYXRvciI6Ii8ifQAAAAAEAAAAbWV0YQAAAAACAAAAfAAAAAQAAACe////FAAAAEAAAABAAAAAAAAABTwAAAABAAAABAAAAIz///8IAAAAFAAAAAoAAABtZWRpYS10eXBlAAAEAAAAbmFtZQAAAAAAAAAAiP///woAAABtZWRpYS10eXBlAAAAABIAGAAUAAAAEwAMAAAACAAEABIAAAAUAAAARAAAAEgAAAAAAAAFRAAAAAEAAAAMAAAACAAMAAgABAAIAAAACAAAABAAAAAEAAAAbmFtZQAAAAAEAAAAbmFtZQAAAAAAAAAABAAEAAQAAAAEAAAAbmFtZQAAAADoAQAAQVJST1cx |
|
||||||
Loading…
Reference in new issue