apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Export: support export in postgresql (#58553)

Browse Source
custom-theme-test^2
Ryan McKinley 3 years ago committed by GitHub
parent 47055561ec
commit f92d978386
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
13 changed files with 109 additions and 45 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 39
      pkg/services/export/commit_helper.go
  2. 16
      pkg/services/export/export_auth.go
  3. 5
      pkg/services/export/export_dash.go
  4. 2
      pkg/services/export/export_dash_thumbs.go
  5. 3
      pkg/services/export/export_live.go
  6. 3
      pkg/services/export/export_plugins.go
  7. 9
      pkg/services/export/export_sys_playlists.go
  8. 3
      pkg/services/export/export_usage.go
  9. 16
      pkg/services/export/git_export_job.go
  10. 18
      pkg/services/export/object_store.go
  11. 7
      pkg/services/export/service.go
  12. 29
      pkg/services/export/utils.go
  13. 4
      pkg/services/store/object/sqlstash/sql_storage_server.go

39
pkg/services/export/commit_helper.go
Unescape View File

@ -14,6 +14,8 @@ import (
jsoniter "github.com/json-iterator/go"
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/services/store"
"github.com/grafana/grafana/pkg/services/user"
)
type commitHelper struct {
@ -44,13 +46,17 @@ type commitOptions struct {
comment string
}
func (ch *commitHelper) initOrg(sql db.DB, orgID int64) error {
func (ch *commitHelper) initOrg(ctx context.Context, sql db.DB, orgID int64) error {
return sql.WithDbSession(ch.ctx, func(sess *db.Session) error {
userprefix := "user"
if isPostgreSQL(sql) {
userprefix = `"user"` // postgres has special needs
}
sess.Table("user").
Join("inner", "org_user", "user.id = org_user.user_id").
Cols("user.*", "org_user.role").
Join("inner", "org_user", userprefix+`.id = org_user.user_id`).
Cols(userprefix+`.*`, "org_user.role").
Where("org_user.org_id = ?", orgID).
Asc("user.id")
Asc(userprefix + `.id`)
rows := make([]*userInfo, 0)
err := sess.Find(&rows)
@ -64,6 +70,14 @@ func (ch *commitHelper) initOrg(sql db.DB, orgID int64) error {
}
ch.users = lookup
ch.orgID = orgID
// Set an admin user with the
rowUser := &user.SignedInUser{
Login: "",
OrgID: orgID, // gets filled in from each row
UserID: 0,
}
ch.ctx = store.ContextWithUser(context.Background(), rowUser)
return err
})
}
@ -140,19 +154,28 @@ func (ch *commitHelper) add(opts commitOptions) error {
}
type userInfo struct {
ID int64 `json:"-" xorm:"id"`
ID int64 `json:"-" db:"id"`
Login string `json:"login"`
Email string `json:"email"`
Name string `json:"name"`
Password string `json:"password"`
Salt string `json:"salt"`
Company string `json:"company,omitempty"`
Rands string `json:"-"`
Role string `json:"org_role"` // org role
Theme string `json:"-"` // managed in preferences
Created time.Time `json:"-"` // managed in git or external source
Updated time.Time `json:"-"` // managed in git or external source
IsDisabled bool `json:"disabled" xorm:"is_disabled"`
IsServiceAccount bool `json:"serviceAccount" xorm:"is_service_account"`
LastSeenAt time.Time `json:"-" xorm:"last_seen_at"`
IsDisabled bool `json:"disabled" db:"is_disabled"`
IsServiceAccount bool `json:"serviceAccount" db:"is_service_account"`
LastSeenAt time.Time `json:"-" db:"last_seen_at"`
// Added to make sqlx happy
Version int `json:"-"`
HelpFlags1 int `json:"-" db:"help_flags1"`
OrgID int64 `json:"-" db:"org_id"`
EmailVerified bool `json:"-" db:"email_verified"`
IsAdmin bool `json:"-" db:"is_admin"`
}
func (u *userInfo) getAuthor() object.Signature {

16
pkg/services/export/export_auth.go
Unescape View File

@ -3,7 +3,6 @@ package export
import (
"path"
"strconv"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/data"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
@ -12,6 +11,8 @@ import (
)
func dumpAuthTables(helper *commitHelper, job *gitExportJob) error {
isMySQL := isMySQLEngine(job.sql)
return job.sql.WithDbSession(helper.ctx, func(sess *db.Session) error {
commit := commitOptions{
comment: "auth tables dump",
@ -27,11 +28,11 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error {
dump := []statsTables{
{
table: "user",
sql: `
SELECT user.*, org_user.role
FROM user
JOIN org_user ON user.id = org_user.user_id
WHERE org_user.org_id =` + strconv.FormatInt(helper.orgID, 10),
sql: removeQuotesFromQuery(`
SELECT "user".*, org_user.role
FROM "user"
JOIN org_user ON "user".id = org_user.user_id
WHERE org_user.org_id =`+strconv.FormatInt(helper.orgID, 10), isMySQL),
converters: []sqlutil.Converter{{Dynamic: true}},
drop: []string{
"id", "version",
@ -74,7 +75,6 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error {
WHERE org_user.org_id =` + strconv.FormatInt(helper.orgID, 10),
},
{table: "team"},
{table: "team_group"},
{table: "team_role"},
{table: "team_member"},
{table: "temp_user"},
@ -99,7 +99,7 @@ func dumpAuthTables(helper *commitHelper, job *gitExportJob) error {
rows, err := sess.DB().QueryContext(helper.ctx, auth.sql)
if err != nil {
if strings.HasPrefix(err.Error(), "no such table") {
if isTableNotExistsError(err) {
continue
}
return err

5
pkg/services/export/export_dash.go
Unescape View File

@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/infra/db"
"github.com/grafana/grafana/pkg/infra/filestorage"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/store/kind/dashboard"
)
@ -30,7 +31,7 @@ func exportDashboards(helper *commitHelper, job *gitExportJob) error {
return err
}
rootDir := path.Join(helper.orgDir, "root")
rootDir := path.Join(helper.orgDir, models.ObjectStoreScopeDrive)
folderStructure := commitOptions{
when: time.Now(),
comment: "Exported folder structure",
@ -95,7 +96,7 @@ func exportDashboards(helper *commitHelper, job *gitExportJob) error {
if row.IsFolder {
continue
}
fname := row.Slug + "-dash.json"
fname := row.Slug + "-dashboard.json"
fpath, ok := folders[row.FolderID]
if ok {
fpath = path.Join(fpath, fname)

2
pkg/services/export/export_dash_thumbs.go
Unescape View File

@ -46,7 +46,7 @@ func exportDashboardThumbnails(helper *commitHelper, job *gitExportJob) error {
err := sess.Find(&rows)
if err != nil {
if strings.HasPrefix(err.Error(), "no such table") {
if isTableNotExistsError(err) {
return nil
}
return err

3
pkg/services/export/export_live.go
Unescape View File

@ -3,7 +3,6 @@ package export
import (
"fmt"
"path"
"strings"
"time"
"github.com/grafana/grafana/pkg/infra/db"
@ -26,7 +25,7 @@ func exportLive(helper *commitHelper, job *gitExportJob) error {
err := sess.Find(&rows)
if err != nil {
if strings.HasPrefix(err.Error(), "no such table") {
if isTableNotExistsError(err) {
return nil
}
return err

3
pkg/services/export/export_plugins.go
Unescape View File

@ -4,7 +4,6 @@ import (
"encoding/json"
"fmt"
"path"
"strings"
"time"
"github.com/grafana/grafana/pkg/infra/db"
@ -29,7 +28,7 @@ func exportPlugins(helper *commitHelper, job *gitExportJob) error {
err := sess.Find(&rows)
if err != nil {
if strings.HasPrefix(err.Error(), "no such table") {
if isTableNotExistsError(err) {
return nil
}
return err

9
pkg/services/export/export_sys_playlists.go
Unescape View File

@ -5,6 +5,7 @@ import (
"path/filepath"
"time"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/services/playlist"
)
@ -37,8 +38,12 @@ func exportSystemPlaylists(helper *commitHelper, job *gitExportJob) error {
}
gitcmd.body = append(gitcmd.body, commitBody{
fpath: filepath.Join(helper.orgDir, "system", "playlists", fmt.Sprintf("%s-playlist.json", playlist.Uid)),
body: prettyJSON(playlist),
fpath: filepath.Join(
helper.orgDir,
models.ObjectStoreScopeEntity,
models.StandardKindPlaylist,
fmt.Sprintf("%s.json", playlist.Uid)),
body: prettyJSON(playlist),
})
}

3
pkg/services/export/export_usage.go
Unescape View File

@ -3,7 +3,6 @@ package export
import (
"path"
"strconv"
"strings"
"github.com/grafana/grafana-plugin-sdk-go/data/sqlutil"
@ -64,7 +63,7 @@ func exportUsage(helper *commitHelper, job *gitExportJob) error {
for _, usage := range dump {
rows, err := sess.DB().QueryContext(helper.ctx, usage.sql)
if err != nil {
if strings.HasPrefix(err.Error(), "no such table") {
if isTableNotExistsError(err) {
continue
}
return err

16
pkg/services/export/git_export_job.go
Unescape View File

@ -5,6 +5,7 @@ import (
"encoding/json"
"fmt"
"path"
"runtime/debug"
"sync"
"time"
@ -37,7 +38,7 @@ type gitExportJob struct {
helper *commitHelper
}
func startGitExportJob(cfg ExportConfig, sql db.DB,
func startGitExportJob(ctx context.Context, cfg ExportConfig, sql db.DB,
dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64,
broadcaster statusBroadcaster, playlistService playlist.Service, orgService org.Service,
datasourceService datasources.DataSourceService) (Job, error) {
@ -60,7 +61,7 @@ func startGitExportJob(cfg ExportConfig, sql db.DB,
}
broadcaster(job.status)
go job.start()
go job.start(ctx)
return job, nil
}
@ -83,7 +84,7 @@ func (e *gitExportJob) requestStop() {
}
// Utility function to export dashboards
func (e *gitExportJob) start() {
func (e *gitExportJob) start(ctx context.Context) {
defer func() {
e.logger.Info("Finished git export job")
e.statusMu.Lock()
@ -91,6 +92,7 @@ func (e *gitExportJob) start() {
s := e.status
if err := recover(); err != nil {
e.logger.Error("export panic", "error", err)
e.logger.Error("trace", "error", string(debug.Stack()))
s.Status = fmt.Sprintf("ERROR: %v", err)
}
// Make sure it finishes OK
@ -106,7 +108,7 @@ func (e *gitExportJob) start() {
e.broadcaster(s)
}()
err := e.doExportWithHistory()
err := e.doExportWithHistory(ctx)
if err != nil {
e.logger.Error("ERROR", "e", err)
e.status.Status = "ERROR"
@ -115,7 +117,7 @@ func (e *gitExportJob) start() {
}
}
func (e *gitExportJob) doExportWithHistory() error {
func (e *gitExportJob) doExportWithHistory(ctx context.Context) error {
r, err := git.PlainInit(e.rootDir, false)
if err != nil {
return err
@ -134,7 +136,7 @@ func (e *gitExportJob) doExportWithHistory() error {
e.helper = &commitHelper{
repo: r,
work: w,
ctx: context.Background(),
ctx: ctx,
workDir: e.rootDir,
orgDir: e.rootDir,
broadcast: func(p string) {
@ -157,7 +159,7 @@ func (e *gitExportJob) doExportWithHistory() error {
e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.ID))
e.status.Count["orgs"] += 1
}
err = e.helper.initOrg(e.sql, org.ID)
err = e.helper.initOrg(ctx, e.sql, org.ID)
if err != nil {
return err
}

18
pkg/services/export/object_store.go
Unescape View File

@ -28,7 +28,7 @@ type objectStoreJob struct {
cfg ExportConfig
broadcaster statusBroadcaster
stopRequested bool
user *user.SignedInUser
ctx context.Context
sess *session.SessionDB
playlistService playlist.Service
@ -36,7 +36,7 @@ type objectStoreJob struct {
dashboardsnapshots dashboardsnapshots.Service
}
func startObjectStoreJob(user *user.SignedInUser,
func startObjectStoreJob(ctx context.Context,
cfg ExportConfig,
broadcaster statusBroadcaster,
db db.DB,
@ -47,7 +47,7 @@ func startObjectStoreJob(user *user.SignedInUser,
job := &objectStoreJob{
logger: log.New("export_to_object_store_job"),
cfg: cfg,
user: user,
ctx: ctx,
broadcaster: broadcaster,
status: ExportStatus{
Running: true,
@ -63,7 +63,7 @@ func startObjectStoreJob(user *user.SignedInUser,
}
broadcaster(job.status)
go job.start()
go job.start(ctx)
return job, nil
}
@ -71,7 +71,7 @@ func (e *objectStoreJob) requestStop() {
e.stopRequested = true
}
func (e *objectStoreJob) start() {
func (e *objectStoreJob) start(ctx context.Context) {
defer func() {
e.logger.Info("Finished dummy export job")
@ -97,11 +97,11 @@ func (e *objectStoreJob) start() {
e.logger.Info("Starting dummy export job")
// Select all dashboards
rowUser := &user.SignedInUser{
Login: "?",
Login: "",
OrgID: 0, // gets filled in from each row
UserID: 0,
}
ctx := store.ContextWithUser(context.Background(), rowUser)
ctx = store.ContextWithUser(ctx, rowUser)
what := models.StandardKindDashboard
e.status.Count[what] = 0
@ -190,10 +190,12 @@ func (e *objectStoreJob) start() {
orgIDs := []int64{1}
what = "snapshot"
for _, orgId := range orgIDs {
rowUser.OrgID = orgId
rowUser.UserID = 1
cmd := &dashboardsnapshots.GetDashboardSnapshotsQuery{
OrgId: orgId,
Limit: 500000,
SignedInUser: e.user,
SignedInUser: rowUser,
}
err := e.dashboardsnapshots.SearchDashboardSnapshots(ctx, cmd)

7
pkg/services/export/service.go
Unescape View File

@ -1,6 +1,7 @@
package export
import (
"context"
"encoding/json"
"fmt"
"net/http"
@ -224,7 +225,7 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res
return response.Error(http.StatusLocked, "export already running", nil)
}
user := store.UserFromContext(c.Req.Context())
ctx := store.ContextWithUser(context.Background(), c.SignedInUser)
var job Job
broadcast := func(s ExportStatus) {
ex.broadcastStatus(c.OrgID, s)
@ -233,13 +234,13 @@ func (ex *StandardExport) HandleRequestExport(c *models.ReqContext) response.Res
case "dummy":
job, err = startDummyExportJob(cfg, broadcast)
case "objectStore":
job, err = startObjectStoreJob(user, cfg, broadcast, ex.db, ex.playlistService, ex.store, ex.dashboardsnapshotsService)
job, err = startObjectStoreJob(ctx, cfg, broadcast, ex.db, ex.playlistService, ex.store, ex.dashboardsnapshotsService)
case "git":
dir := filepath.Join(ex.dataDir, "export_git", fmt.Sprintf("git_%d", time.Now().Unix()))
if err := os.MkdirAll(dir, os.ModePerm); err != nil {
return response.Error(http.StatusBadRequest, "Error creating export folder", nil)
}
job, err = startGitExportJob(cfg, ex.db, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService, ex.orgService, ex.datasourceService)
job, err = startGitExportJob(ctx, cfg, ex.db, ex.dashboardsnapshotsService, dir, c.OrgID, broadcast, ex.playlistService, ex.orgService, ex.datasourceService)
default:
return response.Error(http.StatusBadRequest, "Unsupported job format", nil)
}

29
pkg/services/export/utils.go
Unescape View File

@ -0,0 +1,29 @@
package export
import (
"strings"
"github.com/grafana/grafana/pkg/infra/db"
)
func isTableNotExistsError(err error) bool {
txt := err.Error()
return strings.HasPrefix(txt, "no such table") || // SQLite
strings.HasSuffix(txt, " does not exist") || // PostgreSQL
strings.HasSuffix(txt, " doesn't exist") // MySQL
}
func removeQuotesFromQuery(query string, remove bool) string {
if remove {
return strings.ReplaceAll(query, `"`, "")
}
return query
}
func isMySQLEngine(sql db.DB) bool {
return sql.GetDBType() == "mysql"
}
func isPostgreSQL(sql db.DB) bool {
return sql.GetDBType() == "postgres"
}

4
pkg/services/store/object/sqlstash/sql_storage_server.go
Unescape View File

@ -552,6 +552,10 @@ func (s *sqlObjectServer) History(ctx context.Context, r *object.ObjectHistoryRe
func (s *sqlObjectServer) Search(ctx context.Context, r *object.ObjectSearchRequest) (*object.ObjectSearchResponse, error) {
user := store.UserFromContext(ctx)
if user == nil {
return nil, fmt.Errorf("missing user in context")
}
if r.NextPageToken != "" || len(r.Sort) > 0 || len(r.Labels) > 0 {
return nil, fmt.Errorf("not yet supported")
}

Write Preview
Loading…
Cancel
Save