mirror of https://github.com/grafana/grafana
Export: save all dashboards to git (#48233)
Ryan McKinley
3 years ago
committed by
GitHub
parent
4a00c7ebde
commit
4fa606c600
@ -0,0 +1,153 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"io/ioutil" |
||||
"os" |
||||
"path" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/go-git/go-git/v5" |
||||
"github.com/go-git/go-git/v5/plumbing/object" |
||||
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
jsoniter "github.com/json-iterator/go" |
||||
) |
||||
|
||||
type commitHelper struct { |
||||
ctx context.Context |
||||
repo *git.Repository |
||||
work *git.Worktree |
||||
orgDir string // includes the orgID
|
||||
workDir string // same as the worktree root
|
||||
orgID int64 |
||||
users map[int64]*userInfo |
||||
} |
||||
|
||||
type commitBody struct { |
||||
fpath string // absolute
|
||||
body []byte |
||||
frame *data.Frame |
||||
} |
||||
|
||||
type commitOptions struct { |
||||
body []commitBody |
||||
when time.Time |
||||
userID int64 |
||||
comment string |
||||
} |
||||
|
||||
func (ch *commitHelper) initOrg(sql *sqlstore.SQLStore, orgID int64) error { |
||||
return sql.WithDbSession(ch.ctx, func(sess *sqlstore.DBSession) error { |
||||
sess.Table("user"). |
||||
Join("inner", "org_user", "user.id = org_user.user_id"). |
||||
Cols("user.*", "org_user.role"). |
||||
Where("org_user.org_id = ?", orgID). |
||||
Asc("user.id") |
||||
|
||||
rows := make([]*userInfo, 0) |
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
lookup := make(map[int64]*userInfo, len(rows)) |
||||
for _, row := range rows { |
||||
lookup[row.ID] = row |
||||
} |
||||
ch.users = lookup |
||||
ch.orgID = orgID |
||||
return err |
||||
}) |
||||
} |
||||
|
||||
func (ch *commitHelper) add(opts commitOptions) error { |
||||
for _, b := range opts.body { |
||||
if !strings.HasPrefix(b.fpath, ch.orgDir) { |
||||
return fmt.Errorf("invalid path, must be within the root folder") |
||||
} |
||||
|
||||
// make sure the parent exists
|
||||
err := os.MkdirAll(path.Dir(b.fpath), 0750) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
body := b.body |
||||
if b.frame != nil { |
||||
body, err = jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(b.frame, "", " ") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
err = ioutil.WriteFile(b.fpath, body, 0644) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
sub := b.fpath[len(ch.workDir)+1:] |
||||
_, err = ch.work.Add(sub) |
||||
if err != nil { |
||||
status, e2 := ch.work.Status() |
||||
if e2 != nil { |
||||
return fmt.Errorf("error adding: %s (invalud work status: %s)", sub, e2.Error()) |
||||
} |
||||
fmt.Printf("STATUS: %+v\n", status) |
||||
return fmt.Errorf("unable to add file: %s (%d)", sub, len(b.body)) |
||||
} |
||||
} |
||||
|
||||
user, ok := ch.users[opts.userID] |
||||
if !ok { |
||||
user = &userInfo{ |
||||
Name: "admin", |
||||
Email: "admin@unknown.org", |
||||
} |
||||
} |
||||
sig := user.getAuthor() |
||||
if opts.when.Unix() > 10 { |
||||
sig.When = opts.when |
||||
} |
||||
|
||||
copts := &git.CommitOptions{ |
||||
Author: &sig, |
||||
} |
||||
|
||||
_, err := ch.work.Commit(opts.comment, copts) |
||||
return err |
||||
} |
||||
|
||||
type userInfo struct { |
||||
ID int64 `json:"-" xorm:"id"` |
||||
Login string `json:"login"` |
||||
Email string `json:"email"` |
||||
Name string `json:"name"` |
||||
Password string `json:"password"` |
||||
Salt string `json:"salt"` |
||||
Role string `json:"org_role"` // org role
|
||||
Theme string `json:"-"` // managed in preferences
|
||||
Created time.Time `json:"-"` // managed in git or external source
|
||||
Updated time.Time `json:"-"` // managed in git or external source
|
||||
IsDisabled bool `json:"disabled" xorm:"is_disabled"` |
||||
IsServiceAccount bool `json:"serviceAccount" xorm:"is_service_account"` |
||||
LastSeenAt time.Time `json:"-" xorm:"last_seen_at"` |
||||
} |
||||
|
||||
func (u *userInfo) getAuthor() object.Signature { |
||||
return object.Signature{ |
||||
Name: firstRealStringX(u.Name, u.Login, u.Email, "?"), |
||||
Email: firstRealStringX(u.Email, u.Login, u.Name, "?"), |
||||
} |
||||
} |
||||
|
||||
func firstRealStringX(vals ...string) string { |
||||
for _, v := range vals { |
||||
if v != "" { |
||||
return v |
||||
} |
||||
} |
||||
return "?" |
||||
} |
||||
@ -0,0 +1,129 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
jsoniter "github.com/json-iterator/go" |
||||
) |
||||
|
||||
func exportAnnotations(helper *commitHelper, job *gitExportJob) error { |
||||
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
type annoResult struct { |
||||
ID int64 `xorm:"id"` |
||||
DashboardID int64 `xorm:"dashboard_id"` |
||||
PanelID int64 `xorm:"panel_id"` |
||||
UserID int64 `xorm:"user_id"` |
||||
Text string `xorm:"text"` |
||||
Epoch int64 `xorm:"epoch"` |
||||
EpochEnd int64 `xorm:"epoch_end"` |
||||
Created int64 `xorm:"created"` // not used
|
||||
Tags string `xorm:"tags"` // JSON Array
|
||||
} |
||||
|
||||
type annoEvent struct { |
||||
PanelID int64 `json:"panel"` |
||||
Text string `json:"text"` |
||||
Epoch int64 `json:"epoch"` // dashboard/start+end is really the UID
|
||||
EpochEnd int64 `json:"epoch_end,omitempty"` |
||||
Tags []string |
||||
} |
||||
|
||||
rows := make([]*annoResult, 0) |
||||
|
||||
sess.Table("annotation"). |
||||
Where("org_id = ? AND alert_id = 0", helper.orgID).Asc("epoch") |
||||
|
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
count := len(rows) |
||||
f_ID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) |
||||
f_DashboardID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) |
||||
f_PanelID := data.NewFieldFromFieldType(data.FieldTypeInt64, count) |
||||
f_Epoch := data.NewFieldFromFieldType(data.FieldTypeTime, count) |
||||
f_EpochEnd := data.NewFieldFromFieldType(data.FieldTypeNullableTime, count) |
||||
f_Text := data.NewFieldFromFieldType(data.FieldTypeString, count) |
||||
f_Tags := data.NewFieldFromFieldType(data.FieldTypeJSON, count) |
||||
|
||||
f_ID.Name = "ID" |
||||
f_DashboardID.Name = "DashboardID" |
||||
f_PanelID.Name = "PanelID" |
||||
f_Epoch.Name = "Epoch" |
||||
f_EpochEnd.Name = "EpochEnd" |
||||
f_Text.Name = "Text" |
||||
f_Tags.Name = "Tags" |
||||
|
||||
for id, row := range rows { |
||||
f_ID.Set(id, row.ID) |
||||
f_DashboardID.Set(id, row.DashboardID) |
||||
f_PanelID.Set(id, row.PanelID) |
||||
f_Epoch.Set(id, time.UnixMilli(row.Epoch)) |
||||
if row.Epoch != row.EpochEnd { |
||||
f_EpochEnd.SetConcrete(id, time.UnixMilli(row.EpochEnd)) |
||||
} |
||||
f_Text.Set(id, row.Text) |
||||
f_Tags.Set(id, json.RawMessage(row.Tags)) |
||||
|
||||
// Save a file for each
|
||||
event := &annoEvent{ |
||||
PanelID: row.PanelID, |
||||
Text: row.Text, |
||||
} |
||||
err = json.Unmarshal([]byte(row.Tags), &event.Tags) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
fname := fmt.Sprintf("%d", row.Epoch) |
||||
if row.Epoch != row.EpochEnd { |
||||
fname += "-" + fmt.Sprintf("%d", row.EpochEnd) |
||||
} |
||||
|
||||
err = helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, |
||||
"annotations", |
||||
"dashboard", |
||||
fmt.Sprintf("id-%d", row.DashboardID), |
||||
fname+".json"), |
||||
body: prettyJSON(event), |
||||
}, |
||||
}, |
||||
when: time.UnixMilli(row.Epoch), |
||||
comment: fmt.Sprintf("Added annotation (%d)", row.ID), |
||||
userID: row.UserID, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
frame := data.NewFrame("", f_ID, f_DashboardID, f_PanelID, f_Epoch, f_EpochEnd, f_Text, f_Tags) |
||||
js, err := jsoniter.ConfigCompatibleWithStandardLibrary.MarshalIndent(frame, "", " ") |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, "annotations", "annotations.json"), |
||||
body: js, // TODO, pretty?
|
||||
}, |
||||
}, |
||||
when: time.Now(), |
||||
comment: "Exported annotations", |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return err |
||||
}) |
||||
} |
||||
@ -0,0 +1,73 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
func dumpAuthTables(helper *commitHelper, job *gitExportJob) error { |
||||
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
commit := commitOptions{ |
||||
comment: "auth tables dump", |
||||
} |
||||
|
||||
tables := []string{ |
||||
"user", // joined with "org_user" to get the role
|
||||
"user_role", |
||||
"builtin_role", |
||||
"api_key", |
||||
"team", "team_group", "team_role", "team_member", |
||||
"role", |
||||
"temp_user", |
||||
"user_auth_token", // no org_id... is it temporary?
|
||||
"permission", |
||||
} |
||||
|
||||
for _, table := range tables { |
||||
switch table { |
||||
case "permission": |
||||
sess.Table(table). |
||||
Join("left", "role", "permission.role_id = role.id"). |
||||
Cols("permission.*"). |
||||
Where("org_id = ?", helper.orgID). |
||||
Asc("permission.id") |
||||
case "user": |
||||
sess.Table(table). |
||||
Join("inner", "org_user", "user.id = org_user.user_id"). |
||||
Cols("user.*", "org_user.role"). |
||||
Where("org_user.org_id = ?", helper.orgID). |
||||
Asc("user.id") |
||||
case "user_auth_token": |
||||
sess.Table(table). |
||||
Join("inner", "org_user", "user_auth_token.id = org_user.user_id"). |
||||
Cols("user_auth_token.*"). |
||||
Where("org_user.org_id = ?", helper.orgID). |
||||
Asc("user_auth_token.id") |
||||
default: |
||||
sess.Table(table).Where("org_id = ?", helper.orgID).Asc("id") |
||||
} |
||||
|
||||
raw, err := sess.QueryInterface() |
||||
if err != nil { |
||||
return fmt.Errorf("unable to read: %s // %s", table, err.Error()) |
||||
} |
||||
if len(raw) < 1 { |
||||
continue // don't write empty files
|
||||
} |
||||
frame, err := queryResultToDataFrame(raw, frameOpts{ |
||||
skip: []string{"org_id", "version", "help_flags1", "theme"}, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
frame.Name = table |
||||
commit.body = append(commit.body, commitBody{ |
||||
fpath: path.Join(helper.orgDir, "auth", "sql.dump", table+".json"), |
||||
frame: frame, |
||||
}) |
||||
} |
||||
return helper.add(commit) |
||||
}) |
||||
} |
||||
@ -0,0 +1,229 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"bytes" |
||||
"encoding/json" |
||||
"fmt" |
||||
"path" |
||||
"path/filepath" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/google/uuid" |
||||
"github.com/grafana/grafana/pkg/infra/filestorage" |
||||
"github.com/grafana/grafana/pkg/services/searchV2/extract" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
func exportDashboards(helper *commitHelper, job *gitExportJob, lookup dsLookup) error { |
||||
alias := make(map[string]string, 100) |
||||
ids := make(map[int64]string, 100) |
||||
folders := make(map[int64]string, 100) |
||||
|
||||
// Should root files be at the root or in a subfolder called "general"?
|
||||
if true { |
||||
folders[0] = "general" |
||||
} |
||||
|
||||
rootDir := path.Join(helper.orgDir, "root") |
||||
folderStructure := commitOptions{ |
||||
when: time.Now(), |
||||
comment: "Exported folder structure", |
||||
} |
||||
|
||||
err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
type dashDataQueryResult struct { |
||||
Id int64 |
||||
UID string `xorm:"uid"` |
||||
IsFolder bool `xorm:"is_folder"` |
||||
FolderID int64 `xorm:"folder_id"` |
||||
Slug string `xorm:"slug"` |
||||
Data []byte |
||||
Created time.Time |
||||
Updated time.Time |
||||
} |
||||
|
||||
rows := make([]*dashDataQueryResult, 0) |
||||
|
||||
sess.Table("dashboard"). |
||||
Where("org_id = ?", helper.orgID). |
||||
Cols("id", "is_folder", "folder_id", "data", "slug", "created", "updated", "uid") |
||||
|
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Process all folders
|
||||
for _, row := range rows { |
||||
if !row.IsFolder { |
||||
continue |
||||
} |
||||
dash, err := extract.ReadDashboard(bytes.NewReader(row.Data), lookup) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
dash.UID = row.UID |
||||
slug := cleanFileName(dash.Title) |
||||
folder := map[string]string{ |
||||
"title": dash.Title, |
||||
} |
||||
|
||||
folderStructure.body = append(folderStructure.body, commitBody{ |
||||
fpath: path.Join(rootDir, slug, "__folder.json"), |
||||
body: prettyJSON(folder), |
||||
}) |
||||
|
||||
alias[dash.UID] = slug |
||||
folders[row.Id] = slug |
||||
|
||||
if row.Created.Before(folderStructure.when) { |
||||
folderStructure.when = row.Created |
||||
} |
||||
} |
||||
|
||||
// Now process the dashboards in each folder
|
||||
for _, row := range rows { |
||||
if row.IsFolder { |
||||
continue |
||||
} |
||||
fname := row.Slug + "-dash.json" |
||||
fpath, ok := folders[row.FolderID] |
||||
if ok { |
||||
fpath = path.Join(fpath, fname) |
||||
} else { |
||||
fpath = fname |
||||
} |
||||
|
||||
alias[row.UID] = fpath |
||||
ids[row.Id] = fpath |
||||
} |
||||
|
||||
return err |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = helper.add(folderStructure) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, "root-alias.json"), |
||||
body: prettyJSON(alias), |
||||
}, |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, "root-ids.json"), |
||||
body: prettyJSON(ids), |
||||
}, |
||||
}, |
||||
when: folderStructure.when, |
||||
comment: "adding UID alias structure", |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Now walk the history
|
||||
err = job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
type dashVersionResult struct { |
||||
DashId int64 `xorm:"dashboard_id"` |
||||
Version int64 `xorm:"version"` |
||||
Created time.Time `xorm:"created"` |
||||
CreatedBy int64 `xorm:"created_by"` |
||||
Message string `xorm:"message"` |
||||
Data []byte |
||||
} |
||||
|
||||
rows := make([]*dashVersionResult, 0, len(ids)) |
||||
|
||||
sess.Table("dashboard_version"). |
||||
Join("INNER", "dashboard", "dashboard.id = dashboard_version.dashboard_id"). |
||||
Where("org_id = ?", job.orgID). |
||||
Cols("dashboard_version.dashboard_id", |
||||
"dashboard_version.version", |
||||
"dashboard_version.created", |
||||
"dashboard_version.created_by", |
||||
"dashboard_version.message", |
||||
"dashboard_version.data"). |
||||
Asc("dashboard_version.created") |
||||
|
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
count := int64(0) |
||||
|
||||
// Process all folders (only one level deep!!!)
|
||||
for _, row := range rows { |
||||
fpath, ok := ids[row.DashId] |
||||
if !ok { |
||||
continue |
||||
} |
||||
|
||||
msg := row.Message |
||||
if msg == "" { |
||||
msg = fmt.Sprintf("Version: %d", row.Version) |
||||
} |
||||
|
||||
err = helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(rootDir, fpath), |
||||
body: cleanDashboardJSON(row.Data), |
||||
}, |
||||
}, |
||||
userID: row.CreatedBy, |
||||
when: row.Created, |
||||
comment: msg, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
count++ |
||||
fmt.Printf("COMMIT: %d // %s (%d)\n", count, fpath, row.Version) |
||||
|
||||
job.status.Current = count |
||||
job.status.Last = fpath |
||||
job.status.Changed = time.Now().UnixMilli() |
||||
job.broadcaster(job.status) |
||||
} |
||||
|
||||
return nil |
||||
}) |
||||
|
||||
return err |
||||
} |
||||
|
||||
func cleanDashboardJSON(data []byte) []byte { |
||||
var dash map[string]interface{} |
||||
err := json.Unmarshal(data, &dash) |
||||
if err != nil { |
||||
return nil |
||||
} |
||||
delete(dash, "id") |
||||
delete(dash, "uid") |
||||
delete(dash, "version") |
||||
|
||||
clean, _ := json.MarshalIndent(dash, "", " ") |
||||
return clean |
||||
} |
||||
|
||||
// replace any unsafe file name characters... TODO, but be a standard way to do this cleanly!!!
|
||||
func cleanFileName(name string) string { |
||||
name = strings.ReplaceAll(name, "/", "-") |
||||
name = strings.ReplaceAll(name, "\\", "-") |
||||
name = strings.ReplaceAll(name, ":", "-") |
||||
if err := filestorage.ValidatePath(filestorage.Delimiter + name); err != nil { |
||||
randomName, _ := uuid.NewRandom() |
||||
return randomName.String() |
||||
} |
||||
return name |
||||
} |
||||
@ -0,0 +1,72 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
"sort" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/datasources" |
||||
"github.com/grafana/grafana/pkg/services/searchV2/extract" |
||||
) |
||||
|
||||
type dsLookup func(ref *extract.DataSourceRef) *extract.DataSourceRef |
||||
|
||||
func exportDataSources(helper *commitHelper, job *gitExportJob) (dsLookup, error) { |
||||
cmd := &datasources.GetDataSourcesQuery{ |
||||
OrgId: job.orgID, |
||||
} |
||||
err := job.sql.GetDataSources(helper.ctx, cmd) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
sort.SliceStable(cmd.Result, func(i, j int) bool { |
||||
return cmd.Result[i].Created.After(cmd.Result[j].Created) |
||||
}) |
||||
|
||||
byUID := make(map[string]*extract.DataSourceRef, len(cmd.Result)) |
||||
byName := make(map[string]*extract.DataSourceRef, len(cmd.Result)) |
||||
for _, ds := range cmd.Result { |
||||
ref := &extract.DataSourceRef{ |
||||
UID: ds.Uid, |
||||
Type: ds.Type, |
||||
} |
||||
byUID[ds.Uid] = ref |
||||
byName[ds.Name] = ref |
||||
ds.OrgId = 0 |
||||
ds.Version = 0 |
||||
|
||||
err := helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, "datasources", fmt.Sprintf("%s-ds.json", ds.Uid)), |
||||
body: prettyJSON(ds), |
||||
}, |
||||
}, |
||||
when: ds.Created, |
||||
comment: fmt.Sprintf("Add datasource: %s", ds.Name), |
||||
}) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
} |
||||
|
||||
// Return the lookup function
|
||||
return func(ref *extract.DataSourceRef) *extract.DataSourceRef { |
||||
if ref == nil || ref.UID == "" { |
||||
return &extract.DataSourceRef{ |
||||
UID: "default.uid", |
||||
Type: "default.type", |
||||
} |
||||
} |
||||
v, ok := byUID[ref.UID] |
||||
if ok { |
||||
return v |
||||
} |
||||
v, ok = byName[ref.UID] |
||||
if ok { |
||||
return v |
||||
} |
||||
return nil |
||||
}, nil |
||||
} |
||||
@ -0,0 +1,43 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/dashboardsnapshots" |
||||
) |
||||
|
||||
func exportSnapshots(helper *commitHelper, job *gitExportJob) error { |
||||
cmd := &dashboardsnapshots.GetDashboardSnapshotsQuery{ |
||||
OrgId: job.orgID, |
||||
Limit: 500000, |
||||
SignedInUser: nil, |
||||
} |
||||
if cmd.SignedInUser == nil { |
||||
return fmt.Errorf("snapshots requires an admin user") |
||||
} |
||||
|
||||
err := job.dashboardsnapshotsService.SearchDashboardSnapshots(helper.ctx, cmd) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if len(cmd.Result) < 1 { |
||||
return nil // nothing
|
||||
} |
||||
|
||||
gitcmd := commitOptions{ |
||||
when: time.Now(), |
||||
comment: "Export playlists", |
||||
} |
||||
|
||||
for _, snapshot := range cmd.Result { |
||||
gitcmd.body = append(gitcmd.body, commitBody{ |
||||
fpath: filepath.Join(helper.orgDir, "snapshot", fmt.Sprintf("%d-snapshot.json", snapshot.Id)), |
||||
body: prettyJSON(snapshot), |
||||
}) |
||||
} |
||||
|
||||
return helper.add(gitcmd) |
||||
} |
||||
@ -0,0 +1,40 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/models" |
||||
) |
||||
|
||||
func exportSystemPlaylists(helper *commitHelper, job *gitExportJob) error { |
||||
cmd := &models.GetPlaylistsQuery{ |
||||
OrgId: job.orgID, |
||||
Limit: 500000, |
||||
} |
||||
err := job.sql.SearchPlaylists(helper.ctx, cmd) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if len(cmd.Result) < 1 { |
||||
return nil // nothing
|
||||
} |
||||
|
||||
gitcmd := commitOptions{ |
||||
when: time.Now(), |
||||
comment: "Export playlists", |
||||
} |
||||
|
||||
for _, playlist := range cmd.Result { |
||||
// TODO: fix the playlist API so it returns the json we need :)
|
||||
|
||||
gitcmd.body = append(gitcmd.body, commitBody{ |
||||
fpath: filepath.Join(helper.orgDir, "system", "playlists", fmt.Sprintf("%s-playlist.json", playlist.UID)), |
||||
body: prettyJSON(playlist), |
||||
}) |
||||
} |
||||
|
||||
return helper.add(gitcmd) |
||||
} |
||||
@ -0,0 +1,128 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path" |
||||
"path/filepath" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
func exportSystemPreferences(helper *commitHelper, job *gitExportJob) error { |
||||
type preferences struct { |
||||
UserID int64 `json:"-" xorm:"user_id"` |
||||
TeamID int64 `json:"-" xorm:"team_id"` |
||||
HomeDashboardID int64 `json:"-" xorm:"home_dashboard_id"` |
||||
Updated time.Time `json:"-" xorm:"updated"` |
||||
JSONData map[string]interface{} `json:"-" xorm:"json_data"` |
||||
|
||||
Theme string `json:"theme"` |
||||
Locale string `json:"locale"` |
||||
Timezone string `json:"timezone"` |
||||
WeekStart string `json:"week_start,omitempty"` |
||||
HomeDashboard string `json:"home,omitempty" xorm:"uid"` // dashboard
|
||||
NavBar interface{} `json:"navbar,omitempty"` |
||||
QueryHistory interface{} `json:"queryHistory,omitempty"` |
||||
} |
||||
|
||||
prefsDir := path.Join(helper.orgDir, "system", "preferences") |
||||
users := make(map[int64]*userInfo, len(helper.users)) |
||||
for _, user := range helper.users { |
||||
users[user.ID] = user |
||||
} |
||||
|
||||
return job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
rows := make([]*preferences, 0) |
||||
|
||||
sess.Table("preferences"). |
||||
Join("LEFT", "dashboard", "dashboard.id = preferences.home_dashboard_id"). |
||||
Cols("preferences.*", "dashboard.uid"). |
||||
Where("preferences.org_id = ?", helper.orgID) |
||||
|
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
var comment string |
||||
var fpath string |
||||
for _, row := range rows { |
||||
if row.TeamID > 0 { |
||||
fpath = filepath.Join(prefsDir, "team", fmt.Sprintf("%d.json", row.TeamID)) |
||||
comment = fmt.Sprintf("Team preferences: %d", row.TeamID) |
||||
} else if row.UserID == 0 { |
||||
fpath = filepath.Join(prefsDir, "default.json") |
||||
comment = "Default preferences" |
||||
} else { |
||||
user, ok := users[row.UserID] |
||||
if ok { |
||||
delete(users, row.UserID) |
||||
} else { |
||||
user = &userInfo{ |
||||
Login: fmt.Sprintf("__%d__", row.UserID), |
||||
} |
||||
} |
||||
fpath = filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login)) |
||||
comment = fmt.Sprintf("User preferences: %s", user.getAuthor().Name) |
||||
} |
||||
|
||||
if row.JSONData != nil { |
||||
v, ok := row.JSONData["locale"] |
||||
if ok && row.Locale == "" { |
||||
s, ok := v.(string) |
||||
if ok { |
||||
row.Locale = s |
||||
} |
||||
} |
||||
|
||||
v, ok = row.JSONData["navbar"] |
||||
if ok && row.NavBar == nil { |
||||
row.NavBar = v |
||||
} |
||||
|
||||
v, ok = row.JSONData["queryHistory"] |
||||
if ok && row.QueryHistory == nil { |
||||
row.QueryHistory = v |
||||
} |
||||
} |
||||
|
||||
err := helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: fpath, |
||||
body: prettyJSON(row), |
||||
}, |
||||
}, |
||||
when: row.Updated, |
||||
comment: comment, |
||||
userID: row.UserID, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
// add a file for all useres that may not be in the system
|
||||
for _, user := range users { |
||||
row := preferences{ |
||||
Theme: user.Theme, // never set?
|
||||
} |
||||
err := helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(prefsDir, "user", fmt.Sprintf("%s.json", user.Login)), |
||||
body: prettyJSON(row), |
||||
}, |
||||
}, |
||||
when: user.Updated, |
||||
comment: "user preferences", |
||||
userID: row.UserID, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return err |
||||
}) |
||||
} |
||||
@ -0,0 +1,65 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
|
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
func exportSystemStars(helper *commitHelper, job *gitExportJob) error { |
||||
byUser := make(map[int64][]string, 50) |
||||
|
||||
err := job.sql.WithDbSession(helper.ctx, func(sess *sqlstore.DBSession) error { |
||||
type starResult struct { |
||||
User int64 `xorm:"user_id"` |
||||
UID string `xorm:"uid"` |
||||
} |
||||
|
||||
rows := make([]*starResult, 0) |
||||
|
||||
sess.Table("star"). |
||||
Join("INNER", "dashboard", "dashboard.id = star.dashboard_id"). |
||||
Cols("star.user_id", "dashboard.uid"). |
||||
Where("dashboard.org_id = ?", helper.orgID) |
||||
|
||||
err := sess.Find(&rows) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
for _, row := range rows { |
||||
stars := append(byUser[row.User], fmt.Sprintf("dashboard/%s", row.UID)) |
||||
byUser[row.User] = stars |
||||
} |
||||
return err |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
for userID, stars := range byUser { |
||||
user, ok := helper.users[userID] |
||||
if !ok { |
||||
user = &userInfo{ |
||||
Login: fmt.Sprintf("__unknown_%d", userID), |
||||
} |
||||
} |
||||
|
||||
err := helper.add(commitOptions{ |
||||
body: []commitBody{ |
||||
{ |
||||
fpath: filepath.Join(helper.orgDir, "system", "stars", fmt.Sprintf("%s.json", user.Login)), |
||||
body: prettyJSON(stars), |
||||
}, |
||||
}, |
||||
when: user.Updated, |
||||
comment: "user preferences", |
||||
userID: userID, |
||||
}) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return nil |
||||
} |
||||
@ -0,0 +1,138 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/data" |
||||
) |
||||
|
||||
type fieldInfo struct { |
||||
Name string |
||||
Conv data.FieldConverter |
||||
} |
||||
|
||||
type frameOpts struct { |
||||
schema []fieldInfo |
||||
skip []string |
||||
} |
||||
|
||||
func prettyJSON(v interface{}) []byte { |
||||
b, _ := json.MarshalIndent(v, "", " ") |
||||
return b |
||||
} |
||||
|
||||
func queryResultToDataFrame(rows []map[string]interface{}, opts frameOpts) (*data.Frame, error) { |
||||
count := len(rows) |
||||
if count < 1 { |
||||
return nil, nil // empty frame
|
||||
} |
||||
|
||||
schema := opts.schema |
||||
if len(schema) < 1 { |
||||
skip := make(map[string]bool, len(opts.skip)) |
||||
for _, k := range opts.skip { |
||||
skip[k] = true |
||||
} |
||||
|
||||
for k, v := range rows[0] { |
||||
if skip[k] { |
||||
continue |
||||
} |
||||
field := fieldInfo{ |
||||
Name: k, |
||||
Conv: data.FieldConverter{ |
||||
OutputFieldType: data.FieldTypeFor(v), |
||||
}, |
||||
} |
||||
if field.Conv.OutputFieldType == data.FieldTypeUnknown { |
||||
fmt.Printf("UNKNOWN type: %s / %v\n", k, v) |
||||
continue |
||||
} |
||||
|
||||
// Don't write passwords to disk for now!!!!
|
||||
if k == "password" || k == "salt" { |
||||
field.Conv.Converter = func(v interface{}) (interface{}, error) { |
||||
return fmt.Sprintf("<%s>", k), nil |
||||
} |
||||
} |
||||
|
||||
schema = append(schema, field) |
||||
} |
||||
} |
||||
|
||||
fields := make([]*data.Field, len(schema)) |
||||
for i, s := range schema { |
||||
fields[i] = data.NewFieldFromFieldType(s.Conv.OutputFieldType, count) |
||||
fields[i].Name = s.Name |
||||
} |
||||
|
||||
var err error |
||||
for i, row := range rows { |
||||
for j, s := range schema { |
||||
v, ok := row[s.Name] |
||||
if ok && v != nil { |
||||
if s.Conv.Converter != nil { |
||||
v, err = s.Conv.Converter(v) |
||||
if err != nil { |
||||
return nil, fmt.Errorf("converting field: %s // %s", s.Name, err.Error()) |
||||
} |
||||
} |
||||
fields[j].Set(i, v) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Fields are in random order
|
||||
if len(opts.schema) < 1 { |
||||
last := []*data.Field{} |
||||
frame := data.NewFrame("") |
||||
lookup := make(map[string]*data.Field, len(fields)) |
||||
for _, f := range fields { |
||||
if f.Name == "id" { |
||||
frame.Fields = append(frame.Fields, f) // first
|
||||
continue |
||||
} |
||||
lookup[f.Name] = f |
||||
} |
||||
|
||||
// First items
|
||||
for _, name := range []string{"name", "login", "email", "role", "description", "uid"} { |
||||
f, ok := lookup[name] |
||||
if ok { |
||||
frame.Fields = append(frame.Fields, f) // first
|
||||
delete(lookup, name) |
||||
} |
||||
} |
||||
|
||||
// IDs
|
||||
for k, f := range lookup { |
||||
if strings.HasSuffix(k, "_id") { |
||||
frame.Fields = append(frame.Fields, f) // first
|
||||
delete(lookup, k) |
||||
} else if strings.HasPrefix(k, "is_") { |
||||
last = append(last, f) // first
|
||||
delete(lookup, k) |
||||
} |
||||
} |
||||
|
||||
// Last items
|
||||
for _, name := range []string{"created", "updated"} { |
||||
f, ok := lookup[name] |
||||
if ok { |
||||
last = append(last, f) // first
|
||||
delete(lookup, name) |
||||
} |
||||
} |
||||
|
||||
// Rest
|
||||
for _, f := range lookup { |
||||
frame.Fields = append(frame.Fields, f) |
||||
} |
||||
|
||||
frame.Fields = append(frame.Fields, last...) |
||||
return frame, nil |
||||
} |
||||
return data.NewFrame("", fields...), nil |
||||
} |
||||
@ -0,0 +1,205 @@ |
||||
package export |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"path" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/go-git/go-git/v5" |
||||
"github.com/go-git/go-git/v5/plumbing" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/log" |
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/grafana/grafana/pkg/services/dashboardsnapshots" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
var _ Job = new(gitExportJob) |
||||
|
||||
type gitExportJob struct { |
||||
logger log.Logger |
||||
sql *sqlstore.SQLStore |
||||
dashboardsnapshotsService dashboardsnapshots.Service |
||||
orgID int64 |
||||
rootDir string |
||||
|
||||
statusMu sync.Mutex |
||||
status ExportStatus |
||||
cfg ExportConfig |
||||
broadcaster statusBroadcaster |
||||
} |
||||
|
||||
type simpleExporter = func(helper *commitHelper, job *gitExportJob) error |
||||
|
||||
func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64, broadcaster statusBroadcaster) (Job, error) { |
||||
job := &gitExportJob{ |
||||
logger: log.New("git_export_job"), |
||||
cfg: cfg, |
||||
sql: sql, |
||||
dashboardsnapshotsService: dashboardsnapshotsService, |
||||
orgID: orgID, |
||||
rootDir: rootDir, |
||||
broadcaster: broadcaster, |
||||
status: ExportStatus{ |
||||
Running: true, |
||||
Target: "git export", |
||||
Started: time.Now().UnixMilli(), |
||||
Current: 0, |
||||
}, |
||||
} |
||||
|
||||
broadcaster(job.status) |
||||
go job.start() |
||||
return job, nil |
||||
} |
||||
|
||||
func (e *gitExportJob) getStatus() ExportStatus { |
||||
e.statusMu.Lock() |
||||
defer e.statusMu.Unlock() |
||||
|
||||
return e.status |
||||
} |
||||
|
||||
func (e *gitExportJob) getConfig() ExportConfig { |
||||
e.statusMu.Lock() |
||||
defer e.statusMu.Unlock() |
||||
|
||||
return e.cfg |
||||
} |
||||
|
||||
// Utility function to export dashboards
|
||||
func (e *gitExportJob) start() { |
||||
defer func() { |
||||
e.logger.Info("Finished git export job") |
||||
|
||||
e.statusMu.Lock() |
||||
defer e.statusMu.Unlock() |
||||
s := e.status |
||||
if err := recover(); err != nil { |
||||
e.logger.Error("export panic", "error", err) |
||||
s.Status = fmt.Sprintf("ERROR: %v", err) |
||||
} |
||||
// Make sure it finishes OK
|
||||
if s.Finished < 10 { |
||||
s.Finished = time.Now().UnixMilli() |
||||
} |
||||
s.Running = false |
||||
if s.Status == "" { |
||||
s.Status = "done" |
||||
} |
||||
s.Target = e.rootDir |
||||
e.status = s |
||||
e.broadcaster(s) |
||||
}() |
||||
|
||||
err := e.doExportWithHistory() |
||||
if err != nil { |
||||
e.logger.Error("ERROR", "e", err) |
||||
e.status.Status = "ERROR" |
||||
e.status.Last = err.Error() |
||||
e.broadcaster(e.status) |
||||
} |
||||
} |
||||
|
||||
func (e *gitExportJob) doExportWithHistory() error { |
||||
r, err := git.PlainInit(e.rootDir, false) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
// default to "main" branch
|
||||
h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main")) |
||||
err = r.Storer.SetReference(h) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
w, err := r.Worktree() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
helper := &commitHelper{ |
||||
repo: r, |
||||
work: w, |
||||
ctx: context.Background(), |
||||
workDir: e.rootDir, |
||||
orgDir: e.rootDir, |
||||
} |
||||
|
||||
cmd := &models.SearchOrgsQuery{} |
||||
err = e.sql.SearchOrgs(helper.ctx, cmd) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
// Export each org
|
||||
for _, org := range cmd.Result { |
||||
if len(cmd.Result) > 1 { |
||||
helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id)) |
||||
} |
||||
err = helper.initOrg(e.sql, org.Id) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
err = e.doOrgExportWithHistory(helper) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
// cleanup the folder
|
||||
e.status.Target = "pruning..." |
||||
e.broadcaster(e.status) |
||||
err = r.Prune(git.PruneOptions{}) |
||||
|
||||
// TODO
|
||||
// git gc --prune=now --aggressive
|
||||
|
||||
return err |
||||
} |
||||
|
||||
func (e *gitExportJob) doOrgExportWithHistory(helper *commitHelper) error { |
||||
lookup, err := exportDataSources(helper, e) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
if true { |
||||
err = exportDashboards(helper, e, lookup) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
|
||||
// Run all the simple exporters
|
||||
exporters := []simpleExporter{ |
||||
dumpAuthTables, |
||||
exportSystemPreferences, |
||||
exportSystemStars, |
||||
exportSystemPlaylists, |
||||
exportAnnotations, |
||||
} |
||||
|
||||
// This needs a real admin user to use the interfaces (and decrypt)
|
||||
if false { |
||||
exporters = append(exporters, exportSnapshots) |
||||
} |
||||
|
||||
for _, fn := range exporters { |
||||
err = fn(helper, e) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
} |
||||
return err |
||||
} |
||||
|
||||
/** |
||||
|
||||
git remote add origin git@github.com:ryantxu/test-dash-repo.git |
||||
git branch -M main |
||||
git push -u origin main |
||||
|
||||
**/ |
||||
Loading…
Reference in new issue