From b3b64e24c2d3a62f5137310c368933f20f625a22 Mon Sep 17 00:00:00 2001 From: Ryan McKinley Date: Mon, 18 Jul 2022 08:03:08 -0700 Subject: [PATCH] Export: include section count in status updates (#52368) --- pkg/services/export/commit_helper.go | 3 +++ pkg/services/export/dummy_job.go | 12 +++++++----- pkg/services/export/git_export_job.go | 20 +++++++++++++++----- pkg/services/export/service.go | 10 ++++++++-- pkg/services/export/types.go | 18 +++++++++--------- 5 files changed, 42 insertions(+), 21 deletions(-) diff --git a/pkg/services/export/commit_helper.go b/pkg/services/export/commit_helper.go index 5848db9dd08..a244cc47ae7 100644 --- a/pkg/services/export/commit_helper.go +++ b/pkg/services/export/commit_helper.go @@ -27,6 +27,8 @@ type commitHelper struct { stopRequested bool broadcast func(path string) exporter string // key for the current exporter + + counter int } type commitBody struct { @@ -125,6 +127,7 @@ func (ch *commitHelper) add(opts commitOptions) error { fmt.Printf("STATUS: %+v\n", status) return fmt.Errorf("unable to add file: %s (%d)", sub, len(b.body)) } + ch.counter++ } copts := &git.CommitOptions{ diff --git a/pkg/services/export/dummy_job.go b/pkg/services/export/dummy_job.go index 58a38889398..eeb3a98865d 100644 --- a/pkg/services/export/dummy_job.go +++ b/pkg/services/export/dummy_job.go @@ -20,6 +20,7 @@ type dummyExportJob struct { cfg ExportConfig broadcaster statusBroadcaster stopRequested bool + total int } func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) { @@ -31,9 +32,10 @@ func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, Running: true, Target: "git export", Started: time.Now().UnixMilli(), - Count: int64(math.Round(10 + rand.Float64()*20)), - Current: 0, + Count: make(map[string]int, 10), + Index: 0, }, + total: int(math.Round(10 + rand.Float64()*20)), } broadcaster(job.status) @@ -74,12 +76,12 @@ func (e *dummyExportJob) start() { for t := range ticker.C { e.statusMu.Lock() e.status.Changed = t.UnixMilli() - e.status.Current++ - e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current) + e.status.Index++ + e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Index) e.statusMu.Unlock() // Wait till we are done - shouldStop := e.stopRequested || e.status.Current >= e.status.Count + shouldStop := e.stopRequested || e.status.Index >= e.total e.broadcaster(e.status) if shouldStop { diff --git a/pkg/services/export/git_export_job.go b/pkg/services/export/git_export_job.go index 4b13b5e44c8..b676a38e12f 100644 --- a/pkg/services/export/git_export_job.go +++ b/pkg/services/export/git_export_job.go @@ -44,7 +44,7 @@ func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapsh Running: true, Target: "git export", Started: time.Now().UnixMilli(), - Current: 0, + Count: make(map[string]int, len(exporters)*2), }, } @@ -75,7 +75,6 @@ func (e *gitExportJob) requestStop() { func (e *gitExportJob) start() { defer func() { e.logger.Info("Finished git export job") - e.statusMu.Lock() defer e.statusMu.Unlock() s := e.status @@ -128,6 +127,7 @@ func (e *gitExportJob) doExportWithHistory() error { workDir: e.rootDir, orgDir: e.rootDir, broadcast: func(p string) { + e.status.Index++ e.status.Last = p[len(e.rootDir):] e.status.Changed = time.Now().UnixMilli() e.broadcaster(e.status) @@ -144,6 +144,7 @@ func (e *gitExportJob) doExportWithHistory() error { for _, org := range cmd.Result { if len(cmd.Result) > 1 { e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id)) + e.status.Count["orgs"] += 1 } err = e.helper.initOrg(e.sql, org.Id) if err != nil { @@ -180,18 +181,27 @@ func (e *gitExportJob) process(exporters []Exporter) error { continue } + e.status.Target = exp.Key + e.helper.exporter = exp.Key + + before := e.helper.counter if exp.process != nil { - e.status.Target = exp.Key - e.helper.exporter = exp.Key err := exp.process(e.helper, e) + if err != nil { return err } } if exp.Exporters != nil { - return e.process(exp.Exporters) + err := e.process(exp.Exporters) + if err != nil { + return err + } } + + // Aggregate the counts for each org in the same report + e.status.Count[exp.Key] += (e.helper.counter - before) } return nil } diff --git a/pkg/services/export/service.go b/pkg/services/export/service.go index e8f5e185c7e..41c7c0ca678 100644 --- a/pkg/services/export/service.go +++ b/pkg/services/export/service.go @@ -67,36 +67,42 @@ var exporters = []Exporter{ process: exportDataSources, }, { - Key: "services", - Name: "Services", + Key: "system", + Name: "System", Description: "Save service settings", Exporters: []Exporter{ { + Key: "system_preferences", Name: "Preferences", Description: "User and team preferences", process: exportSystemPreferences, }, { + Key: "system_stars", Name: "Stars", Description: "User stars", process: exportSystemStars, }, { + Key: "system_playlists", Name: "Playlists", Description: "Playlists", process: exportSystemPlaylists, }, { + Key: "system_kv_store", Name: "Key Value store", Description: "Internal KV store", process: exportKVStore, }, { + Key: "system_short_url", Name: "Short URLs", Description: "saved links", process: exportSystemShortURL, }, { + Key: "system_live", Name: "Grafana live", Description: "archived messages", process: exportLive, diff --git a/pkg/services/export/types.go b/pkg/services/export/types.go index 5deade2fd69..a64cabecfef 100644 --- a/pkg/services/export/types.go +++ b/pkg/services/export/types.go @@ -2,15 +2,15 @@ package export // Export status. Only one running at a time type ExportStatus struct { - Running bool `json:"running"` - Target string `json:"target"` // description of where it is going (no secrets) - Started int64 `json:"started,omitempty"` - Finished int64 `json:"finished,omitempty"` - Changed int64 `json:"update,omitempty"` - Count int64 `json:"count,omitempty"` - Current int64 `json:"current,omitempty"` - Last string `json:"last,omitempty"` - Status string `json:"status"` // ERROR, SUCCESS, ETC + Running bool `json:"running"` + Target string `json:"target"` // description of where it is going (no secrets) + Started int64 `json:"started,omitempty"` + Finished int64 `json:"finished,omitempty"` + Changed int64 `json:"update,omitempty"` + Last string `json:"last,omitempty"` + Status string `json:"status"` // ERROR, SUCCESS, ETC + Index int `json:"index,omitempty"` + Count map[string]int `json:"count,omitempty"` } // Basic export config (for now)