Export: include section count in status updates (#52368)

pkg/services/export/commit_helper.go

@@ -27,6 +27,8 @@ type commitHelper struct {
 	stopRequested bool
 	broadcast     func(path string)
 	exporter      string // key for the current exporter
+
+	counter int
 }
 
 type commitBody struct {
@@ -125,6 +127,7 @@ func (ch *commitHelper) add(opts commitOptions) error {
 			fmt.Printf("STATUS: %+v\n", status)
 			return fmt.Errorf("unable to add file: %s (%d)", sub, len(b.body))
 		}
+		ch.counter++
 	}
 
 	copts := &git.CommitOptions{

pkg/services/export/dummy_job.go

@@ -20,6 +20,7 @@ type dummyExportJob struct {
 	cfg           ExportConfig
 	broadcaster   statusBroadcaster
 	stopRequested bool
+	total         int
 }
 
 func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job, error) {
@@ -31,9 +32,10 @@ func startDummyExportJob(cfg ExportConfig, broadcaster statusBroadcaster) (Job,
 			Running: true,
 			Target:  "git export",
 			Started: time.Now().UnixMilli(),
-			Count:   int64(math.Round(10 + rand.Float64()*20)),
+			Count:   make(map[string]int, 10),
-			Current: 0,
+			Index:   0,
 		},
+		total: int(math.Round(10 + rand.Float64()*20)),
 	}
 
 	broadcaster(job.status)
@@ -74,12 +76,12 @@ func (e *dummyExportJob) start() {
 	for t := range ticker.C {
 		e.statusMu.Lock()
 		e.status.Changed = t.UnixMilli()
-		e.status.Current++
+		e.status.Index++
-		e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Current)
+		e.status.Last = fmt.Sprintf("ITEM: %d", e.status.Index)
 		e.statusMu.Unlock()
 
 		// Wait till we are done
-		shouldStop := e.stopRequested || e.status.Current >= e.status.Count
+		shouldStop := e.stopRequested || e.status.Index >= e.total
 		e.broadcaster(e.status)
 
 		if shouldStop {

pkg/services/export/git_export_job.go

@@ -44,7 +44,7 @@ func startGitExportJob(cfg ExportConfig, sql *sqlstore.SQLStore, dashboardsnapsh
 			Running: true,
 			Target:  "git export",
 			Started: time.Now().UnixMilli(),
-			Current: 0,
+			Count:   make(map[string]int, len(exporters)*2),
 		},
 	}
 
@@ -75,7 +75,6 @@ func (e *gitExportJob) requestStop() {
 func (e *gitExportJob) start() {
 	defer func() {
 		e.logger.Info("Finished git export job")
-
 		e.statusMu.Lock()
 		defer e.statusMu.Unlock()
 		s := e.status
@@ -128,6 +127,7 @@ func (e *gitExportJob) doExportWithHistory() error {
 		workDir: e.rootDir,
 		orgDir:  e.rootDir,
 		broadcast: func(p string) {
+			e.status.Index++
 			e.status.Last = p[len(e.rootDir):]
 			e.status.Changed = time.Now().UnixMilli()
 			e.broadcaster(e.status)
@@ -144,6 +144,7 @@ func (e *gitExportJob) doExportWithHistory() error {
 	for _, org := range cmd.Result {
 		if len(cmd.Result) > 1 {
 			e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.Id))
+			e.status.Count["orgs"] += 1
 		}
 		err = e.helper.initOrg(e.sql, org.Id)
 		if err != nil {
@@ -180,19 +181,28 @@ func (e *gitExportJob) process(exporters []Exporter) error {
 			continue
 		}
 
-		if exp.process != nil {
 		e.status.Target = exp.Key
 		e.helper.exporter = exp.Key
+
+		before := e.helper.counter
+		if exp.process != nil {
 			err := exp.process(e.helper, e)
+
 			if err != nil {
 				return err
 			}
 		}
 
 		if exp.Exporters != nil {
-			return e.process(exp.Exporters)
+			err := e.process(exp.Exporters)
+			if err != nil {
+				return err
 			}
 		}
+
+		// Aggregate the counts for each org in the same report
+		e.status.Count[exp.Key] += (e.helper.counter - before)
+	}
 	return nil
 }
 

pkg/services/export/service.go

@@ -67,36 +67,42 @@ var exporters = []Exporter{
 		process:     exportDataSources,
 	},
 	{
-		Key:         "services",
+		Key:         "system",
-		Name:        "Services",
+		Name:        "System",
 		Description: "Save service settings",
 		Exporters: []Exporter{
 			{
+				Key:         "system_preferences",
 				Name:        "Preferences",
 				Description: "User and team preferences",
 				process:     exportSystemPreferences,
 			},
 			{
+				Key:         "system_stars",
 				Name:        "Stars",
 				Description: "User stars",
 				process:     exportSystemStars,
 			},
 			{
+				Key:         "system_playlists",
 				Name:        "Playlists",
 				Description: "Playlists",
 				process:     exportSystemPlaylists,
 			},
 			{
+				Key:         "system_kv_store",
 				Name:        "Key Value store",
 				Description: "Internal KV store",
 				process:     exportKVStore,
 			},
 			{
+				Key:         "system_short_url",
 				Name:        "Short URLs",
 				Description: "saved links",
 				process:     exportSystemShortURL,
 			},
 			{
+				Key:         "system_live",
 				Name:        "Grafana live",
 				Description: "archived messages",
 				process:     exportLive,

pkg/services/export/types.go

@@ -7,10 +7,10 @@ type ExportStatus struct {
 	Started  int64          `json:"started,omitempty"`
 	Finished int64          `json:"finished,omitempty"`
 	Changed  int64          `json:"update,omitempty"`
-	Count    int64  `json:"count,omitempty"`
-	Current  int64  `json:"current,omitempty"`
 	Last     string         `json:"last,omitempty"`
 	Status   string         `json:"status"` // ERROR, SUCCESS, ETC
+	Index    int            `json:"index,omitempty"`
+	Count    map[string]int `json:"count,omitempty"`
 }
 
 // Basic export config (for now)