grafana/pkg/services/export/git_export_job.go

package export

import (
	"context"
	"encoding/json"
	"fmt"
	"path"
	"runtime/debug"
	"sync"
	"time"

	"github.com/go-git/go-git/v5"
	"github.com/go-git/go-git/v5/plumbing"

	"github.com/grafana/grafana/pkg/infra/db"
	"github.com/grafana/grafana/pkg/infra/log"
	"github.com/grafana/grafana/pkg/services/dashboardsnapshots"
	"github.com/grafana/grafana/pkg/services/datasources"
	"github.com/grafana/grafana/pkg/services/org"
	"github.com/grafana/grafana/pkg/services/playlist"
)

var _ Job = new(gitExportJob)

type gitExportJob struct {
	logger                    log.Logger
	sql                       db.DB
	dashboardsnapshotsService dashboardsnapshots.Service
	datasourceService         datasources.DataSourceService
	playlistService           playlist.Service
	orgService                org.Service
	rootDir                   string

	statusMu    sync.Mutex
	status      ExportStatus
	cfg         ExportConfig
	broadcaster statusBroadcaster
	helper      *commitHelper
}

func startGitExportJob(ctx context.Context, cfg ExportConfig, sql db.DB,
	dashboardsnapshotsService dashboardsnapshots.Service, rootDir string, orgID int64,
	broadcaster statusBroadcaster, playlistService playlist.Service, orgService org.Service,
	datasourceService datasources.DataSourceService) (Job, error) {
	job := &gitExportJob{
		logger:                    log.New("git_export_job"),
		cfg:                       cfg,
		sql:                       sql,
		dashboardsnapshotsService: dashboardsnapshotsService,
		playlistService:           playlistService,
		orgService:                orgService,
		datasourceService:         datasourceService,
		rootDir:                   rootDir,
		broadcaster:               broadcaster,
		status: ExportStatus{
			Running: true,
			Target:  "git export",
			Started: time.Now().UnixMilli(),
			Count:   make(map[string]int, len(exporters)*2),
		},
	}

	broadcaster(job.status)
	go job.start(ctx)
	return job, nil
}

func (e *gitExportJob) getStatus() ExportStatus {
	e.statusMu.Lock()
	defer e.statusMu.Unlock()

	return e.status
}

func (e *gitExportJob) getConfig() ExportConfig {
	e.statusMu.Lock()
	defer e.statusMu.Unlock()

	return e.cfg
}

func (e *gitExportJob) requestStop() {
	e.helper.stopRequested = true // will error on the next write
}

// Utility function to export dashboards
func (e *gitExportJob) start(ctx context.Context) {
	defer func() {
		e.logger.Info("Finished git export job")
		e.statusMu.Lock()
		defer e.statusMu.Unlock()
		s := e.status
		if err := recover(); err != nil {
			e.logger.Error("export panic", "error", err)
			e.logger.Error("trace", "error", string(debug.Stack()))
			s.Status = fmt.Sprintf("ERROR: %v", err)
		}
		// Make sure it finishes OK
		if s.Finished < 10 {
			s.Finished = time.Now().UnixMilli()
		}
		s.Running = false
		if s.Status == "" {
			s.Status = "done"
		}
		s.Target = e.rootDir
		e.status = s
		e.broadcaster(s)
	}()

	err := e.doExportWithHistory(ctx)
	if err != nil {
		e.logger.Error("ERROR", "e", err)
		e.status.Status = "ERROR"
		e.status.Last = err.Error()
		e.broadcaster(e.status)
	}
}

func (e *gitExportJob) doExportWithHistory(ctx context.Context) error {
	r, err := git.PlainInit(e.rootDir, false)
	if err != nil {
		return err
	}
	// default to "main" branch
	h := plumbing.NewSymbolicReference(plumbing.HEAD, plumbing.ReferenceName("refs/heads/main"))
	err = r.Storer.SetReference(h)
	if err != nil {
		return err
	}

	w, err := r.Worktree()
	if err != nil {
		return err
	}
	e.helper = &commitHelper{
		repo:    r,
		work:    w,
		ctx:     ctx,
		workDir: e.rootDir,
		orgDir:  e.rootDir,
		broadcast: func(p string) {
			e.status.Index++
			e.status.Last = p[len(e.rootDir):]
			e.status.Changed = time.Now().UnixMilli()
			e.broadcaster(e.status)
		},
	}

	cmd := &org.SearchOrgsQuery{}
	result, err := e.orgService.Search(e.helper.ctx, cmd)
	if err != nil {
		return err
	}

	// Export each org
	for _, org := range result {
		if len(result) > 1 {
			e.helper.orgDir = path.Join(e.rootDir, fmt.Sprintf("org_%d", org.ID))
			e.status.Count["orgs"] += 1
		}
		err = e.helper.initOrg(ctx, e.sql, org.ID)
		if err != nil {
			return err
		}

		err := e.process(exporters)
		if err != nil {
			return err
		}
	}

	// cleanup the folder
	e.status.Target = "pruning..."
	e.broadcaster(e.status)
	err = r.Prune(git.PruneOptions{})

	// TODO
	// git gc --prune=now --aggressive

	return err
}

func (e *gitExportJob) process(exporters []Exporter) error {
	if false { // NEEDS a real user ID first
		err := exportSnapshots(e.helper, e)
		if err != nil {
			return err
		}
	}

	for _, exp := range exporters {
		if e.cfg.Exclude[exp.Key] {
			continue
		}

		e.status.Target = exp.Key
		e.helper.exporter = exp.Key

		before := e.helper.counter
		if exp.process != nil {
			err := exp.process(e.helper, e)

			if err != nil {
				return err
			}
		}

		if exp.Exporters != nil {
			err := e.process(exp.Exporters)
			if err != nil {
				return err
			}
		}

		// Aggregate the counts for each org in the same report
		e.status.Count[exp.Key] += (e.helper.counter - before)
	}
	return nil
}

func prettyJSON(v interface{}) []byte {
	b, _ := json.MarshalIndent(v, "", "  ")
	return b
}

/**

git remote add origin git@github.com:ryantxu/test-dash-repo.git
git branch -M main
git push -u origin main

**/