Previews: create crawler auth setup service (#47349)

* #46968: add `RetrieveServiceAccountIdByName` to serviceaccounts service

* #46968: improve error logging in rendering service

* #46968: add oss crawler account setup

* #46968: fix tests

* #46968: switch back to ROLE_ADMIN

* #46968: rename to crawlerAuth

* comment crawler_auth.go

pkg/server/wireexts_oss.go

@@ -31,6 +31,7 @@ import (
 	"github.com/grafana/grafana/pkg/services/searchusers"
 	"github.com/grafana/grafana/pkg/services/searchusers/filters"
 	"github.com/grafana/grafana/pkg/services/sqlstore/migrations"
+	"github.com/grafana/grafana/pkg/services/thumbs"
 	"github.com/grafana/grafana/pkg/services/validations"
 	"github.com/grafana/grafana/pkg/setting"
 )
@@ -46,6 +47,8 @@ var wireExtsBasicSet = wire.NewSet(
 	ossaccesscontrol.ProvideService,
 	wire.Bind(new(accesscontrol.RoleRegistry), new(*ossaccesscontrol.OSSAccessControlService)),
 	wire.Bind(new(accesscontrol.AccessControl), new(*ossaccesscontrol.OSSAccessControlService)),
+	thumbs.ProvideCrawlerAuthSetupService,
+	wire.Bind(new(thumbs.CrawlerAuthSetupService), new(*thumbs.OSSCrawlerAuthSetupService)),
 	validations.ProvideValidator,
 	wire.Bind(new(models.PluginRequestValidator), new(*validations.OSSPluginRequestValidator)),
 	provisioning.ProvideService,

pkg/services/rendering/http_mode.go

@@ -46,7 +46,8 @@ func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string,
 	}
 
 	queryParams := rendererURL.Query()
-	queryParams.Add("url", rs.getURL(opts.Path))
+	url := rs.getURL(opts.Path)
+	queryParams.Add("url", url)
 	queryParams.Add("renderKey", renderKey)
 	queryParams.Add("width", strconv.Itoa(opts.Width))
 	queryParams.Add("height", strconv.Itoa(opts.Height))
@@ -74,7 +75,7 @@ func (rs *RenderingService) renderViaHTTP(ctx context.Context, renderKey string,
 		}
 	}()
 
-	err = rs.readFileResponse(reqContext, resp, filePath)
+	err = rs.readFileResponse(reqContext, resp, filePath, url)
 	if err != nil {
 		return nil, err
 	}
@@ -94,7 +95,8 @@ func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey stri
 	}
 
 	queryParams := rendererURL.Query()
-	queryParams.Add("url", rs.getURL(opts.Path))
+	url := rs.getURL(opts.Path)
+	queryParams.Add("url", url)
 	queryParams.Add("renderKey", renderKey)
 	queryParams.Add("domain", rs.domain)
 	queryParams.Add("timezone", isoTimeOffsetToPosixTz(opts.Timezone))
@@ -125,7 +127,7 @@ func (rs *RenderingService) renderCSVViaHTTP(ctx context.Context, renderKey stri
 	}
 	downloadFileName := params["filename"]
 
-	err = rs.readFileResponse(reqContext, resp, filePath)
+	err = rs.readFileResponse(reqContext, resp, filePath, url)
 	if err != nil {
 		return nil, err
 	}
@@ -156,7 +158,7 @@ func (rs *RenderingService) doRequest(ctx context.Context, url *url.URL, headers
 	return resp, nil
 }
 
-func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Response, filePath string) error {
+func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Response, filePath string, url string) error {
 	// check for timeout first
 	if errors.Is(ctx.Err(), context.DeadlineExceeded) {
 		rs.log.Info("Rendering timed out")
@@ -165,7 +167,7 @@ func (rs *RenderingService) readFileResponse(ctx context.Context, resp *http.Res
 
 	// if we didn't get a 200 response, something went wrong.
 	if resp.StatusCode != http.StatusOK {
-		rs.log.Error("Remote rendering request failed", "error", resp.Status)
+		rs.log.Error("Remote rendering request failed", "error", resp.Status, "url", url)
 		return fmt.Errorf("remote rendering request failed, status code: %d, status: %s", resp.StatusCode,
 			resp.Status)
 	}

pkg/services/serviceaccounts/database/database.go

@@ -231,6 +231,47 @@ func (s *ServiceAccountsStoreImpl) RetrieveServiceAccount(ctx context.Context, o
 	return serviceAccount, nil
 }
 
+func (s *ServiceAccountsStoreImpl) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
+	serviceAccount := &struct {
+		Id int64
+	}{}
+
+	err := s.sqlStore.WithDbSession(ctx, func(dbSession *sqlstore.DBSession) error {
+		sess := dbSession.Table("user")
+
+		whereConditions := []string{
+			fmt.Sprintf("%s.name = ?",
+				s.sqlStore.Dialect.Quote("user")),
+			fmt.Sprintf("%s.org_id = ?",
+				s.sqlStore.Dialect.Quote("user")),
+			fmt.Sprintf("%s.is_service_account = %s",
+				s.sqlStore.Dialect.Quote("user"),
+				s.sqlStore.Dialect.BooleanStr(true)),
+		}
+		whereParams := []interface{}{name, orgID}
+
+		sess.Where(strings.Join(whereConditions, " AND "), whereParams...)
+
+		sess.Cols(
+			"user.id",
+		)
+
+		if ok, err := sess.Get(serviceAccount); err != nil {
+			return err
+		} else if !ok {
+			return serviceaccounts.ErrServiceAccountNotFound
+		}
+
+		return nil
+	})
+
+	if err != nil {
+		return 0, err
+	}
+
+	return serviceAccount.Id, nil
+}
+
 func (s *ServiceAccountsStoreImpl) UpdateServiceAccount(ctx context.Context,
 	orgID, serviceAccountID int64,
 	saForm *serviceaccounts.UpdateServiceAccountForm) (*serviceaccounts.ServiceAccountProfileDTO, error) {

pkg/services/serviceaccounts/database/database_test.go

@@ -15,17 +15,24 @@ import (
 func TestStore_CreateServiceAccount(t *testing.T) {
 	_, store := setupTestDatabase(t)
 	t.Run("create service account", func(t *testing.T) {
-		saDTO, err := store.CreateServiceAccount(context.Background(), 1, "new Service Account")
+		serviceAccountName := "new Service Account"
+		serviceAccountOrgId := int64(1)
+
+		saDTO, err := store.CreateServiceAccount(context.Background(), serviceAccountOrgId, serviceAccountName)
 		require.NoError(t, err)
 		assert.Equal(t, "sa-new-service-account", saDTO.Login)
-		assert.Equal(t, "new Service Account", saDTO.Name)
+		assert.Equal(t, serviceAccountName, saDTO.Name)
 		assert.Equal(t, 0, int(saDTO.Tokens))
 
-		retrieved, err := store.RetrieveServiceAccount(context.Background(), 1, saDTO.Id)
+		retrieved, err := store.RetrieveServiceAccount(context.Background(), serviceAccountOrgId, saDTO.Id)
 		require.NoError(t, err)
 		assert.Equal(t, "sa-new-service-account", retrieved.Login)
-		assert.Equal(t, "new Service Account", retrieved.Name)
-		assert.Equal(t, 1, int(retrieved.OrgId))
+		assert.Equal(t, serviceAccountName, retrieved.Name)
+		assert.Equal(t, serviceAccountOrgId, retrieved.OrgId)
+
+		retrievedId, err := store.RetrieveServiceAccountIdByName(context.Background(), serviceAccountOrgId, serviceAccountName)
+		require.NoError(t, err)
+		assert.Equal(t, saDTO.Id, retrievedId)
 	})
 }
 

pkg/services/serviceaccounts/manager/service.go

@@ -68,3 +68,11 @@ func (sa *ServiceAccountsService) DeleteServiceAccount(ctx context.Context, orgI
 	}
 	return sa.store.DeleteServiceAccount(ctx, orgID, serviceAccountID)
 }
+
+func (sa *ServiceAccountsService) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
+	if !sa.features.IsEnabled(featuremgmt.FlagServiceAccounts) {
+		sa.log.Debug(ServiceAccountFeatureToggleNotFound)
+		return 0, nil
+	}
+	return sa.store.RetrieveServiceAccountIdByName(ctx, orgID, name)
+}

pkg/services/serviceaccounts/serviceaccounts.go

@@ -10,6 +10,7 @@ import (
 type Service interface {
 	CreateServiceAccount(ctx context.Context, orgID int64, name string) (*ServiceAccountDTO, error)
 	DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error
+	RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error)
 }
 
 type Store interface {
@@ -19,6 +20,7 @@ type Store interface {
 	UpdateServiceAccount(ctx context.Context, orgID, serviceAccountID int64,
 		saForm *UpdateServiceAccountForm) (*ServiceAccountProfileDTO, error)
 	RetrieveServiceAccount(ctx context.Context, orgID, serviceAccountID int64) (*ServiceAccountProfileDTO, error)
+	RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error)
 	DeleteServiceAccount(ctx context.Context, orgID, serviceAccountID int64) error
 	UpgradeServiceAccounts(ctx context.Context) error
 	ConvertToServiceAccounts(ctx context.Context, keys []int64) error

pkg/services/serviceaccounts/tests/common.go

@@ -38,6 +38,10 @@ func SetupUserServiceAccount(t *testing.T, sqlStore *sqlstore.SQLStore, testUser
 // create mock for serviceaccountservice
 type ServiceAccountMock struct{}
 
+func (s *ServiceAccountMock) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
+	return 0, nil
+}
+
 func (s *ServiceAccountMock) CreateServiceAccount(ctx context.Context, orgID int64, name string) (*serviceaccounts.ServiceAccountDTO, error) {
 	return nil, nil
 }
@@ -65,24 +69,31 @@ func SetupMockAccesscontrol(t *testing.T,
 // this is a way to see
 // that the Mock implements the store interface
 var _ serviceaccounts.Store = new(ServiceAccountsStoreMock)
+var _ serviceaccounts.Service = new(ServiceAccountMock)
 
 type Calls struct {
-	CreateServiceAccount      []interface{}
-	RetrieveServiceAccount    []interface{}
-	DeleteServiceAccount      []interface{}
-	UpgradeServiceAccounts    []interface{}
-	ConvertServiceAccounts    []interface{}
-	ListTokens                []interface{}
-	DeleteServiceAccountToken []interface{}
-	UpdateServiceAccount      []interface{}
-	AddServiceAccountToken    []interface{}
-	SearchOrgServiceAccounts  []interface{}
+	CreateServiceAccount           []interface{}
+	RetrieveServiceAccount         []interface{}
+	DeleteServiceAccount           []interface{}
+	UpgradeServiceAccounts         []interface{}
+	ConvertServiceAccounts         []interface{}
+	ListTokens                     []interface{}
+	DeleteServiceAccountToken      []interface{}
+	UpdateServiceAccount           []interface{}
+	AddServiceAccountToken         []interface{}
+	SearchOrgServiceAccounts       []interface{}
+	RetrieveServiceAccountIdByName []interface{}
 }
 
 type ServiceAccountsStoreMock struct {
 	Calls Calls
 }
 
+func (s *ServiceAccountsStoreMock) RetrieveServiceAccountIdByName(ctx context.Context, orgID int64, name string) (int64, error) {
+	s.Calls.RetrieveServiceAccountIdByName = append(s.Calls.RetrieveServiceAccountIdByName, []interface{}{ctx, orgID, name})
+	return 0, nil
+}
+
 func (s *ServiceAccountsStoreMock) CreateServiceAccount(ctx context.Context, orgID int64, name string) (*serviceaccounts.ServiceAccountDTO, error) {
 	// now we can test that the mock has these calls when we call the function
 	s.Calls.CreateServiceAccount = append(s.Calls.CreateServiceAccount, []interface{}{ctx, orgID, name})

pkg/services/thumbs/crawler.go

@@ -26,6 +26,7 @@ type simpleCrawler struct {
 	thumbnailRepo           thumbnailRepo
 	mode                    CrawlerMode
 	thumbnailKind           models.ThumbnailKind
+	auth                    CrawlerAuth
 	opts                    rendering.Opts
 	status                  crawlStatus
 	statusMutex             sync.RWMutex
@@ -67,8 +68,8 @@ func (r *simpleCrawler) next(ctx context.Context) (*models.DashboardWithStaleThu
 
 	authOpts := rendering.AuthOpts{
 		OrgID:   v.OrgId,
-		UserID:  r.opts.AuthOpts.UserID,
-		OrgRole: r.opts.AuthOpts.OrgRole,
+		UserID:  r.auth.GetUserId(v.OrgId),
+		OrgRole: r.auth.GetOrgRole(),
 	}
 
 	if renderingSession, ok := r.renderingSessionByOrgId[v.OrgId]; ok {
@@ -112,7 +113,7 @@ func (d byOrgId) Len() int           { return len(d) }
 func (d byOrgId) Less(i, j int) bool { return d[i].OrgId > d[j].OrgId }
 func (d byOrgId) Swap(i, j int)      { d[i], d[j] = d[j], d[i] }
 
-func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
+func (r *simpleCrawler) Run(ctx context.Context, auth CrawlerAuth, mode CrawlerMode, theme models.Theme, thumbnailKind models.ThumbnailKind) error {
 	res, err := r.renderService.HasCapability(rendering.ScalingDownImages)
 	if err != nil {
 		return err
@@ -150,8 +151,8 @@ func (r *simpleCrawler) Run(ctx context.Context, authOpts rendering.AuthOpts, mo
 
 	r.mode = mode
 	r.thumbnailKind = thumbnailKind
+	r.auth = auth
 	r.opts = rendering.Opts{
-		AuthOpts: authOpts,
 		TimeoutOpts: rendering.TimeoutOpts{
 			Timeout:                  20 * time.Second,
 			RequestTimeoutMultiplier: 3,

pkg/services/thumbs/crawler_auth.go

@@ -0,0 +1,41 @@
+package thumbs
+
+import (
+	"context"
+
+	"github.com/grafana/grafana/pkg/models"
+)
+
+type CrawlerAuthSetupService interface {
+	Setup(ctx context.Context) (CrawlerAuth, error)
+}
+
+func ProvideCrawlerAuthSetupService() *OSSCrawlerAuthSetupService {
+	return &OSSCrawlerAuthSetupService{}
+}
+
+type OSSCrawlerAuthSetupService struct{}
+
+type CrawlerAuth interface {
+	GetUserId(orgId int64) int64
+	GetOrgRole() models.RoleType
+}
+
+type staticCrawlerAuth struct {
+	userId  int64
+	orgRole models.RoleType
+}
+
+func (o *staticCrawlerAuth) GetOrgRole() models.RoleType {
+	return o.orgRole
+}
+
+func (o *staticCrawlerAuth) GetUserId(orgId int64) int64 {
+	return o.userId
+}
+
+func (o *OSSCrawlerAuthSetupService) Setup(ctx context.Context) (CrawlerAuth, error) {
+	// userId:0 and ROLE_ADMIN grants the crawler process permissions to view all dashboards in all folders & orgs
+	// the process doesn't and shouldn't actually need to edit/modify any resources from the UI
+	return &staticCrawlerAuth{userId: 0, orgRole: models.ROLE_ADMIN}, nil
+}

pkg/services/thumbs/models.go

@@ -5,7 +5,6 @@ import (
 	"time"
 
 	"github.com/grafana/grafana/pkg/models"
-	"github.com/grafana/grafana/pkg/services/rendering"
 )
 
 type CrawlerMode string
@@ -66,7 +65,7 @@ type dashboardPreviewsSetupConfig struct {
 type dashRenderer interface {
 
 	// Run Assumes you have already authenticated as admin.
-	Run(ctx context.Context, authOpts rendering.AuthOpts, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error
+	Run(ctx context.Context, auth CrawlerAuth, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) error
 
 	// Assumes you have already authenticated as admin.
 	Stop() (crawlStatus, error)

pkg/services/thumbs/service.go

@@ -56,22 +56,24 @@ type crawlerScheduleOptions struct {
 	maxCrawlDuration time.Duration
 	crawlerMode      CrawlerMode
 	thumbnailKind    models.ThumbnailKind
-	auth             rendering.AuthOpts
 	themes           []models.Theme
+	auth             CrawlerAuth
 }
 
-func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore) Service {
+func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockService *serverlock.ServerLockService, renderService rendering.Service, gl *live.GrafanaLive, store *sqlstore.SQLStore, authSetupService CrawlerAuthSetupService) Service {
 	if !features.IsEnabled(featuremgmt.FlagDashboardPreviews) {
 		return &dummyService{}
 	}
+	logger := log.New("thumbnails_service")
 
 	thumbnailRepo := newThumbnailRepo(store)
 
-	authOpts := rendering.AuthOpts{
-		OrgID:   0,
-		UserID:  0,
-		OrgRole: models.ROLE_ADMIN,
+	crawlerAuth, err := authSetupService.Setup(context.Background())
+	if err != nil {
+		logger.Error("Failed to setup auth for the dashboard previews crawler", "err", err)
+		return &dummyService{}
 	}
+
 	return &thumbService{
 		renderingService:           renderService,
 		renderer:                   newSimpleCrawler(renderService, gl, thumbnailRepo),
@@ -80,7 +82,7 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
 		features:                   features,
 		lockService:                lockService,
 		crawlLockServiceActionName: "dashboard-crawler",
-		log:                        log.New("thumbnails_service"),
+		log:                        logger,
 
 		scheduleOptions: crawlerScheduleOptions{
 			tickerInterval:   time.Hour,
@@ -89,7 +91,7 @@ func ProvideService(cfg *setting.Cfg, features featuremgmt.FeatureToggles, lockS
 			crawlerMode:      CrawlerModeThumbs,
 			thumbnailKind:    models.ThumbnailKindDefault,
 			themes:           []models.Theme{models.ThemeDark, models.ThemeLight},
-			auth:             authOpts,
+			auth:             crawlerAuth,
 		},
 	}
 }
@@ -377,7 +379,7 @@ func (hs *thumbService) runOnDemandCrawl(parentCtx context.Context, theme models
 	// wait for at least a minute after the last completed run
 	interval := time.Minute
 	err := hs.lockService.LockAndExecute(crawlerCtx, hs.crawlLockServiceActionName, interval, func(ctx context.Context) {
-		if err := hs.renderer.Run(crawlerCtx, authOpts, mode, theme, kind); err != nil {
+		if err := hs.renderer.Run(crawlerCtx, hs.scheduleOptions.auth, mode, theme, kind); err != nil {
 			hs.log.Error("On demand crawl error", "mode", mode, "theme", theme, "kind", kind, "userId", authOpts.UserID, "orgId", authOpts.OrgID, "orgRole", authOpts.OrgRole)
 		}
 	})