mirror of https://github.com/grafana/grafana
Dash previews: populate crawler queue from SQL query (#44083)
* add SQL migrations * dashboard previews from sql: poc * added todos * refactor: use the same enums where possible * use useEffect, always return json * added todo * refactor + delete files after use * refactor + fix manual thumbnail upload * refactor: move all interactions with sqlStore to thumbnail repo * refactor: remove file operations in thumb crawler/service * refactor: fix dashboard_thumbs sql store * refactor: extracted thumbnail fetching/updating to a hook * refactor: store thumbnails in redux store * refactor: store thumbnails in redux store * refactor: private'd repo methods * removed redux storage, saving images as blobs * allow for configurable rendering timeouts * added 1) query for dashboards with stale thumbnails, 2) command for marking thumbnails as stale * use sql-based queue in crawler * ui for marking thumbnails as stale * replaced `stale` boolean prop with `state` enum * introduce rendering session * compilation errors * fix crawler stop button * rename thumbnail state frozen to locked * #44449: fix merge conflicts * #44449: remove thumb methods from `Store` interface * #44449: clean filepath, defer file closing * #44449: fix rendering.Theme cyclic import * #44449: linting * #44449: linting * #44449: mutex'd crawlerStatus access * #44449: added integration tests for `sqlstore.dashboard_thumbs` * #44449: added comments to explain the `ThumbnailState` enum * #44449: use os.ReadFile rather then os.Open * #44449: always enable dashboardPreviews feature during integration tests * #44449: remove sleep time, adjust number of threads * #44449: review fix: add `orgId` to `DashboardThumbnailMeta` * #44449: review fix: automatic parsing of thumbnailState * #44449: lint fixes * #44449: review fix: prefer `WithDbSession` over `WithTransactionalDbSession` * #44449: review fix: add a comment explaining source of the filepath * #44449: review fix: added filepath validation * #44449: review fixes https://github.com/grafana/grafana/pull/45063/files @fzambia Co-authored-by: Ryan McKinley <ryantxu@gmail.com> Co-authored-by: Alexander Emelin <frvzmb@gmail.com>pull/45134/head
Artur Wierzbicki
4 years ago
committed by
GitHub
parent
4e3a72fc2a
commit
a025109647
@ -0,0 +1,137 @@ |
||||
package models |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"errors" |
||||
"fmt" |
||||
"time" |
||||
) |
||||
|
||||
type ThumbnailKind string |
||||
type ThumbnailState string |
||||
type CrawlerMode string |
||||
|
||||
const ( |
||||
// ThumbnailKindDefault is a small 320x240 preview
|
||||
ThumbnailKindDefault ThumbnailKind = "thumb" |
||||
|
||||
// unsupported for now
|
||||
// - ThumbnailKindLarge ThumbnailKind = "large"
|
||||
// - ThumbnailKindTall ThumbnailKind = "tall"
|
||||
) |
||||
|
||||
const ( |
||||
// ThumbnailStateDefault is the initial state for all thumbnails. Thumbnails in the "default" state will be considered stale,
|
||||
// and thus refreshed by the crawler, if the dashboard version from the time of taking the thumbnail is different from the current dashboard version
|
||||
ThumbnailStateDefault ThumbnailState = "default" |
||||
|
||||
// ThumbnailStateStale is a manually assigned state. Thumbnails in the "stale" state will be refreshed on the next crawler run
|
||||
ThumbnailStateStale ThumbnailState = "stale" |
||||
|
||||
// ThumbnailStateLocked is a manually assigned state. Thumbnails in the "locked" state will not be refreshed by the crawler as long as they remain in the "locked" state.
|
||||
ThumbnailStateLocked ThumbnailState = "locked" |
||||
) |
||||
|
||||
func (s ThumbnailState) IsValid() bool { |
||||
return s == ThumbnailStateDefault || s == ThumbnailStateStale || s == ThumbnailStateLocked |
||||
} |
||||
|
||||
func (s *ThumbnailState) UnmarshalJSON(data []byte) error { |
||||
var str string |
||||
err := json.Unmarshal(data, &str) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
*s = ThumbnailState(str) |
||||
|
||||
if !s.IsValid() { |
||||
if (*s) != "" { |
||||
return fmt.Errorf("JSON validation error: invalid thumbnail state value: %s", *s) |
||||
} |
||||
|
||||
*s = ThumbnailStateDefault |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
// IsKnownThumbnailKind checks if the value is supported
|
||||
func (p ThumbnailKind) IsKnownThumbnailKind() bool { |
||||
switch p { |
||||
case |
||||
ThumbnailKindDefault: |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func ParseThumbnailKind(str string) (ThumbnailKind, error) { |
||||
switch str { |
||||
case string(ThumbnailKindDefault): |
||||
return ThumbnailKindDefault, nil |
||||
} |
||||
return ThumbnailKindDefault, errors.New("unknown thumbnail kind " + str) |
||||
} |
||||
|
||||
// A DashboardThumbnail includes all metadata for a dashboard thumbnail
|
||||
type DashboardThumbnail struct { |
||||
Id int64 `json:"id"` |
||||
DashboardId int64 `json:"dashboardId"` |
||||
DashboardVersion int `json:"dashboardVersion"` |
||||
State ThumbnailState `json:"state"` |
||||
PanelId int64 `json:"panelId,omitempty"` |
||||
Kind ThumbnailKind `json:"kind"` |
||||
Theme Theme `json:"theme"` |
||||
Image []byte `json:"image"` |
||||
MimeType string `json:"mimeType"` |
||||
Updated time.Time `json:"updated"` |
||||
} |
||||
|
||||
//
|
||||
// Commands
|
||||
//
|
||||
|
||||
// DashboardThumbnailMeta uniquely identifies a thumbnail; a natural key
|
||||
type DashboardThumbnailMeta struct { |
||||
DashboardUID string |
||||
OrgId int64 |
||||
PanelID int64 |
||||
Kind ThumbnailKind |
||||
Theme Theme |
||||
} |
||||
|
||||
type GetDashboardThumbnailCommand struct { |
||||
DashboardThumbnailMeta |
||||
|
||||
Result *DashboardThumbnail |
||||
} |
||||
|
||||
const DashboardVersionForManualThumbnailUpload = -1 |
||||
|
||||
type DashboardWithStaleThumbnail struct { |
||||
Id int64 |
||||
OrgId int64 |
||||
Uid string |
||||
Version int |
||||
Slug string |
||||
} |
||||
|
||||
type FindDashboardsWithStaleThumbnailsCommand struct { |
||||
IncludeManuallyUploadedThumbnails bool |
||||
Result []*DashboardWithStaleThumbnail |
||||
} |
||||
|
||||
type SaveDashboardThumbnailCommand struct { |
||||
DashboardThumbnailMeta |
||||
DashboardVersion int |
||||
Image []byte |
||||
MimeType string |
||||
|
||||
Result *DashboardThumbnail |
||||
} |
||||
|
||||
type UpdateThumbnailStateCommand struct { |
||||
State ThumbnailState |
||||
DashboardThumbnailMeta |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
package models |
||||
|
||||
import "errors" |
||||
|
||||
type Theme string |
||||
|
||||
const ( |
||||
ThemeLight Theme = "light" |
||||
ThemeDark Theme = "dark" |
||||
) |
||||
|
||||
func ParseTheme(str string) (Theme, error) { |
||||
switch str { |
||||
case string(ThemeLight): |
||||
return ThemeLight, nil |
||||
case string(ThemeDark): |
||||
return ThemeDark, nil |
||||
} |
||||
return ThemeDark, errors.New("unknown theme " + str) |
||||
} |
||||
@ -0,0 +1,167 @@ |
||||
package sqlstore |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/models" |
||||
) |
||||
|
||||
func (ss *SQLStore) GetThumbnail(ctx context.Context, query *models.GetDashboardThumbnailCommand) (*models.DashboardThumbnail, error) { |
||||
err := ss.WithDbSession(ctx, func(sess *DBSession) error { |
||||
result, err := findThumbnailByMeta(sess, query.DashboardThumbnailMeta) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
query.Result = result |
||||
return nil |
||||
}) |
||||
|
||||
return query.Result, err |
||||
} |
||||
|
||||
func (ss *SQLStore) SaveThumbnail(ctx context.Context, cmd *models.SaveDashboardThumbnailCommand) (*models.DashboardThumbnail, error) { |
||||
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { |
||||
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta) |
||||
|
||||
if err != nil && !errors.Is(err, models.ErrDashboardThumbnailNotFound) { |
||||
return err |
||||
} |
||||
|
||||
if existing != nil { |
||||
existing.Image = cmd.Image |
||||
existing.MimeType = cmd.MimeType |
||||
existing.Updated = time.Now() |
||||
existing.DashboardVersion = cmd.DashboardVersion |
||||
existing.State = models.ThumbnailStateDefault |
||||
_, err = sess.ID(existing.Id).Update(existing) |
||||
cmd.Result = existing |
||||
return err |
||||
} |
||||
|
||||
thumb := &models.DashboardThumbnail{} |
||||
|
||||
dash, err := findDashboardIdByThumbMeta(sess, cmd.DashboardThumbnailMeta) |
||||
|
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
thumb.Updated = time.Now() |
||||
thumb.Theme = cmd.Theme |
||||
thumb.Kind = cmd.Kind |
||||
thumb.Image = cmd.Image |
||||
thumb.MimeType = cmd.MimeType |
||||
thumb.DashboardId = dash.Id |
||||
thumb.DashboardVersion = cmd.DashboardVersion |
||||
thumb.State = models.ThumbnailStateDefault |
||||
thumb.PanelId = cmd.PanelID |
||||
_, err = sess.Insert(thumb) |
||||
cmd.Result = thumb |
||||
return err |
||||
}) |
||||
|
||||
return cmd.Result, err |
||||
} |
||||
|
||||
func (ss *SQLStore) UpdateThumbnailState(ctx context.Context, cmd *models.UpdateThumbnailStateCommand) error { |
||||
err := ss.WithTransactionalDbSession(ctx, func(sess *DBSession) error { |
||||
existing, err := findThumbnailByMeta(sess, cmd.DashboardThumbnailMeta) |
||||
|
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
existing.State = cmd.State |
||||
_, err = sess.ID(existing.Id).Update(existing) |
||||
return err |
||||
}) |
||||
|
||||
return err |
||||
} |
||||
|
||||
func (ss *SQLStore) FindDashboardsWithStaleThumbnails(ctx context.Context, cmd *models.FindDashboardsWithStaleThumbnailsCommand) ([]*models.DashboardWithStaleThumbnail, error) { |
||||
err := ss.WithDbSession(ctx, func(sess *DBSession) error { |
||||
sess.Table("dashboard") |
||||
sess.Join("LEFT", "dashboard_thumbnail", "dashboard.id = dashboard_thumbnail.dashboard_id") |
||||
sess.Where("dashboard.is_folder = ?", dialect.BooleanStr(false)) |
||||
sess.Where("(dashboard.version != dashboard_thumbnail.dashboard_version "+ |
||||
"OR dashboard_thumbnail.state = ? "+ |
||||
"OR dashboard_thumbnail.id IS NULL)", models.ThumbnailStateStale) |
||||
|
||||
if !cmd.IncludeManuallyUploadedThumbnails { |
||||
sess.Where("(dashboard_thumbnail.id is not null AND dashboard_thumbnail.dashboard_version != ?) "+ |
||||
"OR dashboard_thumbnail.id is null "+ |
||||
"OR dashboard_thumbnail.state = ?", models.DashboardVersionForManualThumbnailUpload, models.ThumbnailStateStale) |
||||
} |
||||
|
||||
sess.Where("(dashboard_thumbnail.id IS NULL OR dashboard_thumbnail.state != ?)", models.ThumbnailStateLocked) |
||||
|
||||
sess.Cols("dashboard.id", |
||||
"dashboard.uid", |
||||
"dashboard.org_id", |
||||
"dashboard.version", |
||||
"dashboard.slug") |
||||
|
||||
var dashboards = make([]*models.DashboardWithStaleThumbnail, 0) |
||||
err := sess.Find(&dashboards) |
||||
|
||||
if err != nil { |
||||
return err |
||||
} |
||||
cmd.Result = dashboards |
||||
return err |
||||
}) |
||||
|
||||
return cmd.Result, err |
||||
} |
||||
|
||||
func findThumbnailByMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) { |
||||
result := &models.DashboardThumbnail{} |
||||
|
||||
sess.Table("dashboard_thumbnail") |
||||
sess.Join("INNER", "dashboard", "dashboard.id = dashboard_thumbnail.dashboard_id") |
||||
sess.Where("dashboard.uid = ? AND dashboard.org_id = ? AND panel_id = ? AND kind = ? AND theme = ?", meta.DashboardUID, meta.OrgId, meta.PanelID, meta.Kind, meta.Theme) |
||||
sess.Cols("dashboard_thumbnail.id", |
||||
"dashboard_thumbnail.dashboard_id", |
||||
"dashboard_thumbnail.panel_id", |
||||
"dashboard_thumbnail.image", |
||||
"dashboard_thumbnail.dashboard_version", |
||||
"dashboard_thumbnail.state", |
||||
"dashboard_thumbnail.kind", |
||||
"dashboard_thumbnail.mime_type", |
||||
"dashboard_thumbnail.theme", |
||||
"dashboard_thumbnail.updated") |
||||
exists, err := sess.Get(result) |
||||
|
||||
if !exists { |
||||
return nil, models.ErrDashboardThumbnailNotFound |
||||
} |
||||
|
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
return result, nil |
||||
} |
||||
|
||||
type dash struct { |
||||
Id int64 |
||||
} |
||||
|
||||
func findDashboardIdByThumbMeta(sess *DBSession, meta models.DashboardThumbnailMeta) (*dash, error) { |
||||
result := &dash{} |
||||
|
||||
sess.Table("dashboard").Where("dashboard.uid = ? AND dashboard.org_id = ?", meta.DashboardUID, meta.OrgId).Cols("id") |
||||
exists, err := sess.Get(result) |
||||
|
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
if !exists { |
||||
return nil, models.ErrDashboardNotFound |
||||
} |
||||
|
||||
return result, err |
||||
} |
||||
@ -0,0 +1,216 @@ |
||||
//go:build integration
|
||||
// +build integration
|
||||
|
||||
package sqlstore |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestSqlStorage(t *testing.T) { |
||||
|
||||
var sqlStore *SQLStore |
||||
var savedFolder *models.Dashboard |
||||
|
||||
setup := func() { |
||||
sqlStore = InitTestDB(t) |
||||
savedFolder = insertTestDashboard(t, sqlStore, "1 test dash folder", 1, 0, true, "prod", "webapp") |
||||
} |
||||
|
||||
t.Run("Should insert dashboard in default state", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId) |
||||
|
||||
require.Positive(t, thumb.Id) |
||||
require.Equal(t, models.ThumbnailStateDefault, thumb.State) |
||||
require.Equal(t, dash.Version, thumb.DashboardVersion) |
||||
}) |
||||
|
||||
t.Run("Should be able to update the thumbnail", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
thumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId) |
||||
|
||||
insertedThumbnailId := thumb.Id |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version+1) |
||||
|
||||
updatedThumb := getThumbnail(t, sqlStore, dash.Uid, dash.OrgId) |
||||
require.Equal(t, insertedThumbnailId, updatedThumb.Id) |
||||
require.Equal(t, dash.Version+1, updatedThumb.DashboardVersion) |
||||
}) |
||||
|
||||
t.Run("Should return empty array if all dashboards have thumbnails", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
|
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 0) |
||||
}) |
||||
|
||||
t.Run("Should return dashboards with thumbnails marked as stale", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 1) |
||||
require.Equal(t, dash.Id, res[0].Id) |
||||
}) |
||||
|
||||
t.Run("Should not return dashboards with updated thumbnails that had been marked as stale", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateStale) |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 0) |
||||
}) |
||||
|
||||
t.Run("Should find dashboards without thumbnails", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 1) |
||||
require.Equal(t, dash.Id, res[0].Id) |
||||
}) |
||||
|
||||
t.Run("Should find dashboards with outdated thumbnails", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
|
||||
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{ |
||||
"tags": "different-tag", |
||||
}) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 1) |
||||
require.Equal(t, dash.Id, res[0].Id) |
||||
}) |
||||
|
||||
t.Run("Should not return dashboards with locked thumbnails even if they are outdated", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, dash.Version) |
||||
updateThumbnailState(t, sqlStore, dash.Uid, dash.OrgId, models.ThumbnailStateLocked) |
||||
|
||||
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{ |
||||
"tags": "different-tag", |
||||
}) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 0) |
||||
}) |
||||
|
||||
t.Run("Should not return dashboards with manually uploaded thumbnails by default", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload) |
||||
|
||||
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{ |
||||
"tags": "different-tag", |
||||
}) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 0) |
||||
}) |
||||
|
||||
t.Run("Should return dashboards with manually uploaded thumbnails if requested", func(t *testing.T) { |
||||
setup() |
||||
dash := insertTestDashboard(t, sqlStore, "test dash 23", 1, savedFolder.Id, false, "prod", "webapp") |
||||
upsertTestDashboardThumbnail(t, sqlStore, dash.Uid, dash.OrgId, models.DashboardVersionForManualThumbnailUpload) |
||||
|
||||
updateTestDashboard(t, sqlStore, dash, map[string]interface{}{ |
||||
"tags": "different-tag", |
||||
}) |
||||
|
||||
cmd := models.FindDashboardsWithStaleThumbnailsCommand{ |
||||
IncludeManuallyUploadedThumbnails: true, |
||||
} |
||||
res, err := sqlStore.FindDashboardsWithStaleThumbnails(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.Len(t, res, 1) |
||||
require.Equal(t, dash.Id, res[0].Id) |
||||
}) |
||||
} |
||||
|
||||
func getThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64) *models.DashboardThumbnail { |
||||
t.Helper() |
||||
cmd := models.GetDashboardThumbnailCommand{ |
||||
DashboardThumbnailMeta: models.DashboardThumbnailMeta{ |
||||
DashboardUID: dashboardUID, |
||||
OrgId: orgId, |
||||
PanelID: 0, |
||||
Kind: models.ThumbnailKindDefault, |
||||
Theme: models.ThemeDark, |
||||
}, |
||||
} |
||||
|
||||
thumb, err := sqlStore.GetThumbnail(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
return thumb |
||||
} |
||||
|
||||
func upsertTestDashboardThumbnail(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, dashboardVersion int) *models.DashboardThumbnail { |
||||
t.Helper() |
||||
cmd := models.SaveDashboardThumbnailCommand{ |
||||
DashboardThumbnailMeta: models.DashboardThumbnailMeta{ |
||||
DashboardUID: dashboardUID, |
||||
OrgId: orgId, |
||||
PanelID: 0, |
||||
Kind: models.ThumbnailKindDefault, |
||||
Theme: models.ThemeDark, |
||||
}, |
||||
DashboardVersion: dashboardVersion, |
||||
Image: make([]byte, 0), |
||||
MimeType: "image/png", |
||||
} |
||||
dash, err := sqlStore.SaveThumbnail(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, dash) |
||||
|
||||
return dash |
||||
} |
||||
|
||||
func updateThumbnailState(t *testing.T, sqlStore *SQLStore, dashboardUID string, orgId int64, state models.ThumbnailState) { |
||||
t.Helper() |
||||
cmd := models.UpdateThumbnailStateCommand{ |
||||
DashboardThumbnailMeta: models.DashboardThumbnailMeta{ |
||||
DashboardUID: dashboardUID, |
||||
OrgId: orgId, |
||||
PanelID: 0, |
||||
Kind: models.ThumbnailKindDefault, |
||||
Theme: models.ThemeDark, |
||||
}, |
||||
State: state, |
||||
} |
||||
err := sqlStore.UpdateThumbnailState(context.Background(), &cmd) |
||||
require.NoError(t, err) |
||||
} |
||||
@ -0,0 +1,27 @@ |
||||
package migrations |
||||
|
||||
import "github.com/grafana/grafana/pkg/services/sqlstore/migrator" |
||||
|
||||
func addDashboardThumbsMigrations(mg *migrator.Migrator) { |
||||
dashThumbs := migrator.Table{ |
||||
Name: "dashboard_thumbnail", |
||||
Columns: []*migrator.Column{ |
||||
{Name: "id", Type: migrator.DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, |
||||
{Name: "dashboard_id", Type: migrator.DB_BigInt, Nullable: false}, // can join with dashboard table
|
||||
{Name: "dashboard_version", Type: migrator.DB_Int, Nullable: false}, // screenshoted version of the dashboard
|
||||
{Name: "state", Type: migrator.DB_NVarchar, Length: 10, Nullable: false}, // stale | locked
|
||||
{Name: "panel_id", Type: migrator.DB_SmallInt, Nullable: false, Default: "0"}, // for panel thumbnails
|
||||
{Name: "image", Type: migrator.DB_MediumBlob, Nullable: false}, // image stored as blob. MediumBlob has a max limit of 16mb in MySQL
|
||||
{Name: "mime_type", Type: migrator.DB_NVarchar, Length: 255, Nullable: false}, // e.g. image/png, image/webp
|
||||
{Name: "kind", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // thumb | tall
|
||||
{Name: "theme", Type: migrator.DB_NVarchar, Length: 8, Nullable: false}, // light|dark
|
||||
{Name: "updated", Type: migrator.DB_DateTime, Nullable: false}, |
||||
}, |
||||
Indices: []*migrator.Index{ |
||||
{Cols: []string{"dashboard_id", "panel_id", "kind", "theme"}, Type: migrator.UniqueIndex}, |
||||
}, |
||||
} |
||||
|
||||
mg.AddMigration("create dashboard_thumbnail table", migrator.NewAddTableMigration(dashThumbs)) |
||||
mg.AddMigration("add unique indexes for dashboard_thumbnail", migrator.NewAddIndexMigration(dashThumbs, dashThumbs.Indices[0])) |
||||
} |
||||
@ -1,107 +1,73 @@ |
||||
package thumbs |
||||
|
||||
import ( |
||||
"context" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/grafana/grafana/pkg/services/rendering" |
||||
) |
||||
|
||||
type PreviewSize string |
||||
type CrawlerMode string |
||||
|
||||
const ( |
||||
// PreviewSizeThumb is a small 320x240 preview
|
||||
PreviewSizeThumb PreviewSize = "thumb" |
||||
|
||||
// PreviewSizeLarge is a large image 2000x1500
|
||||
PreviewSizeLarge PreviewSize = "large" |
||||
|
||||
// PreviewSizeLarge is a large image 512x????
|
||||
PreviewSizeTall PreviewSize = "tall" |
||||
|
||||
// CrawlerModeThumbs will create small thumbnails for everything
|
||||
// CrawlerModeThumbs will create small thumbnails for everything.
|
||||
CrawlerModeThumbs CrawlerMode = "thumbs" |
||||
|
||||
// CrawlerModeAnalytics will get full page results for everythign
|
||||
// CrawlerModeAnalytics will get full page results for everything.
|
||||
CrawlerModeAnalytics CrawlerMode = "analytics" |
||||
|
||||
// CrawlerModeMigrate will migrate all dashboards with old schema
|
||||
// CrawlerModeMigrate will migrate all dashboards with old schema.
|
||||
CrawlerModeMigrate CrawlerMode = "migrate" |
||||
) |
||||
|
||||
// IsKnownSize checks if the value is a standard size
|
||||
func (p PreviewSize) IsKnownSize() bool { |
||||
switch p { |
||||
case |
||||
PreviewSizeThumb, |
||||
PreviewSizeLarge, |
||||
PreviewSizeTall: |
||||
return true |
||||
} |
||||
return false |
||||
} |
||||
|
||||
func getPreviewSize(str string) (PreviewSize, bool) { |
||||
switch str { |
||||
case string(PreviewSizeThumb): |
||||
return PreviewSizeThumb, true |
||||
case string(PreviewSizeLarge): |
||||
return PreviewSizeLarge, true |
||||
case string(PreviewSizeTall): |
||||
return PreviewSizeTall, true |
||||
} |
||||
return PreviewSizeThumb, false |
||||
} |
||||
type crawlerState string |
||||
|
||||
func getTheme(str string) (rendering.Theme, bool) { |
||||
switch str { |
||||
case "light": |
||||
return rendering.ThemeLight, true |
||||
case "dark": |
||||
return rendering.ThemeDark, true |
||||
} |
||||
return rendering.ThemeDark, false |
||||
} |
||||
const ( |
||||
initializing crawlerState = "initializing" |
||||
running crawlerState = "running" |
||||
stopping crawlerState = "stopping" |
||||
stopped crawlerState = "stopped" |
||||
) |
||||
|
||||
type previewRequest struct { |
||||
OrgID int64 `json:"orgId"` |
||||
UID string `json:"uid"` |
||||
Size PreviewSize `json:"size"` |
||||
Theme rendering.Theme `json:"theme"` |
||||
} |
||||
|
||||
type previewResponse struct { |
||||
Code int `json:"code"` // 200 | 202
|
||||
Path string `json:"path"` // local file path to serve
|
||||
URL string `json:"url"` // redirect to this URL
|
||||
OrgID int64 `json:"orgId"` |
||||
UID string `json:"uid"` |
||||
Kind models.ThumbnailKind `json:"kind"` |
||||
Theme models.Theme `json:"theme"` |
||||
} |
||||
|
||||
type crawlCmd struct { |
||||
Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate
|
||||
Theme rendering.Theme `json:"theme"` // light | dark
|
||||
Mode CrawlerMode `json:"mode"` // thumbs | analytics | migrate
|
||||
Theme models.Theme `json:"theme"` // light | dark
|
||||
} |
||||
|
||||
type crawlStatus struct { |
||||
State string `json:"state"` |
||||
Started time.Time `json:"started,omitempty"` |
||||
Finished time.Time `json:"finished,omitempty"` |
||||
Complete int `json:"complete"` |
||||
Errors int `json:"errors"` |
||||
Queue int `json:"queue"` |
||||
Last time.Time `json:"last,omitempty"` |
||||
State crawlerState `json:"state"` |
||||
Started time.Time `json:"started,omitempty"` |
||||
Finished time.Time `json:"finished,omitempty"` |
||||
Complete int `json:"complete"` |
||||
Errors int `json:"errors"` |
||||
Queue int `json:"queue"` |
||||
Last time.Time `json:"last,omitempty"` |
||||
} |
||||
|
||||
type dashRenderer interface { |
||||
// Assumes you have already authenticated as admin
|
||||
GetPreview(req *previewRequest) *previewResponse |
||||
|
||||
// Assumes you have already authenticated as admin
|
||||
Start(c *models.ReqContext, mode CrawlerMode, theme rendering.Theme) (crawlStatus, error) |
||||
// Run Assumes you have already authenticated as admin.
|
||||
Start(c *models.ReqContext, mode CrawlerMode, theme models.Theme, kind models.ThumbnailKind) (crawlStatus, error) |
||||
|
||||
// Assumes you have already authenticated as admin
|
||||
// Assumes you have already authenticated as admin.
|
||||
Stop() (crawlStatus, error) |
||||
|
||||
// Assumes you have already authenticated as admin
|
||||
// Assumes you have already authenticated as admin.
|
||||
Status() (crawlStatus, error) |
||||
} |
||||
|
||||
type thumbnailRepo interface { |
||||
updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error |
||||
saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) |
||||
saveFromBytes(ctx context.Context, bytes []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) |
||||
getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) |
||||
findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) |
||||
} |
||||
|
||||
@ -0,0 +1,88 @@ |
||||
package thumbs |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"os" |
||||
"path/filepath" |
||||
"strings" |
||||
|
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/grafana/grafana/pkg/services/sqlstore" |
||||
) |
||||
|
||||
func newThumbnailRepo(store *sqlstore.SQLStore) thumbnailRepo { |
||||
repo := &sqlThumbnailRepository{ |
||||
store: store, |
||||
} |
||||
return repo |
||||
} |
||||
|
||||
type sqlThumbnailRepository struct { |
||||
store *sqlstore.SQLStore |
||||
} |
||||
|
||||
func (r *sqlThumbnailRepository) saveFromFile(ctx context.Context, filePath string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) { |
||||
// the filePath variable is never set by the user. it refers to a temporary file created either in
|
||||
// 1. thumbs/service.go, when user uploads a thumbnail
|
||||
// 2. the rendering service, when image-renderer returns a screenshot
|
||||
|
||||
if !filepath.IsAbs(filePath) { |
||||
tlog.Error("Received relative path", "dashboardUID", meta.DashboardUID, "err", filePath) |
||||
return 0, errors.New("relative paths are not supported") |
||||
} |
||||
|
||||
content, err := os.ReadFile(filepath.Clean(filePath)) |
||||
|
||||
if err != nil { |
||||
tlog.Error("error reading file", "dashboardUID", meta.DashboardUID, "err", err) |
||||
return 0, err |
||||
} |
||||
|
||||
return r.saveFromBytes(ctx, content, getMimeType(filePath), meta, dashboardVersion) |
||||
} |
||||
|
||||
func getMimeType(filePath string) string { |
||||
if strings.HasSuffix(filePath, ".webp") { |
||||
return "image/webp" |
||||
} |
||||
|
||||
return "image/png" |
||||
} |
||||
|
||||
func (r *sqlThumbnailRepository) saveFromBytes(ctx context.Context, content []byte, mimeType string, meta models.DashboardThumbnailMeta, dashboardVersion int) (int64, error) { |
||||
cmd := &models.SaveDashboardThumbnailCommand{ |
||||
DashboardThumbnailMeta: meta, |
||||
Image: content, |
||||
MimeType: mimeType, |
||||
DashboardVersion: dashboardVersion, |
||||
} |
||||
|
||||
_, err := r.store.SaveThumbnail(ctx, cmd) |
||||
if err != nil { |
||||
tlog.Error("error saving to the db", "dashboardUID", meta.DashboardUID, "err", err) |
||||
return 0, err |
||||
} |
||||
|
||||
return cmd.Result.Id, nil |
||||
} |
||||
|
||||
func (r *sqlThumbnailRepository) updateThumbnailState(ctx context.Context, state models.ThumbnailState, meta models.DashboardThumbnailMeta) error { |
||||
return r.store.UpdateThumbnailState(ctx, &models.UpdateThumbnailStateCommand{ |
||||
State: state, |
||||
DashboardThumbnailMeta: meta, |
||||
}) |
||||
} |
||||
|
||||
func (r *sqlThumbnailRepository) getThumbnail(ctx context.Context, meta models.DashboardThumbnailMeta) (*models.DashboardThumbnail, error) { |
||||
query := &models.GetDashboardThumbnailCommand{ |
||||
DashboardThumbnailMeta: meta, |
||||
} |
||||
return r.store.GetThumbnail(ctx, query) |
||||
} |
||||
|
||||
func (r *sqlThumbnailRepository) findDashboardsWithStaleThumbnails(ctx context.Context) ([]*models.DashboardWithStaleThumbnail, error) { |
||||
return r.store.FindDashboardsWithStaleThumbnails(ctx, &models.FindDashboardsWithStaleThumbnailsCommand{ |
||||
IncludeManuallyUploadedThumbnails: false, |
||||
}) |
||||
} |
||||
@ -1,14 +0,0 @@ |
||||
package thumbs |
||||
|
||||
import ( |
||||
"fmt" |
||||
"path/filepath" |
||||
) |
||||
|
||||
func getFilePath(root string, req *previewRequest) string { |
||||
ext := "webp" |
||||
if req.Size != PreviewSizeThumb { |
||||
ext = "png" |
||||
} |
||||
return filepath.Join(root, fmt.Sprintf("%s-%s-%s.%s", req.UID, req.Size, req.Theme, ext)) |
||||
} |
||||
Loading…
Reference in new issue