feat: purge old files and snapshots

closes #4087
closes #2172
pull/6137/head
bergquist 9 years ago
parent 1a32ab64b6
commit dbb7852f21
  1. 9
      conf/defaults.ini
  2. 6
      conf/sample.ini
  3. 3
      pkg/cmd/grafana-server/main.go
  4. 7
      pkg/models/timer.go
  5. 39
      pkg/services/backgroundtasks/background_tasks.go
  6. 38
      pkg/services/backgroundtasks/remove_tmp_images.go
  7. 27
      pkg/services/sqlstore/dashboard_snapshot.go
  8. 18
      pkg/setting/setting.go

@ -161,6 +161,12 @@ external_enabled = true
external_snapshot_url = https://snapshots-origin.raintank.io
external_snapshot_name = Publish to snapshot.raintank.io
# remove expired snapshot
snapshot_remove_expired = true
# remove snapshots after 90 days
snapshot_TTL_days = 90
#################################### Users ####################################
[users]
# disable user signup / registration
@ -267,6 +273,9 @@ from_address = admin@grafana.localhost
welcome_email_on_sign_up = false
templates_pattern = emails/*.html
[tmp.files]
rendered_image_ttl_days = 14
#################################### Logging ##########################
[log]
# Either "console", "file", "syslog". Default is console and file

@ -149,6 +149,12 @@ check_for_updates = true
;external_snapshot_url = https://snapshots-origin.raintank.io
;external_snapshot_name = Publish to snapshot.raintank.io
# remove expired snapshot
;snapshot_remove_expired = true
# remove snapshots after 90 days
;snapshot_TTL_days = 90
#################################### Users ####################################
[users]
# disable user signup / registration

@ -17,6 +17,7 @@ import (
"github.com/grafana/grafana/pkg/metrics"
"github.com/grafana/grafana/pkg/plugins"
alertingInit "github.com/grafana/grafana/pkg/services/alerting/init"
"github.com/grafana/grafana/pkg/services/backgroundtasks"
"github.com/grafana/grafana/pkg/services/eventpublisher"
"github.com/grafana/grafana/pkg/services/notifications"
"github.com/grafana/grafana/pkg/services/search"
@ -62,13 +63,13 @@ func main() {
writePIDFile()
initRuntime()
metrics.Init()
search.Init()
login.Init()
social.NewOAuthService()
eventpublisher.Init()
plugins.Init()
alertingInit.Init()
backgroundtasks.Init()
if err := notifications.Init(); err != nil {
log.Fatal(3, "Notification service failed to initialize", err)

@ -0,0 +1,7 @@
package models
import "time"
type HourCommand struct {
Time time.Time
}

@ -0,0 +1,39 @@
//"I want to be a cleaner, just like you," said Mathilda
//"Okay," replied Leon
package backgroundtasks
import (
"time"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
)
var (
tlog log.Logger = log.New("ticker")
)
func Init() {
go start()
}
func start() {
go cleanup(time.Now())
ticker := time.NewTicker(time.Hour * 1)
for {
select {
case tick := <-ticker.C:
go cleanup(tick)
}
}
}
func cleanup(now time.Time) {
err := bus.Publish(&models.HourCommand{Time: now})
if err != nil {
tlog.Error("Cleanup job failed", "error", err)
}
}

@ -0,0 +1,38 @@
package backgroundtasks
import (
"io/ioutil"
"os"
"path"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
)
func init() {
bus.AddEventListener(CleanTmpFiles)
}
func CleanTmpFiles(cmd *models.HourCommand) error {
files, err := ioutil.ReadDir(setting.ImagesDir)
var toDelete []os.FileInfo
for _, file := range files {
if file.ModTime().AddDate(0, 0, setting.RenderedImageTTLDays).Before(cmd.Time) {
toDelete = append(toDelete, file)
}
}
for _, file := range toDelete {
fullPath := path.Join(setting.ImagesDir, file.Name())
err := os.Remove(fullPath)
if err != nil {
return err
}
}
tlog.Debug("Found old rendered image to delete", "deleted", len(toDelete), "keept", len(files))
return err
}

@ -5,7 +5,9 @@ import (
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/bus"
"github.com/grafana/grafana/pkg/log"
m "github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/setting"
)
func init() {
@ -13,6 +15,31 @@ func init() {
bus.AddHandler("sql", GetDashboardSnapshot)
bus.AddHandler("sql", DeleteDashboardSnapshot)
bus.AddHandler("sql", SearchDashboardSnapshots)
bus.AddEventListener(DeleteExpiredSnapshots)
}
func DeleteExpiredSnapshots(cmd *m.HourCommand) error {
return inTransaction(func(sess *xorm.Session) error {
var expiredCount int64 = 0
var oldCount int64 = 0
if setting.SnapShotRemoveExpired {
deleteExpiredSql := "DELETE FROM dashboard_snapshot WHERE expires < ?"
expiredResponse, err := x.Exec(deleteExpiredSql, cmd.Time)
if err != nil {
return err
}
expiredCount, _ = expiredResponse.RowsAffected()
}
oldSnapshotsSql := "DELETE FROM dashboard_snapshot WHERE created < ?"
oldResponse, err := x.Exec(oldSnapshotsSql, cmd.Time.AddDate(0, 0, setting.SnapShotTTLDays*-1))
oldCount, _ = oldResponse.RowsAffected()
log.Debug2("Deleted old/expired snaphots", "to old", oldCount, "expired", expiredCount)
return err
})
}
func CreateDashboardSnapshot(cmd *m.CreateDashboardSnapshotCommand) error {

@ -78,9 +78,11 @@ var (
DataProxyWhiteList map[string]bool
// Snapshots
ExternalSnapshotUrl string
ExternalSnapshotName string
ExternalEnabled bool
ExternalSnapshotUrl string
ExternalSnapshotName string
ExternalEnabled bool
SnapShotTTLDays int
SnapShotRemoveExpired bool
// User settings
AllowUserSignUp bool
@ -118,8 +120,9 @@ var (
IsWindows bool
// PhantomJs Rendering
ImagesDir string
PhantomDir string
ImagesDir string
PhantomDir string
RenderedImageTTLDays int
// for logging purposes
configFiles []string
@ -495,6 +498,8 @@ func NewConfigContext(args *CommandLineArgs) error {
ExternalSnapshotUrl = snapshots.Key("external_snapshot_url").String()
ExternalSnapshotName = snapshots.Key("external_snapshot_name").String()
ExternalEnabled = snapshots.Key("external_enabled").MustBool(true)
SnapShotRemoveExpired = snapshots.Key("snapshot_remove_expired").MustBool(true)
SnapShotTTLDays = snapshots.Key("snapshot_TTL_days").MustInt(90)
// read data source proxy white list
DataProxyWhiteList = make(map[string]bool)
@ -535,6 +540,9 @@ func NewConfigContext(args *CommandLineArgs) error {
ImagesDir = filepath.Join(DataPath, "png")
PhantomDir = filepath.Join(HomePath, "vendor/phantomjs")
tmpFilesSection := Cfg.Section("tmp.files")
RenderedImageTTLDays = tmpFilesSection.Key("rendered_image_ttl_days").MustInt(14)
analytics := Cfg.Section("analytics")
ReportingEnabled = analytics.Key("reporting_enabled").MustBool(true)
CheckForUpdates = analytics.Key("check_for_updates").MustBool(true)

Loading…
Cancel
Save