diff --git a/conf/defaults.ini b/conf/defaults.ini
index f0c2ccf52f7..44fbbf67930 100644
--- a/conf/defaults.ini
+++ b/conf/defaults.ini
@@ -599,6 +599,36 @@ max_attempts = 3
# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
min_interval_seconds = 1
+# Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as an duration. Ex 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_annotation_age =
+
+# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+max_annotations_to_keep =
+
+#################################### Annotations #########################
+
+[annotations.dashboard]
+# Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_age =
+
+# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+max_annotations_to_keep =
+
+[annotations.api]
+# API annotations means that the annotations have been created using the API without any
+# association with a dashboard.
+
+# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+max_age =
+
+# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+max_annotations_to_keep =
+
#################################### Explore #############################
[explore]
# Enable the Explore section
diff --git a/conf/sample.ini b/conf/sample.ini
index 9fecf0b06cc..4e415cc05db 100644
--- a/conf/sample.ini
+++ b/conf/sample.ini
@@ -591,6 +591,36 @@
# Makes it possible to enforce a minimal interval between evaluations, to reduce load on the backend
;min_interval_seconds = 1
+# Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_annotation_age =
+
+# Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+;max_annotations_to_keep =
+
+#################################### Annotations #########################
+
+[annotations.dashboard]
+# Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+# Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_age =
+
+# Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+;max_annotations_to_keep =
+
+[annotations.api]
+# API annotations means that the annotations have been created using the API without any
+# association with a dashboard.
+
+# Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+# This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+;max_age =
+
+# Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+;max_annotations_to_keep =
+
#################################### Explore #############################
[explore]
# Enable the Explore section
diff --git a/docs/sources/administration/configuration.md b/docs/sources/administration/configuration.md
index 3fa12227cf9..aa0a0ea2e39 100644
--- a/docs/sources/administration/configuration.md
+++ b/docs/sources/administration/configuration.md
@@ -976,6 +976,43 @@ Sets the minimum interval between rule evaluations. Default value is `1`.
> **Note.** This setting has precedence over each individual rule frequency. If a rule frequency is lower than this value, then this value is enforced.
+### max_annotation_age =
+
+Configures for how long alert annotations are stored. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep =
+
+Configures max number of alert annotations that Grafana stores. Default value is 0, which keeps all alert annotations.
+
+
+
+## [annotations.dashboard]
+
+Dashboard annotations means that annotations are associated with the dashboard they are created on.
+
+### max_age
+
+Configures how long dashboard annotations are stored. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep
+
+Configures max number of dashboard annotations that Grafana stores. Default value is 0, which keeps all dashboard annotations.
+
+## [annotations.api]
+
+API annotations means that the annotations have been created using the API without any association with a dashboard.
+
+### max_age
+
+Configures how long Grafana stores API annotations. Default is 0, which keeps them forever.
+This setting should be expressed as a duration. Examples: 6h (hours), 10d (days), 2w (weeks), 1M (month).
+
+### max_annotations_to_keep
+
+Configures max number of API annotations that Grafana keeps. Default value is 0, which keeps all API annotations.
+
## [explore]
diff --git a/pkg/services/annotations/annotations.go b/pkg/services/annotations/annotations.go
index 73706ea5850..ceb19d5f5cc 100644
--- a/pkg/services/annotations/annotations.go
+++ b/pkg/services/annotations/annotations.go
@@ -1,6 +1,11 @@
package annotations
-import "github.com/grafana/grafana/pkg/components/simplejson"
+import (
+ "context"
+
+ "github.com/grafana/grafana/pkg/components/simplejson"
+ "github.com/grafana/grafana/pkg/setting"
+)
type Repository interface {
Save(item *Item) error
@@ -9,6 +14,11 @@ type Repository interface {
Delete(params *DeleteParams) error
}
+// AnnotationCleaner is responsible for cleaning up old annotations
+type AnnotationCleaner interface {
+ CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error
+}
+
type ItemQuery struct {
OrgId int64 `json:"orgId"`
From int64 `json:"from"`
@@ -43,6 +53,15 @@ type DeleteParams struct {
}
var repositoryInstance Repository
+var cleanerInstance AnnotationCleaner
+
+func GetAnnotationCleaner() AnnotationCleaner {
+ return cleanerInstance
+}
+
+func SetAnnotationCleaner(rep AnnotationCleaner) {
+ cleanerInstance = rep
+}
func GetRepository() Repository {
return repositoryInstance
@@ -74,6 +93,10 @@ type Item struct {
Title string
}
+func (i Item) TableName() string {
+ return "annotation"
+}
+
type ItemDTO struct {
Id int64 `json:"id"`
AlertId int64 `json:"alertId"`
diff --git a/pkg/services/cleanup/cleanup.go b/pkg/services/cleanup/cleanup.go
index 3b61d0d9c51..f2eb19b6179 100644
--- a/pkg/services/cleanup/cleanup.go
+++ b/pkg/services/cleanup/cleanup.go
@@ -12,6 +12,7 @@ import (
"github.com/grafana/grafana/pkg/infra/serverlock"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/registry"
+ "github.com/grafana/grafana/pkg/services/annotations"
"github.com/grafana/grafana/pkg/setting"
)
@@ -37,9 +38,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error {
for {
select {
case <-ticker.C:
+ ctxWithTimeout, cancelFn := context.WithTimeout(ctx, time.Minute*9)
+ defer cancelFn()
+
srv.cleanUpTmpFiles()
srv.deleteExpiredSnapshots()
srv.deleteExpiredDashboardVersions()
+ srv.cleanUpOldAnnotations(ctxWithTimeout)
+
err := srv.ServerLockService.LockAndExecute(ctx, "delete old login attempts",
time.Minute*10, func() {
srv.deleteOldLoginAttempts()
@@ -53,6 +59,14 @@ func (srv *CleanUpService) Run(ctx context.Context) error {
}
}
+func (srv *CleanUpService) cleanUpOldAnnotations(ctx context.Context) {
+ cleaner := annotations.GetAnnotationCleaner()
+ err := cleaner.CleanAnnotations(ctx, srv.Cfg)
+ if err != nil {
+ srv.log.Error("failed to clean up old annotations", "error", err)
+ }
+}
+
func (srv *CleanUpService) cleanUpTmpFiles() {
if _, err := os.Stat(srv.Cfg.ImagesDir); os.IsNotExist(err) {
return
diff --git a/pkg/services/sqlstore/annotation_cleanup.go b/pkg/services/sqlstore/annotation_cleanup.go
new file mode 100644
index 00000000000..ad1ba15ece5
--- /dev/null
+++ b/pkg/services/sqlstore/annotation_cleanup.go
@@ -0,0 +1,87 @@
+package sqlstore
+
+import (
+ "context"
+ "fmt"
+ "time"
+
+ "github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/setting"
+)
+
+// AnnotationCleanupService is responseible for cleaning old annotations.
+type AnnotationCleanupService struct {
+ batchSize int64
+ log log.Logger
+}
+
+const (
+ alertAnnotationType = "alert_id <> 0"
+ dashboardAnnotationType = "dashboard_id <> 0 AND alert_id = 0"
+ apiAnnotationType = "alert_id = 0 AND dashboard_id = 0"
+)
+
+// CleanAnnotations deletes old annotations created by
+// alert rules, API requests and human made in the UI.
+func (acs *AnnotationCleanupService) CleanAnnotations(ctx context.Context, cfg *setting.Cfg) error {
+ err := acs.cleanAnnotations(ctx, cfg.AlertingAnnotationCleanupSetting, alertAnnotationType)
+ if err != nil {
+ return err
+ }
+
+ err = acs.cleanAnnotations(ctx, cfg.APIAnnotationCleanupSettings, apiAnnotationType)
+ if err != nil {
+ return err
+ }
+
+ return acs.cleanAnnotations(ctx, cfg.DashboardAnnotationCleanupSettings, dashboardAnnotationType)
+}
+
+func (acs *AnnotationCleanupService) cleanAnnotations(ctx context.Context, cfg setting.AnnotationCleanupSettings, annotationType string) error {
+ if cfg.MaxAge > 0 {
+ cutoffDate := time.Now().Add(-cfg.MaxAge).UnixNano() / int64(time.Millisecond)
+ deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s AND created < %v ORDER BY id DESC %s) a)`
+ sql := fmt.Sprintf(deleteQuery, annotationType, cutoffDate, dialect.Limit(acs.batchSize))
+
+ err := acs.executeUntilDoneOrCancelled(ctx, sql)
+ if err != nil {
+ return err
+ }
+ }
+
+ if cfg.MaxCount > 0 {
+ deleteQuery := `DELETE FROM annotation WHERE id IN (SELECT id FROM (SELECT id FROM annotation WHERE %s ORDER BY id DESC %s) a)`
+ sql := fmt.Sprintf(deleteQuery, annotationType, dialect.LimitOffset(acs.batchSize, cfg.MaxCount))
+ return acs.executeUntilDoneOrCancelled(ctx, sql)
+ }
+
+ return nil
+}
+
+func (acs *AnnotationCleanupService) executeUntilDoneOrCancelled(ctx context.Context, sql string) error {
+ for {
+ select {
+ case <-ctx.Done():
+ return ctx.Err()
+ default:
+ var affected int64
+ err := withDbSession(ctx, func(session *DBSession) error {
+ res, err := session.Exec(sql)
+ if err != nil {
+ return err
+ }
+
+ affected, err = res.RowsAffected()
+
+ return err
+ })
+ if err != nil {
+ return err
+ }
+
+ if affected == 0 {
+ return nil
+ }
+ }
+ }
+}
diff --git a/pkg/services/sqlstore/annotation_cleanup_test.go b/pkg/services/sqlstore/annotation_cleanup_test.go
new file mode 100644
index 00000000000..16fe7053f1f
--- /dev/null
+++ b/pkg/services/sqlstore/annotation_cleanup_test.go
@@ -0,0 +1,194 @@
+package sqlstore
+
+import (
+ "context"
+ "testing"
+ "time"
+
+ "github.com/grafana/grafana/pkg/infra/log"
+ "github.com/grafana/grafana/pkg/services/annotations"
+ "github.com/grafana/grafana/pkg/setting"
+ "github.com/stretchr/testify/require"
+)
+
+func TestAnnotationCleanUp(t *testing.T) {
+ fakeSQL := InitTestDB(t)
+
+ t.Cleanup(func() {
+ _ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error {
+ _, err := session.Exec("DELETE FROM annotation")
+ require.Nil(t, err, "cleaning up all annotations should not cause problems")
+ return err
+ })
+ })
+
+ createTestAnnotations(t, fakeSQL, 21, 6)
+ assertAnnotationCount(t, fakeSQL, "", 21)
+
+ tests := []struct {
+ name string
+ cfg *setting.Cfg
+ alertAnnotationCount int64
+ dashboardAnnotationCount int64
+ APIAnnotationCount int64
+ }{
+ {
+ name: "default settings should not delete any annotations",
+ cfg: &setting.Cfg{
+ AlertingAnnotationCleanupSetting: settingsFn(0, 0),
+ DashboardAnnotationCleanupSettings: settingsFn(0, 0),
+ APIAnnotationCleanupSettings: settingsFn(0, 0),
+ },
+ alertAnnotationCount: 7,
+ dashboardAnnotationCount: 7,
+ APIAnnotationCount: 7,
+ },
+ {
+ name: "should remove annotations created before cut off point",
+ cfg: &setting.Cfg{
+ AlertingAnnotationCleanupSetting: settingsFn(time.Hour*48, 0),
+ DashboardAnnotationCleanupSettings: settingsFn(time.Hour*48, 0),
+ APIAnnotationCleanupSettings: settingsFn(time.Hour*48, 0),
+ },
+ alertAnnotationCount: 5,
+ dashboardAnnotationCount: 5,
+ APIAnnotationCount: 5,
+ },
+ {
+ name: "should only keep three annotations",
+ cfg: &setting.Cfg{
+ AlertingAnnotationCleanupSetting: settingsFn(0, 3),
+ DashboardAnnotationCleanupSettings: settingsFn(0, 3),
+ APIAnnotationCleanupSettings: settingsFn(0, 3),
+ },
+ alertAnnotationCount: 3,
+ dashboardAnnotationCount: 3,
+ APIAnnotationCount: 3,
+ },
+ {
+ name: "running the max count delete again should not remove any annotations",
+ cfg: &setting.Cfg{
+ AlertingAnnotationCleanupSetting: settingsFn(0, 3),
+ DashboardAnnotationCleanupSettings: settingsFn(0, 3),
+ APIAnnotationCleanupSettings: settingsFn(0, 3),
+ },
+ alertAnnotationCount: 3,
+ dashboardAnnotationCount: 3,
+ APIAnnotationCount: 3,
+ },
+ }
+
+ for _, test := range tests {
+ t.Run(test.name, func(t *testing.T) {
+ cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")}
+ err := cleaner.CleanAnnotations(context.Background(), test.cfg)
+ require.NoError(t, err)
+
+ assertAnnotationCount(t, fakeSQL, alertAnnotationType, test.alertAnnotationCount)
+ assertAnnotationCount(t, fakeSQL, dashboardAnnotationType, test.dashboardAnnotationCount)
+ assertAnnotationCount(t, fakeSQL, apiAnnotationType, test.APIAnnotationCount)
+ })
+ }
+}
+
+func TestOldAnnotationsAreDeletedFirst(t *testing.T) {
+ fakeSQL := InitTestDB(t)
+
+ t.Cleanup(func() {
+ _ = fakeSQL.WithDbSession(context.Background(), func(session *DBSession) error {
+ _, err := session.Exec("DELETE FROM annotation")
+ require.Nil(t, err, "cleaning up all annotations should not cause problems")
+ return err
+ })
+ })
+
+ //create some test annotations
+ a := annotations.Item{
+ DashboardId: 1,
+ OrgId: 1,
+ UserId: 1,
+ PanelId: 1,
+ AlertId: 10,
+ Text: "",
+ Created: time.Now().AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond),
+ }
+
+ session := fakeSQL.NewSession()
+ defer session.Close()
+
+ _, err := session.Insert(a)
+ require.NoError(t, err, "cannot insert annotation")
+ _, err = session.Insert(a)
+ require.NoError(t, err, "cannot insert annotation")
+
+ a.AlertId = 20
+ _, err = session.Insert(a)
+ require.NoError(t, err, "cannot insert annotation")
+
+ // run the clean up task to keep one annotation.
+ cleaner := &AnnotationCleanupService{batchSize: 1, log: log.New("test-logger")}
+ err = cleaner.cleanAnnotations(context.Background(), setting.AnnotationCleanupSettings{MaxCount: 1}, alertAnnotationType)
+ require.NoError(t, err)
+
+ // assert that the last annotations were kept
+ countNew, err := session.Where("alert_id = 20").Count(&annotations.Item{})
+ require.NoError(t, err)
+ require.Equal(t, int64(1), countNew, "the last annotations should be kept")
+
+ countOld, err := session.Where("alert_id = 10").Count(&annotations.Item{})
+ require.NoError(t, err)
+ require.Equal(t, int64(0), countOld, "the two first annotations should have been deleted.")
+}
+
+func assertAnnotationCount(t *testing.T, fakeSQL *SqlStore, sql string, expectedCount int64) {
+ t.Helper()
+
+ session := fakeSQL.NewSession()
+ defer session.Close()
+ count, err := session.Where(sql).Count(&annotations.Item{})
+ require.NoError(t, err)
+ require.Equal(t, expectedCount, count)
+}
+
+func createTestAnnotations(t *testing.T, sqlstore *SqlStore, expectedCount int, oldAnnotations int) {
+ t.Helper()
+
+ cutoffDate := time.Now()
+
+ for i := 0; i < expectedCount; i++ {
+ a := &annotations.Item{
+ DashboardId: 1,
+ OrgId: 1,
+ UserId: 1,
+ PanelId: 1,
+ Text: "",
+ }
+
+ // mark every third as an API annotation
+ // that doesnt belong to a dashboard
+ if i%3 == 1 {
+ a.DashboardId = 0
+ }
+
+ // mark every third annotation as an alert annotation
+ if i%3 == 0 {
+ a.AlertId = 10
+ a.DashboardId = 2
+ }
+
+ // create epoch as int annotations.go line 40
+ a.Created = cutoffDate.UnixNano() / int64(time.Millisecond)
+
+ // set a really old date for the first six annotations
+ if i < oldAnnotations {
+ a.Created = cutoffDate.AddDate(-10, 0, -10).UnixNano() / int64(time.Millisecond)
+ }
+
+ _, err := sqlstore.NewSession().Insert(a)
+ require.NoError(t, err, "should be able to save annotation", err)
+ }
+}
+
+func settingsFn(maxAge time.Duration, maxCount int64) setting.AnnotationCleanupSettings {
+ return setting.AnnotationCleanupSettings{MaxAge: maxAge, MaxCount: maxCount}
+}
diff --git a/pkg/services/sqlstore/sqlstore.go b/pkg/services/sqlstore/sqlstore.go
index 59efd4d6680..406d2e194db 100644
--- a/pkg/services/sqlstore/sqlstore.go
+++ b/pkg/services/sqlstore/sqlstore.go
@@ -96,6 +96,7 @@ func (ss *SqlStore) Init() error {
// Init repo instances
annotations.SetRepository(&SqlAnnotationRepo{})
+ annotations.SetAnnotationCleaner(&AnnotationCleanupService{batchSize: 100, log: log.New("annotationcleaner")})
ss.Bus.SetTransactionManager(ss)
// Register handlers
diff --git a/pkg/services/sqlstore/sqlutil/sqlutil.go b/pkg/services/sqlstore/sqlutil/sqlutil.go
index 7416bc801dd..98363bf1cba 100644
--- a/pkg/services/sqlstore/sqlutil/sqlutil.go
+++ b/pkg/services/sqlstore/sqlutil/sqlutil.go
@@ -11,6 +11,7 @@ type TestDB struct {
}
func Sqlite3TestDB() TestDB {
+ // To run all tests in a local test database, set ConnStr to "grafana_test.db"
return TestDB{
DriverName: "sqlite3",
ConnStr: ":memory:",
diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go
index fe88656c6c7..dfc3f5d7744 100644
--- a/pkg/setting/setting.go
+++ b/pkg/setting/setting.go
@@ -19,6 +19,7 @@ import (
"github.com/go-macaron/session"
ini "gopkg.in/ini.v1"
+ "github.com/grafana/grafana/pkg/components/gtime"
"github.com/grafana/grafana/pkg/infra/log"
"github.com/grafana/grafana/pkg/util"
)
@@ -302,6 +303,11 @@ type Cfg struct {
FeatureToggles map[string]bool
AnonymousHideVersion bool
+
+ // Annotations
+ AlertingAnnotationCleanupSetting AnnotationCleanupSettings
+ DashboardAnnotationCleanupSettings AnnotationCleanupSettings
+ APIAnnotationCleanupSettings AnnotationCleanupSettings
}
// IsExpressionsEnabled returns whether the expressions feature is enabled.
@@ -396,6 +402,33 @@ func applyEnvVariableOverrides(file *ini.File) error {
return nil
}
+func (cfg *Cfg) readAnnotationSettings() {
+ dashboardAnnotation := cfg.Raw.Section("annotations.dashboard")
+ apiIAnnotation := cfg.Raw.Section("annotations.api")
+ alertingSection := cfg.Raw.Section("alerting")
+
+ var newAnnotationCleanupSettings = func(section *ini.Section, maxAgeField string) AnnotationCleanupSettings {
+ maxAge, err := gtime.ParseInterval(section.Key(maxAgeField).MustString(""))
+ if err != nil {
+ maxAge = 0
+ }
+
+ return AnnotationCleanupSettings{
+ MaxAge: maxAge,
+ MaxCount: section.Key("max_annotations_to_keep").MustInt64(0),
+ }
+ }
+
+ cfg.AlertingAnnotationCleanupSetting = newAnnotationCleanupSettings(alertingSection, "max_annotation_age")
+ cfg.DashboardAnnotationCleanupSettings = newAnnotationCleanupSettings(dashboardAnnotation, "max_age")
+ cfg.APIAnnotationCleanupSettings = newAnnotationCleanupSettings(apiIAnnotation, "max_age")
+}
+
+type AnnotationCleanupSettings struct {
+ MaxAge time.Duration
+ MaxCount int64
+}
+
func envKey(sectionName string, keyName string) string {
sN := strings.ToUpper(strings.Replace(sectionName, ".", "_", -1))
sN = strings.Replace(sN, "-", "_", -1)
@@ -758,6 +791,7 @@ func (cfg *Cfg) Load(args *CommandLineArgs) error {
cfg.readSessionConfig()
cfg.readSmtpSettings()
cfg.readQuotaSettings()
+ cfg.readAnnotationSettings()
if VerifyEmailEnabled && !cfg.Smtp.Enabled {
log.Warnf("require_email_validation is enabled but smtp is disabled")