From b2736ac1fea71ebeb4255cf71fb488b2c117114d Mon Sep 17 00:00:00 2001 From: Artur Wierzbicki Date: Mon, 18 Jul 2022 15:24:39 +0400 Subject: [PATCH] Storage: limit the number of uploaded files (#50796) * #50608: sql file upload quotas * rename `files_in_sql` to `file` * merge conflict --- .betterer.results | 7 ++----- conf/defaults.ini | 3 +++ pkg/services/quota/quotaimpl/quota.go | 5 +++++ pkg/services/sqlstore/quota.go | 15 ++++++++++++++- pkg/services/store/http.go | 19 ++++++++++++++++--- pkg/setting/setting_quota.go | 2 ++ 6 files changed, 42 insertions(+), 9 deletions(-) diff --git a/.betterer.results b/.betterer.results index 343a79f5634..0ae53be6813 100644 --- a/.betterer.results +++ b/.betterer.results @@ -5517,11 +5517,8 @@ exports[`better eslint`] = { [0, 0, 0, "Unexpected any. Specify a different type.", "20"], [0, 0, 0, "Do not use any type assertions.", "21"], [0, 0, 0, "Unexpected any. Specify a different type.", "22"], - [0, 0, 0, "Do not use any type assertions.", "23"], - [0, 0, 0, "Do not use any type assertions.", "24"], - [0, 0, 0, "Do not use any type assertions.", "25"], - [0, 0, 0, "Unexpected any. Specify a different type.", "26"], - [0, 0, 0, "Do not use any type assertions.", "27"] + [0, 0, 0, "Unexpected any. Specify a different type.", "23"], + [0, 0, 0, "Do not use any type assertions.", "24"] ], "public/app/features/plugins/hooks/tests/useImportAppPlugin.test.tsx:5381": [ [0, 0, 0, "Unexpected any. Specify a different type.", "0"], diff --git a/conf/defaults.ini b/conf/defaults.ini index 02093ab900c..ad5affa2e7a 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -806,6 +806,9 @@ global_session = -1 # global limit of alerts global_alert_rule = -1 +# global limit of files uploaded to the SQL DB +global_file = 1000 + #################################### Unified Alerting #################### [unified_alerting] # Enable the Unified Alerting sub-system and interface. When enabled we'll migrate all of your alert rules and notification channels to the new system. New alert rules will be created and your notification channels will be converted into an Alertmanager configuration. Previous data is preserved to enable backwards compatibility but new data is removed when switching. When this configuration section and flag are not defined, the state is defined at runtime. See the documentation for more details. diff --git a/pkg/services/quota/quotaimpl/quota.go b/pkg/services/quota/quotaimpl/quota.go index f16cecaae74..fc85fc6d97f 100644 --- a/pkg/services/quota/quotaimpl/quota.go +++ b/pkg/services/quota/quotaimpl/quota.go @@ -190,6 +190,11 @@ func (s *Service) getQuotaScopes(target string) ([]models.QuotaScope, error) { models.QuotaScope{Name: "org", Target: target, DefaultLimit: s.Cfg.Quota.Org.AlertRule}, ) return scopes, nil + case "file": + scopes = append(scopes, + models.QuotaScope{Name: "global", Target: target, DefaultLimit: s.Cfg.Quota.Global.File}, + ) + return scopes, nil default: return scopes, quota.ErrInvalidQuotaTarget } diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index 3ed4428fa71..6f7fc67f28b 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -12,6 +12,7 @@ import ( const ( alertRuleTarget = "alert_rule" dashboardTarget = "dashboard" + filesTarget = "file" ) type targetCount struct { @@ -255,7 +256,19 @@ func (ss *SQLStore) UpdateUserQuota(ctx context.Context, cmd *models.UpdateUserQ func (ss *SQLStore) GetGlobalQuotaByTarget(ctx context.Context, query *models.GetGlobalQuotaByTargetQuery) error { return ss.WithDbSession(ctx, func(sess *DBSession) error { var used int64 - if query.Target != alertRuleTarget || query.UnifiedAlertingEnabled { + + if query.Target == filesTarget { + // get quota used. + rawSQL := fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", + dialect.Quote("file")) + + notFolderCondition := fmt.Sprintf(" WHERE path NOT LIKE '%s'", "%/") + resp := make([]*targetCount, 0) + if err := sess.SQL(rawSQL + notFolderCondition).Find(&resp); err != nil { + return err + } + used = resp[0].Count + } else if query.Target != alertRuleTarget || query.UnifiedAlertingEnabled { // get quota used. rawSQL := fmt.Sprintf("SELECT COUNT(*) AS count FROM %s", dialect.Quote(query.Target)) diff --git a/pkg/services/store/http.go b/pkg/services/store/http.go index 2523e500bc0..49ea3199481 100644 --- a/pkg/services/store/http.go +++ b/pkg/services/store/http.go @@ -11,6 +11,7 @@ import ( "github.com/grafana/grafana/pkg/api/response" "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/services/quota" "github.com/grafana/grafana/pkg/util" "github.com/grafana/grafana/pkg/web" ) @@ -26,12 +27,14 @@ type HTTPStorageService interface { } type httpStorage struct { - store StorageService + store StorageService + quotaService quota.Service } -func ProvideHTTPService(store StorageService) HTTPStorageService { +func ProvideHTTPService(store StorageService, quotaService quota.Service) HTTPStorageService { return &httpStorage{ - store: store, + store: store, + quotaService: quotaService, } } @@ -58,6 +61,16 @@ func UploadErrorToStatusCode(err error) int { } func (s *httpStorage) Upload(c *models.ReqContext) response.Response { + // assumes we are only uploading to the SQL database - TODO: refactor once we introduce object stores + quotaReached, err := s.quotaService.CheckQuotaReached(c.Req.Context(), "file", nil) + if err != nil { + return response.Error(500, "Internal server error", err) + } + + if quotaReached { + return response.Error(400, "File quota reached", errors.New("file quota reached")) + } + type rspInfo struct { Message string `json:"message,omitempty"` Path string `json:"path,omitempty"` diff --git a/pkg/setting/setting_quota.go b/pkg/setting/setting_quota.go index 139189f7e84..b3cd6d01115 100644 --- a/pkg/setting/setting_quota.go +++ b/pkg/setting/setting_quota.go @@ -24,6 +24,7 @@ type GlobalQuota struct { ApiKey int64 `target:"api_key"` Session int64 `target:"-"` AlertRule int64 `target:"alert_rule"` + File int64 `target:"file"` } func (q *OrgQuota) ToMap() map[string]int64 { @@ -94,6 +95,7 @@ func (cfg *Cfg) readQuotaSettings() { Dashboard: quota.Key("global_dashboard").MustInt64(-1), ApiKey: quota.Key("global_api_key").MustInt64(-1), Session: quota.Key("global_session").MustInt64(-1), + File: quota.Key("global_file").MustInt64(-1), AlertRule: alertGlobalQuota, }