From 9023171940e17c44072a86ee40705964473485ef Mon Sep 17 00:00:00 2001 From: woodsaj Date: Mon, 20 Jul 2015 20:51:27 +0800 Subject: [PATCH 01/16] inital backend suport for quotas. issue #321 Conflicts: conf/defaults.ini main.go pkg/services/sqlstore/migrations/migrations.go --- conf/defaults.ini | 7 +- main.go | 3 + pkg/api/api.go | 2 + pkg/api/quota.go | 26 ++++++ pkg/models/quotas.go | 63 +++++++++++++ .../sqlstore/migrations/migrations.go | 1 + pkg/services/sqlstore/migrations/quota_mig.go | 27 ++++++ pkg/services/sqlstore/quota.go | 90 +++++++++++++++++++ 8 files changed, 218 insertions(+), 1 deletion(-) create mode 100644 pkg/api/quota.go create mode 100644 pkg/models/quotas.go create mode 100644 pkg/services/sqlstore/migrations/quota_mig.go create mode 100644 pkg/services/sqlstore/quota.go diff --git a/conf/defaults.ini b/conf/defaults.ini index 941813fb4c6..bfa13c231e5 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -253,4 +253,9 @@ exchange = grafana_events enabled = false path = /var/lib/grafana/dashboards - +[quota] +user = 10 +dashboard = 100 +data_source = 10 +endpoint = 10 +collector = 10 diff --git a/main.go b/main.go index 045527a6811..cdbf248358c 100644 --- a/main.go +++ b/main.go @@ -15,6 +15,7 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/metrics" + "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/eventpublisher" "github.com/grafana/grafana/pkg/services/notifications" @@ -56,6 +57,8 @@ func main() { eventpublisher.Init() plugins.Init() + models.InitQuotaDefaults() + if err := notifications.Init(); err != nil { log.Fatal(3, "Notification service failed to initialize", err) } diff --git a/pkg/api/api.go b/pkg/api/api.go index 6c56e6aa9f1..fce99e733f5 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -122,6 +122,8 @@ func Register(r *macaron.Macaron) { r.Post("/users", bind(m.AddOrgUserCommand{}), wrap(AddOrgUser)) r.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUser)) r.Delete("/users/:userId", wrap(RemoveOrgUser)) + r.Get("/quotas", wrap(GetOrgQuotas)) + r.Put("/quotas/:target", bind(m.UpdateQuotaCmd{}), wrap(UpdateOrgQuota)) }, reqGrafanaAdmin) // auth api keys diff --git a/pkg/api/quota.go b/pkg/api/quota.go new file mode 100644 index 00000000000..13e0e8c9781 --- /dev/null +++ b/pkg/api/quota.go @@ -0,0 +1,26 @@ +package api + +import ( + "github.com/grafana/grafana/pkg/bus" + "github.com/grafana/grafana/pkg/middleware" + m "github.com/grafana/grafana/pkg/models" +) + +func GetOrgQuotas(c *middleware.Context) Response { + query := m.GetQuotasQuery{OrgId: c.ParamsInt64(":orgId")} + + if err := bus.Dispatch(&query); err != nil { + return ApiError(500, "Failed to get org quotas", err) + } + + return Json(200, query.Result) +} + +func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateQuotaCmd) Response { + cmd.OrgId = c.ParamsInt64(":orgId") + cmd.Target = m.QuotaTarget(c.Params(":target")) + if err := bus.Dispatch(&cmd); err != nil { + return ApiError(500, "Failed to update org quotas", err) + } + return ApiSuccess("Organization quota updated") +} diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go new file mode 100644 index 00000000000..e296ddedb8e --- /dev/null +++ b/pkg/models/quotas.go @@ -0,0 +1,63 @@ +package models + +import ( + "github.com/grafana/grafana/pkg/setting" + "time" +) + +type QuotaTarget string + +const ( + QUOTA_USER QuotaTarget = "user" //SQL table to query. ie. "select count(*) from user where org_id=?" + QUOTA_DATASOURCE QuotaTarget = "data_source" + QUOTA_DASHBOARD QuotaTarget = "dashboard" + QUOTA_ENDPOINT QuotaTarget = "endpoint" + QUOTA_COLLECTOR QuotaTarget = "collector" +) + +// defaults are set from settings package. +var DefaultQuotas map[QuotaTarget]int64 + +func InitQuotaDefaults() { + // set global defaults. + DefaultQuotas = make(map[QuotaTarget]int64) + quota := setting.Cfg.Section("quota") + DefaultQuotas[QUOTA_USER] = quota.Key("user").MustInt64(10) + DefaultQuotas[QUOTA_DATASOURCE] = quota.Key("data_source").MustInt64(10) + DefaultQuotas[QUOTA_DASHBOARD] = quota.Key("dashboard").MustInt64(10) + DefaultQuotas[QUOTA_ENDPOINT] = quota.Key("endpoint").MustInt64(10) + DefaultQuotas[QUOTA_COLLECTOR] = quota.Key("collector").MustInt64(10) +} + +type Quota struct { + Id int64 + OrgId int64 + Target QuotaTarget + Limit int64 + Created time.Time + Updated time.Time +} + +type QuotaDTO struct { + OrgId int64 `json:"org_id"` + Target QuotaTarget `json:"target"` + Limit int64 `json:"limit"` + Used int64 `json:"used"` +} + +type GetQuotaByTargetQuery struct { + Target QuotaTarget + OrgId int64 + Result *QuotaDTO +} + +type GetQuotasQuery struct { + OrgId int64 + Result []*QuotaDTO +} + +type UpdateQuotaCmd struct { + Target QuotaTarget `json:"target"` + Limit int64 `json:"limit"` + OrgId int64 `json:"-"` +} diff --git a/pkg/services/sqlstore/migrations/migrations.go b/pkg/services/sqlstore/migrations/migrations.go index 70acc9ef736..8f7054d3959 100644 --- a/pkg/services/sqlstore/migrations/migrations.go +++ b/pkg/services/sqlstore/migrations/migrations.go @@ -17,6 +17,7 @@ func AddMigrations(mg *Migrator) { addDataSourceMigration(mg) addApiKeyMigrations(mg) addDashboardSnapshotMigrations(mg) + addQuotaMigration(mg) } func addMigrationLogMigrations(mg *Migrator) { diff --git a/pkg/services/sqlstore/migrations/quota_mig.go b/pkg/services/sqlstore/migrations/quota_mig.go new file mode 100644 index 00000000000..d0b3b14fd5a --- /dev/null +++ b/pkg/services/sqlstore/migrations/quota_mig.go @@ -0,0 +1,27 @@ +package migrations + +import ( + . "github.com/grafana/grafana/pkg/services/sqlstore/migrator" +) + +func addQuotaMigration(mg *Migrator) { + + var quotaV1 = Table{ + Name: "quota", + Columns: []*Column{ + {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, + {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "target", Type: DB_NVarchar, Length: 255, Nullable: false}, + {Name: "limit", Type: DB_BigInt, Nullable: false}, + {Name: "created", Type: DB_DateTime, Nullable: false}, + {Name: "updated", Type: DB_DateTime, Nullable: false}, + }, + Indices: []*Index{ + {Cols: []string{"org_id", "target"}, Type: UniqueIndex}, + }, + } + mg.AddMigration("create quota table v1", NewAddTableMigration(quotaV1)) + + //------- indexes ------------------ + addTableIndicesMigrations(mg, "v1", quotaV1) +} diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go new file mode 100644 index 00000000000..47b7d253ffc --- /dev/null +++ b/pkg/services/sqlstore/quota.go @@ -0,0 +1,90 @@ +package sqlstore + +import ( + "fmt" + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" +) + +func init() { + bus.AddHandler("sql", GetQuotaByTarget) + bus.AddHandler("sql", GetQuotas) + bus.AddHandler("sql", UpdateQuota) +} + +type targetCount struct { + Count int64 +} + +func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { + quota := m.Quota{ + Target: query.Target, + OrgId: query.OrgId, + } + has, err := x.Get(quota) + if err != nil { + return err + } else if has == false { + quota.Limit = m.DefaultQuotas[query.Target] + } + + //get quota used. + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", string(query.Target)) + resp := make([]*targetCount, 0) + if err := x.Sql(rawSql, query.OrgId).Find(&resp); err != nil { + return err + } + + query.Result = &m.QuotaDTO{ + Target: query.Target, + Limit: quota.Limit, + OrgId: query.OrgId, + Used: resp[0].Count, + } + + return nil +} + +func GetQuotas(query *m.GetQuotasQuery) error { + quotas := make([]*m.Quota, 0) + sess := x.Table("quota") + if err := sess.Where("org_id=?", query.OrgId).Find("as); err != nil { + return err + } + + seenTargets := make(map[m.QuotaTarget]bool) + for _, q := range quotas { + seenTargets[q.Target] = true + } + + for t, v := range m.DefaultQuotas { + if _, ok := seenTargets[t]; !ok { + quotas = append(quotas, &m.Quota{ + OrgId: query.OrgId, + Target: t, + Limit: v, + }) + } + } + result := make([]*m.QuotaDTO, len(quotas)) + for i, q := range quotas { + //get quota used. + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", string(q.Target)) + resp := make([]*targetCount, 0) + if err := x.Sql(rawSql, q.OrgId).Find(&resp); err != nil { + return err + } + result[i] = &m.QuotaDTO{ + Target: q.Target, + Limit: q.Limit, + OrgId: q.OrgId, + Used: resp[0].Count, + } + } + query.Result = result + return nil +} + +func UpdateQuota(cmd *m.UpdateQuotaCmd) error { + return nil +} From 0688050552e10969f036f658e70aa16e7833baa1 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Mon, 20 Jul 2015 22:38:25 +0800 Subject: [PATCH 02/16] add quota middleware to enforce quotas. issue #321 Conflicts: pkg/api/api.go --- pkg/api/api.go | 6 ++++-- pkg/api/dashboard.go | 12 ++++++++++++ pkg/middleware/middleware.go | 13 +++++++++++++ pkg/models/quotas.go | 12 ++++++++++++ 4 files changed, 41 insertions(+), 2 deletions(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index fce99e733f5..2b87863d188 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -14,6 +14,7 @@ func Register(r *macaron.Macaron) { reqGrafanaAdmin := middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}) reqEditorRole := middleware.RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN) regOrgAdmin := middleware.RoleAuth(m.ROLE_ADMIN) + limitQuota := middleware.LimitQuota bind := binding.Bind // not logged in views @@ -95,7 +96,7 @@ func Register(r *macaron.Macaron) { r.Get("/", wrap(GetOrgCurrent)) r.Put("/", bind(dtos.UpdateOrgForm{}), wrap(UpdateOrgCurrent)) r.Put("/address", bind(dtos.UpdateOrgAddressForm{}), wrap(UpdateOrgAddressCurrent)) - r.Post("/users", bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) + r.Post("/users", limitQuota(m.QUOTA_USER), bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) r.Get("/users", wrap(GetOrgUsersForCurrentOrg)) r.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUserForCurrentOrg)) r.Delete("/users/:userId", wrap(RemoveOrgUserForCurrentOrg)) @@ -136,7 +137,7 @@ func Register(r *macaron.Macaron) { // Data sources r.Group("/datasources", func() { r.Get("/", GetDataSources) - r.Post("/", bind(m.AddDataSourceCommand{}), AddDataSource) + r.Post("/", limitQuota(m.QUOTA_DATASOURCE), bind(m.AddDataSourceCommand{}), AddDataSource) r.Put("/:id", bind(m.UpdateDataSourceCommand{}), UpdateDataSource) r.Delete("/:id", DeleteDataSource) r.Get("/:id", GetDataSourceById) @@ -161,6 +162,7 @@ func Register(r *macaron.Macaron) { // metrics r.Get("/metrics/test", GetTestMetrics) + }, reqSignedIn) // admin api diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index a10c3c92f96..21b69265da1 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -86,6 +86,18 @@ func DeleteDashboard(c *middleware.Context) { func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) { cmd.OrgId = c.OrgId + dash := cmd.GetDashboardModel() + if dash.Id == 0 { + limitReached, err := m.QuotaReached(cmd.OrgId, m.QUOTA_DASHBOARD) + if err != nil { + c.JsonApiErr(500, "failed to get quota", err) + } + if limitReached { + c.JsonApiErr(403, "Quota reached", nil) + return + } + } + err := bus.Dispatch(&cmd) if err != nil { if err == m.ErrDashboardWithSameNameExists { diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 8704ec5a787..e10b2f76781 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -253,3 +253,16 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) { ctx.JSON(status, resp) } + +func LimitQuota(target m.QuotaTarget) macaron.Handler { + return func(c *Context) { + limitReached, err := m.QuotaReached(c.OrgId, target) + if err != nil { + c.JsonApiErr(500, "failed to get quota", err) + } + if limitReached { + c.JsonApiErr(403, "Quota reached", nil) + return + } + } +} diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go index e296ddedb8e..d16bd0325cc 100644 --- a/pkg/models/quotas.go +++ b/pkg/models/quotas.go @@ -1,6 +1,7 @@ package models import ( + "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/setting" "time" ) @@ -61,3 +62,14 @@ type UpdateQuotaCmd struct { Limit int64 `json:"limit"` OrgId int64 `json:"-"` } + +func QuotaReached(org_id int64, target QuotaTarget) (bool, error) { + query := GetQuotaByTargetQuery{OrgId: org_id, Target: target} + if err := bus.Dispatch(&query); err != nil { + return true, err + } + if query.Result.Used >= query.Result.Limit { + return true, nil + } + return false, nil +} From 76e9ebde36cef5abc4438b255b5990879ce3fe62 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Mon, 20 Jul 2015 22:42:31 +0800 Subject: [PATCH 03/16] always return after errors. --- pkg/api/dashboard.go | 1 + pkg/middleware/middleware.go | 1 + 2 files changed, 2 insertions(+) diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 21b69265da1..132e838b534 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -91,6 +91,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) { limitReached, err := m.QuotaReached(cmd.OrgId, m.QUOTA_DASHBOARD) if err != nil { c.JsonApiErr(500, "failed to get quota", err) + return } if limitReached { c.JsonApiErr(403, "Quota reached", nil) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index e10b2f76781..a6963af61ed 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -259,6 +259,7 @@ func LimitQuota(target m.QuotaTarget) macaron.Handler { limitReached, err := m.QuotaReached(c.OrgId, target) if err != nil { c.JsonApiErr(500, "failed to get quota", err) + return } if limitReached { c.JsonApiErr(403, "Quota reached", nil) From c23813084283eea5de6b82fb9742202978e8a944 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Mon, 20 Jul 2015 22:45:00 +0800 Subject: [PATCH 04/16] quote table names passed by arguments --- pkg/services/sqlstore/quota.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index 47b7d253ffc..244c49cc023 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -29,7 +29,7 @@ func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { } //get quota used. - rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", string(query.Target)) + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(string(query.Target))) resp := make([]*targetCount, 0) if err := x.Sql(rawSql, query.OrgId).Find(&resp); err != nil { return err @@ -69,7 +69,7 @@ func GetQuotas(query *m.GetQuotasQuery) error { result := make([]*m.QuotaDTO, len(quotas)) for i, q := range quotas { //get quota used. - rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", string(q.Target)) + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(string(q.Target))) resp := make([]*targetCount, 0) if err := x.Sql(rawSql, q.OrgId).Find(&resp); err != nil { return err From 3d4d8225280dd2fe9292d98f86099281dec50866 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 21 Jul 2015 18:30:31 +0800 Subject: [PATCH 05/16] implement updateQuota function --- pkg/api/quota.go | 5 +++++ pkg/models/quotas.go | 11 +++++++++++ pkg/services/sqlstore/quota.go | 26 +++++++++++++++++++++++++- 3 files changed, 41 insertions(+), 1 deletion(-) diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 13e0e8c9781..1811270c3a4 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -19,6 +19,11 @@ func GetOrgQuotas(c *middleware.Context) Response { func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateQuotaCmd) Response { cmd.OrgId = c.ParamsInt64(":orgId") cmd.Target = m.QuotaTarget(c.Params(":target")) + + if !cmd.Target.IsValid() { + return ApiError(404, "Invalid quota target", nil) + } + if err := bus.Dispatch(&cmd); err != nil { return ApiError(500, "Failed to update org quotas", err) } diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go index d16bd0325cc..27ab74cb1e2 100644 --- a/pkg/models/quotas.go +++ b/pkg/models/quotas.go @@ -1,6 +1,7 @@ package models import ( + "errors" "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/setting" "time" @@ -16,6 +17,13 @@ const ( QUOTA_COLLECTOR QuotaTarget = "collector" ) +var ErrInvalidQuotaTarget = errors.New("Invalid quota target") + +func (q QuotaTarget) IsValid() bool { + _, ok := DefaultQuotas[q] + return ok +} + // defaults are set from settings package. var DefaultQuotas map[QuotaTarget]int64 @@ -64,6 +72,9 @@ type UpdateQuotaCmd struct { } func QuotaReached(org_id int64, target QuotaTarget) (bool, error) { + if !target.IsValid() { + return true, ErrInvalidQuotaTarget + } query := GetQuotaByTargetQuery{OrgId: org_id, Target: target} if err := bus.Dispatch(&query); err != nil { return true, err diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index 244c49cc023..a5ce44fc4b6 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -86,5 +86,29 @@ func GetQuotas(query *m.GetQuotasQuery) error { } func UpdateQuota(cmd *m.UpdateQuotaCmd) error { - return nil + return inTransaction2(func(sess *session) error { + //Check if quota is already defined in the DB + quota := m.Quota{ + Target: cmd.Target, + OrgId: cmd.OrgId, + } + has, err := sess.Get(quota) + if err != nil { + return err + } + quota.Limit = cmd.Limit + if has == false { + //No quota in the DB for this target, so create a new one. + if _, err := sess.Insert("a); err != nil { + return err + } + } else { + //update existing quota entry in the DB. + if _, err := sess.Id(quota.Id).Update("a); err != nil { + return err + } + } + + return nil + }) } From 555cbeffa5ae8c5d5dc93b9764a963b13272ac66 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Fri, 11 Sep 2015 01:18:36 +0800 Subject: [PATCH 06/16] allow all users to retrieve org and quota data. --- pkg/api/api.go | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/pkg/api/api.go b/pkg/api/api.go index 2b87863d188..90118d93912 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -91,9 +91,14 @@ func Register(r *macaron.Macaron) { r.Put("/:id", bind(m.UpdateUserCommand{}), wrap(UpdateUser)) }, reqGrafanaAdmin) - // current org + // org information available to all users. r.Group("/org", func() { r.Get("/", wrap(GetOrgCurrent)) + r.Get("/quotas", wrap(GetQuotas)) + }) + + // current org + r.Group("/org", func() { r.Put("/", bind(dtos.UpdateOrgForm{}), wrap(UpdateOrgCurrent)) r.Put("/address", bind(dtos.UpdateOrgAddressForm{}), wrap(UpdateOrgAddressCurrent)) r.Post("/users", limitQuota(m.QUOTA_USER), bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) From 852f9bd27700f9335dd46ba75609f59e3dc001b9 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Fri, 11 Sep 2015 01:47:33 +0800 Subject: [PATCH 07/16] refactor quota settings --- conf/defaults.ini | 4 ++-- main.go | 3 --- pkg/api/dashboard.go | 2 +- pkg/api/quota.go | 11 +++++++++++ pkg/middleware/middleware.go | 19 ++++++++++++++++++- pkg/models/quotas.go | 33 +-------------------------------- pkg/services/sqlstore/quota.go | 9 +++++---- pkg/setting/setting.go | 4 ++++ pkg/setting/setting_quota.go | 17 +++++++++++++++++ 9 files changed, 59 insertions(+), 43 deletions(-) create mode 100644 pkg/setting/setting_quota.go diff --git a/conf/defaults.ini b/conf/defaults.ini index bfa13c231e5..e1c06a0223c 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -254,8 +254,8 @@ enabled = false path = /var/lib/grafana/dashboards [quota] +enabled = false user = 10 dashboard = 100 data_source = 10 -endpoint = 10 -collector = 10 + diff --git a/main.go b/main.go index cdbf248358c..045527a6811 100644 --- a/main.go +++ b/main.go @@ -15,7 +15,6 @@ import ( "github.com/grafana/grafana/pkg/log" "github.com/grafana/grafana/pkg/login" "github.com/grafana/grafana/pkg/metrics" - "github.com/grafana/grafana/pkg/models" "github.com/grafana/grafana/pkg/plugins" "github.com/grafana/grafana/pkg/services/eventpublisher" "github.com/grafana/grafana/pkg/services/notifications" @@ -57,8 +56,6 @@ func main() { eventpublisher.Init() plugins.Init() - models.InitQuotaDefaults() - if err := notifications.Init(); err != nil { log.Fatal(3, "Notification service failed to initialize", err) } diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index 132e838b534..cc6ccd29064 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -88,7 +88,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) { dash := cmd.GetDashboardModel() if dash.Id == 0 { - limitReached, err := m.QuotaReached(cmd.OrgId, m.QUOTA_DASHBOARD) + limitReached, err := middleware.QuotaReached(cmd.OrgId, m.QUOTA_DASHBOARD) if err != nil { c.JsonApiErr(500, "failed to get quota", err) return diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 1811270c3a4..9c5dd615247 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -16,6 +16,17 @@ func GetOrgQuotas(c *middleware.Context) Response { return Json(200, query.Result) } +// allow users to query the quotas of their own org. +func GetQuotas(c *middleware.Context) Response { + query := m.GetQuotasQuery{OrgId: c.OrgId} + + if err := bus.Dispatch(&query); err != nil { + return ApiError(500, "Failed to get quotas", err) + } + + return Json(200, query.Result) +} + func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateQuotaCmd) Response { cmd.OrgId = c.ParamsInt64(":orgId") cmd.Target = m.QuotaTarget(c.Params(":target")) diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index a6963af61ed..1b218a6eedc 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -256,7 +256,7 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) { func LimitQuota(target m.QuotaTarget) macaron.Handler { return func(c *Context) { - limitReached, err := m.QuotaReached(c.OrgId, target) + limitReached, err := QuotaReached(c.OrgId, target) if err != nil { c.JsonApiErr(500, "failed to get quota", err) return @@ -267,3 +267,20 @@ func LimitQuota(target m.QuotaTarget) macaron.Handler { } } } + +func QuotaReached(org_id int64, target m.QuotaTarget) (bool, error) { + if !setting.Quota.Enabled { + return false, nil + } + if !target.IsValid() { + return true, m.ErrInvalidQuotaTarget + } + query := m.GetQuotaByTargetQuery{OrgId: org_id, Target: target} + if err := bus.Dispatch(&query); err != nil { + return true, err + } + if query.Result.Used >= query.Result.Limit { + return true, nil + } + return false, nil +} diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go index 27ab74cb1e2..75df5cd3765 100644 --- a/pkg/models/quotas.go +++ b/pkg/models/quotas.go @@ -2,7 +2,6 @@ package models import ( "errors" - "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/setting" "time" ) @@ -13,31 +12,15 @@ const ( QUOTA_USER QuotaTarget = "user" //SQL table to query. ie. "select count(*) from user where org_id=?" QUOTA_DATASOURCE QuotaTarget = "data_source" QUOTA_DASHBOARD QuotaTarget = "dashboard" - QUOTA_ENDPOINT QuotaTarget = "endpoint" - QUOTA_COLLECTOR QuotaTarget = "collector" ) var ErrInvalidQuotaTarget = errors.New("Invalid quota target") func (q QuotaTarget) IsValid() bool { - _, ok := DefaultQuotas[q] + _, ok := setting.Quota.Default[string(q)] return ok } -// defaults are set from settings package. -var DefaultQuotas map[QuotaTarget]int64 - -func InitQuotaDefaults() { - // set global defaults. - DefaultQuotas = make(map[QuotaTarget]int64) - quota := setting.Cfg.Section("quota") - DefaultQuotas[QUOTA_USER] = quota.Key("user").MustInt64(10) - DefaultQuotas[QUOTA_DATASOURCE] = quota.Key("data_source").MustInt64(10) - DefaultQuotas[QUOTA_DASHBOARD] = quota.Key("dashboard").MustInt64(10) - DefaultQuotas[QUOTA_ENDPOINT] = quota.Key("endpoint").MustInt64(10) - DefaultQuotas[QUOTA_COLLECTOR] = quota.Key("collector").MustInt64(10) -} - type Quota struct { Id int64 OrgId int64 @@ -70,17 +53,3 @@ type UpdateQuotaCmd struct { Limit int64 `json:"limit"` OrgId int64 `json:"-"` } - -func QuotaReached(org_id int64, target QuotaTarget) (bool, error) { - if !target.IsValid() { - return true, ErrInvalidQuotaTarget - } - query := GetQuotaByTargetQuery{OrgId: org_id, Target: target} - if err := bus.Dispatch(&query); err != nil { - return true, err - } - if query.Result.Used >= query.Result.Limit { - return true, nil - } - return false, nil -} diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index a5ce44fc4b6..b54f6a83a57 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -4,6 +4,7 @@ import ( "fmt" "github.com/grafana/grafana/pkg/bus" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func init() { @@ -25,7 +26,7 @@ func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { if err != nil { return err } else if has == false { - quota.Limit = m.DefaultQuotas[query.Target] + quota.Limit = setting.Quota.Default[string(query.Target)] } //get quota used. @@ -57,11 +58,11 @@ func GetQuotas(query *m.GetQuotasQuery) error { seenTargets[q.Target] = true } - for t, v := range m.DefaultQuotas { - if _, ok := seenTargets[t]; !ok { + for t, v := range setting.Quota.Default { + if _, ok := seenTargets[m.QuotaTarget(t)]; !ok { quotas = append(quotas, &m.Quota{ OrgId: query.OrgId, - Target: t, + Target: m.QuotaTarget(t), Limit: v, }) } diff --git a/pkg/setting/setting.go b/pkg/setting/setting.go index a8b3383270e..8ff58402c20 100644 --- a/pkg/setting/setting.go +++ b/pkg/setting/setting.go @@ -126,6 +126,9 @@ var ( // SMTP email settings Smtp SmtpSettings + + // QUOTA + Quota QuotaSettings ) type CommandLineArgs struct { @@ -434,6 +437,7 @@ func NewConfigContext(args *CommandLineArgs) { readSessionConfig() readSmtpSettings() + readQuotaSettings() if VerifyEmailEnabled && !Smtp.Enabled { log.Warn("require_email_validation is enabled but smpt is disabled") diff --git a/pkg/setting/setting_quota.go b/pkg/setting/setting_quota.go new file mode 100644 index 00000000000..d6ca0cd7ba8 --- /dev/null +++ b/pkg/setting/setting_quota.go @@ -0,0 +1,17 @@ +package setting + +type QuotaSettings struct { + Enabled bool + Default map[string]int64 +} + +func readQuotaSettings() { + // set global defaults. + DefaultQuotas := make(map[string]int64) + quota := Cfg.Section("quota") + Quota.Enabled = quota.Key("enabled").MustBool(false) + DefaultQuotas["user"] = quota.Key("user").MustInt64(10) + DefaultQuotas["data_source"] = quota.Key("data_source").MustInt64(10) + DefaultQuotas["dashboard"] = quota.Key("dashboard").MustInt64(10) + Quota.Default = DefaultQuotas +} From 47bf1bd21afd9ab0164952c79a91f06209589476 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Fri, 11 Sep 2015 01:51:12 +0800 Subject: [PATCH 08/16] return 404 when quotas not enabled. --- pkg/api/quota.go | 10 ++++++++++ 1 file changed, 10 insertions(+) diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 9c5dd615247..797ebd1661e 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -4,9 +4,13 @@ import ( "github.com/grafana/grafana/pkg/bus" "github.com/grafana/grafana/pkg/middleware" m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" ) func GetOrgQuotas(c *middleware.Context) Response { + if !setting.Quota.Enabled { + return ApiError(404, "Quotas not enabled", nil) + } query := m.GetQuotasQuery{OrgId: c.ParamsInt64(":orgId")} if err := bus.Dispatch(&query); err != nil { @@ -18,6 +22,9 @@ func GetOrgQuotas(c *middleware.Context) Response { // allow users to query the quotas of their own org. func GetQuotas(c *middleware.Context) Response { + if !setting.Quota.Enabled { + return ApiError(404, "Quotas not enabled", nil) + } query := m.GetQuotasQuery{OrgId: c.OrgId} if err := bus.Dispatch(&query); err != nil { @@ -28,6 +35,9 @@ func GetQuotas(c *middleware.Context) Response { } func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateQuotaCmd) Response { + if !setting.Quota.Enabled { + return ApiError(404, "Quotas not enabled", nil) + } cmd.OrgId = c.ParamsInt64(":orgId") cmd.Target = m.QuotaTarget(c.Params(":target")) From 6488324cf18906c7e4d2aaf4e09ec818fda22b62 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Fri, 11 Sep 2015 23:17:10 +0800 Subject: [PATCH 09/16] enhance quota support. now includes: - perOrg (users, dashboards, datasources, api_keys) - perUser (orgs) - global (users, orgs, dashboards, datasources, api_keys, sessions) --- conf/defaults.ini | 35 +++- pkg/api/api.go | 27 +-- pkg/api/dashboard.go | 2 +- pkg/api/login_oauth.go | 10 +- pkg/api/quota.go | 34 +++- pkg/middleware/middleware.go | 87 ++++++++-- pkg/models/quotas.go | 157 +++++++++++++---- pkg/services/sqlstore/migrations/quota_mig.go | 5 +- pkg/services/sqlstore/quota.go | 158 ++++++++++++++++-- pkg/setting/setting_quota.go | 53 +++++- 10 files changed, 476 insertions(+), 92 deletions(-) diff --git a/conf/defaults.ini b/conf/defaults.ini index e1c06a0223c..d640820fe65 100644 --- a/conf/defaults.ini +++ b/conf/defaults.ini @@ -87,6 +87,7 @@ cookie_secure = false # Session life time, default is 86400 session_life_time = 86400 +gc_interval_time = 86400 #################################### Analytics #################################### [analytics] @@ -253,9 +254,37 @@ exchange = grafana_events enabled = false path = /var/lib/grafana/dashboards +#################################### Usage Quotas ########################## [quota] enabled = false -user = 10 -dashboard = 100 -data_source = 10 +#### set quotas to -1 to make unlimited. #### +# limit number of users per Org. +org_user = 10 + +# limit number of dashboards per Org. +org_dashboard = 100 + +# limit number of data_sources per Org. +org_data_source = 10 + +# limit number of api_keys per Org. +org_api_key = 10 + +# limit number of orgs a user can create. +user_org = 10 + +# Global limit of users. +global_user = -1 + +# global limit of orgs. +global_org = -1 + +# global limit of dashboards +global_dashboard = -1 + +# global limit of api_keys +global_api_key = -1 + +# global limit on number of logged in users. +global_session = -1 diff --git a/pkg/api/api.go b/pkg/api/api.go index 90118d93912..27eb3c749db 100644 --- a/pkg/api/api.go +++ b/pkg/api/api.go @@ -14,14 +14,14 @@ func Register(r *macaron.Macaron) { reqGrafanaAdmin := middleware.Auth(&middleware.AuthOptions{ReqSignedIn: true, ReqGrafanaAdmin: true}) reqEditorRole := middleware.RoleAuth(m.ROLE_EDITOR, m.ROLE_ADMIN) regOrgAdmin := middleware.RoleAuth(m.ROLE_ADMIN) - limitQuota := middleware.LimitQuota + quota := middleware.Quota bind := binding.Bind // not logged in views r.Get("/", reqSignedIn, Index) r.Get("/logout", Logout) - r.Post("/login", bind(dtos.LoginCommand{}), wrap(LoginPost)) - r.Get("/login/:name", OAuthLogin) + r.Post("/login", quota("session"), bind(dtos.LoginCommand{}), wrap(LoginPost)) + r.Get("/login/:name", quota("session"), OAuthLogin) r.Get("/login", LoginView) r.Get("/invite/:code", Index) @@ -45,7 +45,7 @@ func Register(r *macaron.Macaron) { // sign up r.Get("/signup", Index) r.Get("/api/user/signup/options", wrap(GetSignUpOptions)) - r.Post("/api/user/signup", bind(dtos.SignUpForm{}), wrap(SignUp)) + r.Post("/api/user/signup", quota("user"), bind(dtos.SignUpForm{}), wrap(SignUp)) r.Post("/api/user/signup/step2", bind(dtos.SignUpStep2Form{}), wrap(SignUpStep2)) // invited @@ -67,7 +67,7 @@ func Register(r *macaron.Macaron) { r.Get("/api/snapshots-delete/:key", DeleteDashboardSnapshot) // api renew session based on remember cookie - r.Get("/api/login/ping", LoginApiPing) + r.Get("/api/login/ping", quota("session"), LoginApiPing) // authed api r.Group("/api", func() { @@ -81,6 +81,7 @@ func Register(r *macaron.Macaron) { r.Post("/stars/dashboard/:id", wrap(StarDashboard)) r.Delete("/stars/dashboard/:id", wrap(UnstarDashboard)) r.Put("/password", bind(m.ChangeUserPasswordCommand{}), wrap(ChangeUserPassword)) + r.Get("/quotas", wrap(GetUserQuotas)) }) // users (admin permission required) @@ -94,26 +95,26 @@ func Register(r *macaron.Macaron) { // org information available to all users. r.Group("/org", func() { r.Get("/", wrap(GetOrgCurrent)) - r.Get("/quotas", wrap(GetQuotas)) + r.Get("/quotas", wrap(GetOrgQuotas)) }) // current org r.Group("/org", func() { r.Put("/", bind(dtos.UpdateOrgForm{}), wrap(UpdateOrgCurrent)) r.Put("/address", bind(dtos.UpdateOrgAddressForm{}), wrap(UpdateOrgAddressCurrent)) - r.Post("/users", limitQuota(m.QUOTA_USER), bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) + r.Post("/users", quota("user"), bind(m.AddOrgUserCommand{}), wrap(AddOrgUserToCurrentOrg)) r.Get("/users", wrap(GetOrgUsersForCurrentOrg)) r.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUserForCurrentOrg)) r.Delete("/users/:userId", wrap(RemoveOrgUserForCurrentOrg)) // invites r.Get("/invites", wrap(GetPendingOrgInvites)) - r.Post("/invites", bind(dtos.AddInviteForm{}), wrap(AddOrgInvite)) + r.Post("/invites", quota("user"), bind(dtos.AddInviteForm{}), wrap(AddOrgInvite)) r.Patch("/invites/:code/revoke", wrap(RevokeInvite)) }, regOrgAdmin) // create new org - r.Post("/orgs", bind(m.CreateOrgCommand{}), wrap(CreateOrg)) + r.Post("/orgs", quota("org"), bind(m.CreateOrgCommand{}), wrap(CreateOrg)) // search all orgs r.Get("/orgs", reqGrafanaAdmin, wrap(SearchOrgs)) @@ -129,20 +130,20 @@ func Register(r *macaron.Macaron) { r.Patch("/users/:userId", bind(m.UpdateOrgUserCommand{}), wrap(UpdateOrgUser)) r.Delete("/users/:userId", wrap(RemoveOrgUser)) r.Get("/quotas", wrap(GetOrgQuotas)) - r.Put("/quotas/:target", bind(m.UpdateQuotaCmd{}), wrap(UpdateOrgQuota)) + r.Put("/quotas/:target", bind(m.UpdateOrgQuotaCmd{}), wrap(UpdateOrgQuota)) }, reqGrafanaAdmin) // auth api keys r.Group("/auth/keys", func() { r.Get("/", wrap(GetApiKeys)) - r.Post("/", bind(m.AddApiKeyCommand{}), wrap(AddApiKey)) + r.Post("/", quota("api_key"), bind(m.AddApiKeyCommand{}), wrap(AddApiKey)) r.Delete("/:id", wrap(DeleteApiKey)) }, regOrgAdmin) // Data sources r.Group("/datasources", func() { r.Get("/", GetDataSources) - r.Post("/", limitQuota(m.QUOTA_DATASOURCE), bind(m.AddDataSourceCommand{}), AddDataSource) + r.Post("/", quota("data_source"), bind(m.AddDataSourceCommand{}), AddDataSource) r.Put("/:id", bind(m.UpdateDataSourceCommand{}), UpdateDataSource) r.Delete("/:id", DeleteDataSource) r.Get("/:id", GetDataSourceById) @@ -177,6 +178,8 @@ func Register(r *macaron.Macaron) { r.Put("/users/:id/password", bind(dtos.AdminUpdateUserPasswordForm{}), AdminUpdateUserPassword) r.Put("/users/:id/permissions", bind(dtos.AdminUpdateUserPermissionsForm{}), AdminUpdateUserPermissions) r.Delete("/users/:id", AdminDeleteUser) + r.Get("/users/:id/quotas", wrap(GetUserQuotas)) + r.Put("/users/:id/quotas/:target", bind(m.UpdateUserQuotaCmd{}), wrap(UpdateUserQuota)) }, reqGrafanaAdmin) // rendering diff --git a/pkg/api/dashboard.go b/pkg/api/dashboard.go index cc6ccd29064..d7f9f2cd740 100644 --- a/pkg/api/dashboard.go +++ b/pkg/api/dashboard.go @@ -88,7 +88,7 @@ func PostDashboard(c *middleware.Context, cmd m.SaveDashboardCommand) { dash := cmd.GetDashboardModel() if dash.Id == 0 { - limitReached, err := middleware.QuotaReached(cmd.OrgId, m.QUOTA_DASHBOARD) + limitReached, err := middleware.QuotaReached(c, "dashboard") if err != nil { c.JsonApiErr(500, "failed to get quota", err) return diff --git a/pkg/api/login_oauth.go b/pkg/api/login_oauth.go index 796599df864..4244feef664 100644 --- a/pkg/api/login_oauth.go +++ b/pkg/api/login_oauth.go @@ -74,7 +74,15 @@ func OAuthLogin(ctx *middleware.Context) { ctx.Redirect(setting.AppSubUrl + "/login") return } - + limitReached, err := middleware.QuotaReached(ctx, "user") + if err != nil { + ctx.Handle(500, "Failed to get user quota", err) + return + } + if limitReached { + ctx.Redirect(setting.AppSubUrl + "/login") + return + } cmd := m.CreateUserCommand{ Login: userInfo.Email, Email: userInfo.Email, diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 797ebd1661e..490a8e90729 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -11,7 +11,7 @@ func GetOrgQuotas(c *middleware.Context) Response { if !setting.Quota.Enabled { return ApiError(404, "Quotas not enabled", nil) } - query := m.GetQuotasQuery{OrgId: c.ParamsInt64(":orgId")} + query := m.GetOrgQuotasQuery{OrgId: c.ParamsInt64(":orgId")} if err := bus.Dispatch(&query); err != nil { return ApiError(500, "Failed to get org quotas", err) @@ -20,28 +20,44 @@ func GetOrgQuotas(c *middleware.Context) Response { return Json(200, query.Result) } -// allow users to query the quotas of their own org. -func GetQuotas(c *middleware.Context) Response { +func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateOrgQuotaCmd) Response { if !setting.Quota.Enabled { return ApiError(404, "Quotas not enabled", nil) } - query := m.GetQuotasQuery{OrgId: c.OrgId} + cmd.OrgId = c.ParamsInt64(":orgId") + cmd.Target = c.Params(":target") + + if _, ok := m.QuotaToMap(setting.Quota.Org)[cmd.Target]; !ok { + return ApiError(404, "Invalid quota target", nil) + } + + if err := bus.Dispatch(&cmd); err != nil { + return ApiError(500, "Failed to update org quotas", err) + } + return ApiSuccess("Organization quota updated") +} + +func GetUserQuotas(c *middleware.Context) Response { + if !setting.Quota.Enabled { + return ApiError(404, "Quotas not enabled", nil) + } + query := m.GetUserQuotasQuery{UserId: c.ParamsInt64(":id")} if err := bus.Dispatch(&query); err != nil { - return ApiError(500, "Failed to get quotas", err) + return ApiError(500, "Failed to get org quotas", err) } return Json(200, query.Result) } -func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateQuotaCmd) Response { +func UpdateUserQuota(c *middleware.Context, cmd m.UpdateUserQuotaCmd) Response { if !setting.Quota.Enabled { return ApiError(404, "Quotas not enabled", nil) } - cmd.OrgId = c.ParamsInt64(":orgId") - cmd.Target = m.QuotaTarget(c.Params(":target")) + cmd.UserId = c.ParamsInt64(":id") + cmd.Target = c.Params(":target") - if !cmd.Target.IsValid() { + if _, ok := m.QuotaToMap(setting.Quota.User)[cmd.Target]; !ok { return ApiError(404, "Invalid quota target", nil) } diff --git a/pkg/middleware/middleware.go b/pkg/middleware/middleware.go index 1b218a6eedc..8a61bdb3a0b 100644 --- a/pkg/middleware/middleware.go +++ b/pkg/middleware/middleware.go @@ -1,6 +1,7 @@ package middleware import ( + "fmt" "strconv" "strings" @@ -254,33 +255,95 @@ func (ctx *Context) JsonApiErr(status int, message string, err error) { ctx.JSON(status, resp) } -func LimitQuota(target m.QuotaTarget) macaron.Handler { +func Quota(target string) macaron.Handler { return func(c *Context) { - limitReached, err := QuotaReached(c.OrgId, target) + limitReached, err := QuotaReached(c, target) if err != nil { c.JsonApiErr(500, "failed to get quota", err) return } if limitReached { - c.JsonApiErr(403, "Quota reached", nil) + c.JsonApiErr(403, fmt.Sprintf("%s Quota reached", target), nil) return } } } -func QuotaReached(org_id int64, target m.QuotaTarget) (bool, error) { +func QuotaReached(c *Context, target string) (bool, error) { if !setting.Quota.Enabled { return false, nil } - if !target.IsValid() { - return true, m.ErrInvalidQuotaTarget - } - query := m.GetQuotaByTargetQuery{OrgId: org_id, Target: target} - if err := bus.Dispatch(&query); err != nil { - return true, err + + // get the list of scopes that this target is valid for. Org, User, Global + scopes, err := m.GetQuotaScopes(target) + if err != nil { + return false, err } - if query.Result.Used >= query.Result.Limit { - return true, nil + log.Info(fmt.Sprintf("checking quota for %s in scopes %v", target, scopes)) + + for _, scope := range scopes { + log.Info(fmt.Sprintf("checking scope %s", scope.Name)) + switch scope.Name { + case "global": + if scope.DefaultLimit < 0 { + continue + } + if scope.DefaultLimit == 0 { + return true, nil + } + if target == "session" { + usedSessions := sessionManager.Count() + if int64(usedSessions) > scope.DefaultLimit { + log.Info(fmt.Sprintf("%d sessions active, limit is %d", usedSessions, scope.DefaultLimit)) + return true, nil + } + continue + } + query := m.GetGlobalQuotaByTargetQuery{Target: scope.Target} + if err := bus.Dispatch(&query); err != nil { + return true, err + } + if query.Result.Used >= scope.DefaultLimit { + return true, nil + } + case "org": + if !c.IsSignedIn { + continue + } + query := m.GetOrgQuotaByTargetQuery{OrgId: c.OrgId, Target: scope.Target, Default: scope.DefaultLimit} + if err := bus.Dispatch(&query); err != nil { + return true, err + } + if query.Result.Limit < 0 { + continue + } + if query.Result.Limit == 0 { + return true, nil + } + + if query.Result.Used >= query.Result.Limit { + return true, nil + } + case "user": + if !c.IsSignedIn || c.UserId == 0 { + continue + } + query := m.GetUserQuotaByTargetQuery{UserId: c.UserId, Target: scope.Target, Default: scope.DefaultLimit} + if err := bus.Dispatch(&query); err != nil { + return true, err + } + if query.Result.Limit < 0 { + continue + } + if query.Result.Limit == 0 { + return true, nil + } + + if query.Result.Used >= query.Result.Limit { + return true, nil + } + } } + return false, nil } diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go index 75df5cd3765..ba17b848403 100644 --- a/pkg/models/quotas.go +++ b/pkg/models/quotas.go @@ -3,53 +3,152 @@ package models import ( "errors" "github.com/grafana/grafana/pkg/setting" + "reflect" "time" ) -type QuotaTarget string - -const ( - QUOTA_USER QuotaTarget = "user" //SQL table to query. ie. "select count(*) from user where org_id=?" - QUOTA_DATASOURCE QuotaTarget = "data_source" - QUOTA_DASHBOARD QuotaTarget = "dashboard" -) - var ErrInvalidQuotaTarget = errors.New("Invalid quota target") -func (q QuotaTarget) IsValid() bool { - _, ok := setting.Quota.Default[string(q)] - return ok -} - type Quota struct { Id int64 OrgId int64 - Target QuotaTarget + UserId int64 + Target string Limit int64 Created time.Time Updated time.Time } -type QuotaDTO struct { - OrgId int64 `json:"org_id"` - Target QuotaTarget `json:"target"` - Limit int64 `json:"limit"` - Used int64 `json:"used"` +type QuotaScope struct { + Name string + Target string + DefaultLimit int64 } -type GetQuotaByTargetQuery struct { - Target QuotaTarget - OrgId int64 - Result *QuotaDTO +type OrgQuotaDTO struct { + OrgId int64 `json:"org_id"` + Target string `json:"target"` + Limit int64 `json:"limit"` + Used int64 `json:"used"` +} + +type UserQuotaDTO struct { + UserId int64 `json:"user_id"` + Target string `json:"target"` + Limit int64 `json:"limit"` + Used int64 `json:"used"` +} + +type GlobalQuotaDTO struct { + Target string `json:"target"` + Limit int64 `json:"limit"` + Used int64 `json:"used"` +} + +type GetOrgQuotaByTargetQuery struct { + Target string + OrgId int64 + Default int64 + Result *OrgQuotaDTO } -type GetQuotasQuery struct { +type GetOrgQuotasQuery struct { OrgId int64 - Result []*QuotaDTO + Result []*OrgQuotaDTO +} + +type GetUserQuotaByTargetQuery struct { + Target string + UserId int64 + Default int64 + Result *UserQuotaDTO +} + +type GetUserQuotasQuery struct { + UserId int64 + Result []*UserQuotaDTO +} + +type GetGlobalQuotaByTargetQuery struct { + Target string + Default int64 + Result *GlobalQuotaDTO +} + +type UpdateOrgQuotaCmd struct { + Target string `json:"target"` + Limit int64 `json:"limit"` + OrgId int64 `json:"-"` +} + +type UpdateUserQuotaCmd struct { + Target string `json:"target"` + Limit int64 `json:"limit"` + UserId int64 `json:"-"` +} + +func GetQuotaScopes(target string) ([]QuotaScope, error) { + scopes := make([]QuotaScope, 0) + switch target { + case "user": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.User}, + QuotaScope{Name: "org", Target: "org_user", DefaultLimit: setting.Quota.Org.User}, + ) + return scopes, nil + case "org": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.Org}, + QuotaScope{Name: "user", Target: "org_user", DefaultLimit: setting.Quota.User.Org}, + ) + return scopes, nil + case "dashboard": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.Dashboard}, + QuotaScope{Name: "org", Target: target, DefaultLimit: setting.Quota.Org.Dashboard}, + ) + return scopes, nil + case "data_source": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.DataSource}, + QuotaScope{Name: "org", Target: target, DefaultLimit: setting.Quota.Org.DataSource}, + ) + return scopes, nil + case "api_key": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.ApiKey}, + QuotaScope{Name: "org", Target: target, DefaultLimit: setting.Quota.Org.ApiKey}, + ) + return scopes, nil + case "session": + scopes = append(scopes, + QuotaScope{Name: "global", Target: target, DefaultLimit: setting.Quota.Global.Session}, + ) + return scopes, nil + default: + return scopes, ErrInvalidQuotaTarget + } } -type UpdateQuotaCmd struct { - Target QuotaTarget `json:"target"` - Limit int64 `json:"limit"` - OrgId int64 `json:"-"` +func QuotaToMap(q interface{}) map[string]int64 { + qMap := make(map[string]int64) + typ := reflect.TypeOf(q) + val := reflect.ValueOf(q) + if typ.Kind() == reflect.Ptr { + typ = typ.Elem() + val = val.Elem() + } + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + name := field.Tag.Get("target") + if name == "" { + name = field.Name + } + if name == "-" { + continue + } + value := val.Field(i) + qMap[name] = value.Int() + } + return qMap } diff --git a/pkg/services/sqlstore/migrations/quota_mig.go b/pkg/services/sqlstore/migrations/quota_mig.go index d0b3b14fd5a..d877bb4c3c1 100644 --- a/pkg/services/sqlstore/migrations/quota_mig.go +++ b/pkg/services/sqlstore/migrations/quota_mig.go @@ -10,14 +10,15 @@ func addQuotaMigration(mg *Migrator) { Name: "quota", Columns: []*Column{ {Name: "id", Type: DB_BigInt, IsPrimaryKey: true, IsAutoIncrement: true}, - {Name: "org_id", Type: DB_BigInt, Nullable: false}, + {Name: "org_id", Type: DB_BigInt, Nullable: true}, + {Name: "user_id", Type: DB_BigInt, Nullable: true}, {Name: "target", Type: DB_NVarchar, Length: 255, Nullable: false}, {Name: "limit", Type: DB_BigInt, Nullable: false}, {Name: "created", Type: DB_DateTime, Nullable: false}, {Name: "updated", Type: DB_DateTime, Nullable: false}, }, Indices: []*Index{ - {Cols: []string{"org_id", "target"}, Type: UniqueIndex}, + {Cols: []string{"org_id", "user_id", "target"}, Type: UniqueIndex}, }, } mg.AddMigration("create quota table v1", NewAddTableMigration(quotaV1)) diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index b54f6a83a57..23bea264e87 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -8,16 +8,20 @@ import ( ) func init() { - bus.AddHandler("sql", GetQuotaByTarget) - bus.AddHandler("sql", GetQuotas) - bus.AddHandler("sql", UpdateQuota) + bus.AddHandler("sql", GetOrgQuotaByTarget) + bus.AddHandler("sql", GetOrgQuotas) + bus.AddHandler("sql", UpdateOrgQuota) + bus.AddHandler("sql", GetUserQuotaByTarget) + bus.AddHandler("sql", GetUserQuotas) + bus.AddHandler("sql", UpdateUserQuota) + bus.AddHandler("sql", GetGlobalQuotaByTarget) } type targetCount struct { Count int64 } -func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { +func GetOrgQuotaByTarget(query *m.GetOrgQuotaByTargetQuery) error { quota := m.Quota{ Target: query.Target, OrgId: query.OrgId, @@ -26,17 +30,17 @@ func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { if err != nil { return err } else if has == false { - quota.Limit = setting.Quota.Default[string(query.Target)] + quota.Limit = query.Default } //get quota used. - rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(string(query.Target))) + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(query.Target)) resp := make([]*targetCount, 0) if err := x.Sql(rawSql, query.OrgId).Find(&resp); err != nil { return err } - query.Result = &m.QuotaDTO{ + query.Result = &m.OrgQuotaDTO{ Target: query.Target, Limit: quota.Limit, OrgId: query.OrgId, @@ -46,36 +50,39 @@ func GetQuotaByTarget(query *m.GetQuotaByTargetQuery) error { return nil } -func GetQuotas(query *m.GetQuotasQuery) error { +func GetOrgQuotas(query *m.GetOrgQuotasQuery) error { quotas := make([]*m.Quota, 0) sess := x.Table("quota") - if err := sess.Where("org_id=?", query.OrgId).Find("as); err != nil { + if err := sess.Where("org_id=? AND user_id=0", query.OrgId).Find("as); err != nil { return err } - seenTargets := make(map[m.QuotaTarget]bool) + defaultQuotas := m.QuotaToMap(setting.Quota.Org) + + seenTargets := make(map[string]bool) for _, q := range quotas { seenTargets[q.Target] = true } - for t, v := range setting.Quota.Default { - if _, ok := seenTargets[m.QuotaTarget(t)]; !ok { + for t, v := range defaultQuotas { + if _, ok := seenTargets[t]; !ok { quotas = append(quotas, &m.Quota{ OrgId: query.OrgId, - Target: m.QuotaTarget(t), + Target: t, Limit: v, }) } } - result := make([]*m.QuotaDTO, len(quotas)) + + result := make([]*m.OrgQuotaDTO, len(quotas)) for i, q := range quotas { //get quota used. - rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(string(q.Target))) + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where org_id=?", dialect.Quote(q.Target)) resp := make([]*targetCount, 0) if err := x.Sql(rawSql, q.OrgId).Find(&resp); err != nil { return err } - result[i] = &m.QuotaDTO{ + result[i] = &m.OrgQuotaDTO{ Target: q.Target, Limit: q.Limit, OrgId: q.OrgId, @@ -86,7 +93,7 @@ func GetQuotas(query *m.GetQuotasQuery) error { return nil } -func UpdateQuota(cmd *m.UpdateQuotaCmd) error { +func UpdateOrgQuota(cmd *m.UpdateOrgQuotaCmd) error { return inTransaction2(func(sess *session) error { //Check if quota is already defined in the DB quota := m.Quota{ @@ -113,3 +120,120 @@ func UpdateQuota(cmd *m.UpdateQuotaCmd) error { return nil }) } + +func GetUserQuotaByTarget(query *m.GetUserQuotaByTargetQuery) error { + quota := m.Quota{ + Target: query.Target, + UserId: query.UserId, + } + has, err := x.Get(quota) + if err != nil { + return err + } else if has == false { + quota.Limit = query.Default + } + + //get quota used. + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where user_id=?", dialect.Quote(query.Target)) + resp := make([]*targetCount, 0) + if err := x.Sql(rawSql, query.UserId).Find(&resp); err != nil { + return err + } + + query.Result = &m.UserQuotaDTO{ + Target: query.Target, + Limit: quota.Limit, + UserId: query.UserId, + Used: resp[0].Count, + } + + return nil +} + +func GetUserQuotas(query *m.GetUserQuotasQuery) error { + quotas := make([]*m.Quota, 0) + sess := x.Table("quota") + if err := sess.Where("user_id=? AND org_id=0", query.UserId).Find("as); err != nil { + return err + } + + defaultQuotas := m.QuotaToMap(setting.Quota.User) + + seenTargets := make(map[string]bool) + for _, q := range quotas { + seenTargets[q.Target] = true + } + + for t, v := range defaultQuotas { + if _, ok := seenTargets[t]; !ok { + quotas = append(quotas, &m.Quota{ + UserId: query.UserId, + Target: t, + Limit: v, + }) + } + } + + result := make([]*m.UserQuotaDTO, len(quotas)) + for i, q := range quotas { + //get quota used. + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s where user_id=?", dialect.Quote(q.Target)) + resp := make([]*targetCount, 0) + if err := x.Sql(rawSql, q.UserId).Find(&resp); err != nil { + return err + } + result[i] = &m.UserQuotaDTO{ + Target: q.Target, + Limit: q.Limit, + UserId: q.UserId, + Used: resp[0].Count, + } + } + query.Result = result + return nil +} + +func UpdateUserQuota(cmd *m.UpdateUserQuotaCmd) error { + return inTransaction2(func(sess *session) error { + //Check if quota is already defined in the DB + quota := m.Quota{ + Target: cmd.Target, + UserId: cmd.UserId, + } + has, err := sess.Get(quota) + if err != nil { + return err + } + quota.Limit = cmd.Limit + if has == false { + //No quota in the DB for this target, so create a new one. + if _, err := sess.Insert("a); err != nil { + return err + } + } else { + //update existing quota entry in the DB. + if _, err := sess.Id(quota.Id).Update("a); err != nil { + return err + } + } + + return nil + }) +} + +func GetGlobalQuotaByTarget(query *m.GetGlobalQuotaByTargetQuery) error { + //get quota used. + rawSql := fmt.Sprintf("SELECT COUNT(*) as count from %s", dialect.Quote(query.Target)) + resp := make([]*targetCount, 0) + if err := x.Sql(rawSql).Find(&resp); err != nil { + return err + } + + query.Result = &m.GlobalQuotaDTO{ + Target: query.Target, + Limit: query.Default, + Used: resp[0].Count, + } + + return nil +} diff --git a/pkg/setting/setting_quota.go b/pkg/setting/setting_quota.go index d6ca0cd7ba8..df412fe23e4 100644 --- a/pkg/setting/setting_quota.go +++ b/pkg/setting/setting_quota.go @@ -1,17 +1,58 @@ package setting +type OrgQuota struct { + User int64 `target:"org_user"` + DataSource int64 `target:"data_source"` + Dashboard int64 `target:"dashboard"` + ApiKey int64 `target:"api_key"` +} + +type UserQuota struct { + Org int64 `target:"org_user"` +} + +type GlobalQuota struct { + Org int64 `target:"org"` + User int64 `target:"user"` + DataSource int64 `target:"data_source"` + Dashboard int64 `target:"dashboard"` + ApiKey int64 `target:"api_key"` + Session int64 `target:"-"` +} + type QuotaSettings struct { Enabled bool - Default map[string]int64 + Org *OrgQuota + User *UserQuota + Global *GlobalQuota } func readQuotaSettings() { // set global defaults. - DefaultQuotas := make(map[string]int64) quota := Cfg.Section("quota") Quota.Enabled = quota.Key("enabled").MustBool(false) - DefaultQuotas["user"] = quota.Key("user").MustInt64(10) - DefaultQuotas["data_source"] = quota.Key("data_source").MustInt64(10) - DefaultQuotas["dashboard"] = quota.Key("dashboard").MustInt64(10) - Quota.Default = DefaultQuotas + + // per ORG Limits + Quota.Org = &OrgQuota{ + User: quota.Key("org_user").MustInt64(10), + DataSource: quota.Key("org_data_source").MustInt64(10), + Dashboard: quota.Key("org_dashboard").MustInt64(10), + ApiKey: quota.Key("org_api_key").MustInt64(10), + } + + // per User limits + Quota.User = &UserQuota{ + Org: quota.Key("user_org").MustInt64(10), + } + + // Global Limits + Quota.Global = &GlobalQuota{ + User: quota.Key("global_user").MustInt64(-1), + Org: quota.Key("global_org").MustInt64(-1), + DataSource: quota.Key("global_data_source").MustInt64(-1), + Dashboard: quota.Key("global_dashboard").MustInt64(-1), + ApiKey: quota.Key("global_api_key").MustInt64(-1), + Session: quota.Key("global_session").MustInt64(-1), + } + } From 3cf2cd4684d03d52415ca6cb16ebbba89c809ed9 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 15 Sep 2015 17:10:16 +0800 Subject: [PATCH 10/16] be sure to pass result obj by reference to xorm. --- pkg/services/sqlstore/quota.go | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index 23bea264e87..afc3c028dcf 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -26,7 +26,7 @@ func GetOrgQuotaByTarget(query *m.GetOrgQuotaByTargetQuery) error { Target: query.Target, OrgId: query.OrgId, } - has, err := x.Get(quota) + has, err := x.Get("a) if err != nil { return err } else if has == false { @@ -100,7 +100,7 @@ func UpdateOrgQuota(cmd *m.UpdateOrgQuotaCmd) error { Target: cmd.Target, OrgId: cmd.OrgId, } - has, err := sess.Get(quota) + has, err := sess.Get("a) if err != nil { return err } @@ -126,7 +126,7 @@ func GetUserQuotaByTarget(query *m.GetUserQuotaByTargetQuery) error { Target: query.Target, UserId: query.UserId, } - has, err := x.Get(quota) + has, err := x.Get("a) if err != nil { return err } else if has == false { @@ -200,7 +200,7 @@ func UpdateUserQuota(cmd *m.UpdateUserQuotaCmd) error { Target: cmd.Target, UserId: cmd.UserId, } - has, err := sess.Get(quota) + has, err := sess.Get("a) if err != nil { return err } From b7de847236a1388f43e08922830d784026e2107f Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 15 Sep 2015 17:10:46 +0800 Subject: [PATCH 11/16] add unittests for quota sqltore methods. --- pkg/services/sqlstore/quota_test.go | 171 ++++++++++++++++++++++++++++ 1 file changed, 171 insertions(+) create mode 100644 pkg/services/sqlstore/quota_test.go diff --git a/pkg/services/sqlstore/quota_test.go b/pkg/services/sqlstore/quota_test.go new file mode 100644 index 00000000000..5ef618e166d --- /dev/null +++ b/pkg/services/sqlstore/quota_test.go @@ -0,0 +1,171 @@ +package sqlstore + +import ( + "testing" + + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" +) + +func TestQuotaCommandsAndQueries(t *testing.T) { + + Convey("Testing Qutoa commands & queries", t, func() { + InitTestDB(t) + userId := int64(1) + orgId := int64(0) + + setting.Quota = setting.QuotaSettings{ + Enabled: true, + Org: &setting.OrgQuota{ + User: 5, + Dashboard: 5, + DataSource: 5, + ApiKey: 5, + }, + User: &setting.UserQuota{ + Org: 5, + }, + Global: &setting.GlobalQuota{ + Org: 5, + User: 5, + Dashboard: 5, + DataSource: 5, + ApiKey: 5, + Session: 5, + }, + } + + // create a new org and add user_id 1 as admin. + // we will then have an org with 1 user. and a user + // with 1 org. + userCmd := m.CreateOrgCommand{ + Name: "TestOrg", + UserId: 1, + } + err := CreateOrg(&userCmd) + So(err, ShouldBeNil) + orgId = userCmd.Result.Id + + Convey("Given saved org quota for users", func() { + orgCmd := m.UpdateOrgQuotaCmd{ + OrgId: orgId, + Target: "org_user", + Limit: 10, + } + err := UpdateOrgQuota(&orgCmd) + So(err, ShouldBeNil) + + Convey("Should be able to get saved quota by org id and target", func() { + query := m.GetOrgQuotaByTargetQuery{OrgId: orgId, Target: "org_user", Default: 1} + err = GetOrgQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Limit, ShouldEqual, 10) + }) + Convey("Should be able to get default quota by org id and target", func() { + query := m.GetOrgQuotaByTargetQuery{OrgId: 123, Target: "org_user", Default: 11} + err = GetOrgQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Limit, ShouldEqual, 11) + }) + Convey("Should be able to get used org quota when rows exist", func() { + query := m.GetOrgQuotaByTargetQuery{OrgId: orgId, Target: "org_user", Default: 11} + err = GetOrgQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Used, ShouldEqual, 1) + }) + Convey("Should be able to get used org quota when no rows exist", func() { + query := m.GetOrgQuotaByTargetQuery{OrgId: 2, Target: "org_user", Default: 11} + err = GetOrgQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Used, ShouldEqual, 0) + }) + Convey("Should be able to quota list for org", func() { + query := m.GetOrgQuotasQuery{OrgId: orgId} + err = GetOrgQuotas(&query) + + So(err, ShouldBeNil) + So(len(query.Result), ShouldEqual, 4) + for _, res := range query.Result { + limit := 5 //default quota limit + used := 0 + if res.Target == "org_user" { + limit = 10 //customized quota limit. + used = 1 + } + So(res.Limit, ShouldEqual, limit) + So(res.Used, ShouldEqual, used) + + } + }) + }) + Convey("Given saved user quota for org", func() { + userQoutaCmd := m.UpdateUserQuotaCmd{ + UserId: userId, + Target: "org_user", + Limit: 10, + } + err := UpdateUserQuota(&userQoutaCmd) + So(err, ShouldBeNil) + + Convey("Should be able to get saved quota by user id and target", func() { + query := m.GetUserQuotaByTargetQuery{UserId: userId, Target: "org_user", Default: 1} + err = GetUserQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Limit, ShouldEqual, 10) + }) + Convey("Should be able to get default quota by user id and target", func() { + query := m.GetUserQuotaByTargetQuery{UserId: 9, Target: "org_user", Default: 11} + err = GetUserQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Limit, ShouldEqual, 11) + }) + Convey("Should be able to get used user quota when rows exist", func() { + query := m.GetUserQuotaByTargetQuery{UserId: userId, Target: "org_user", Default: 11} + err = GetUserQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Used, ShouldEqual, 1) + }) + Convey("Should be able to get used user quota when no rows exist", func() { + query := m.GetUserQuotaByTargetQuery{UserId: 2, Target: "org_user", Default: 11} + err = GetUserQuotaByTarget(&query) + + So(err, ShouldBeNil) + So(query.Result.Used, ShouldEqual, 0) + }) + Convey("Should be able to quota list for user", func() { + query := m.GetUserQuotasQuery{UserId: userId} + err = GetUserQuotas(&query) + + So(err, ShouldBeNil) + So(len(query.Result), ShouldEqual, 1) + So(query.Result[0].Limit, ShouldEqual, 10) + So(query.Result[0].Used, ShouldEqual, 1) + }) + }) + + Convey("Should be able to global user quota", func() { + query := m.GetGlobalQuotaByTargetQuery{Target: "user", Default: 5} + err = GetGlobalQuotaByTarget(&query) + So(err, ShouldBeNil) + + So(query.Result.Limit, ShouldEqual, 5) + So(query.Result.Used, ShouldEqual, 0) + }) + Convey("Should be able to global org quota", func() { + query := m.GetGlobalQuotaByTargetQuery{Target: "org", Default: 5} + err = GetGlobalQuotaByTarget(&query) + So(err, ShouldBeNil) + + So(query.Result.Limit, ShouldEqual, 5) + So(query.Result.Used, ShouldEqual, 1) + }) + }) +} From 86ed85aa6e66916b5cdc1068fb9d49b245a360fb Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 15 Sep 2015 17:18:26 +0800 Subject: [PATCH 12/16] move toMap function to be a method on the quota structs --- pkg/models/quotas.go | 24 ----------------------- pkg/services/sqlstore/quota.go | 4 ++-- pkg/setting/setting_quota.go | 36 ++++++++++++++++++++++++++++++++++ 3 files changed, 38 insertions(+), 26 deletions(-) diff --git a/pkg/models/quotas.go b/pkg/models/quotas.go index ba17b848403..85159d830d1 100644 --- a/pkg/models/quotas.go +++ b/pkg/models/quotas.go @@ -3,7 +3,6 @@ package models import ( "errors" "github.com/grafana/grafana/pkg/setting" - "reflect" "time" ) @@ -129,26 +128,3 @@ func GetQuotaScopes(target string) ([]QuotaScope, error) { return scopes, ErrInvalidQuotaTarget } } - -func QuotaToMap(q interface{}) map[string]int64 { - qMap := make(map[string]int64) - typ := reflect.TypeOf(q) - val := reflect.ValueOf(q) - if typ.Kind() == reflect.Ptr { - typ = typ.Elem() - val = val.Elem() - } - for i := 0; i < typ.NumField(); i++ { - field := typ.Field(i) - name := field.Tag.Get("target") - if name == "" { - name = field.Name - } - if name == "-" { - continue - } - value := val.Field(i) - qMap[name] = value.Int() - } - return qMap -} diff --git a/pkg/services/sqlstore/quota.go b/pkg/services/sqlstore/quota.go index afc3c028dcf..53ea8889c56 100644 --- a/pkg/services/sqlstore/quota.go +++ b/pkg/services/sqlstore/quota.go @@ -57,7 +57,7 @@ func GetOrgQuotas(query *m.GetOrgQuotasQuery) error { return err } - defaultQuotas := m.QuotaToMap(setting.Quota.Org) + defaultQuotas := setting.Quota.Org.ToMap() seenTargets := make(map[string]bool) for _, q := range quotas { @@ -157,7 +157,7 @@ func GetUserQuotas(query *m.GetUserQuotasQuery) error { return err } - defaultQuotas := m.QuotaToMap(setting.Quota.User) + defaultQuotas := setting.Quota.User.ToMap() seenTargets := make(map[string]bool) for _, q := range quotas { diff --git a/pkg/setting/setting_quota.go b/pkg/setting/setting_quota.go index df412fe23e4..49769d9930f 100644 --- a/pkg/setting/setting_quota.go +++ b/pkg/setting/setting_quota.go @@ -1,5 +1,9 @@ package setting +import ( + "reflect" +) + type OrgQuota struct { User int64 `target:"org_user"` DataSource int64 `target:"data_source"` @@ -20,6 +24,38 @@ type GlobalQuota struct { Session int64 `target:"-"` } +func (q *OrgQuota) ToMap() map[string]int64 { + return quotaToMap(*q) +} + +func (q *UserQuota) ToMap() map[string]int64 { + return quotaToMap(*q) +} + +func (q *GlobalQuota) ToMap() map[string]int64 { + return quotaToMap(*q) +} + +func quotaToMap(q interface{}) map[string]int64 { + qMap := make(map[string]int64) + typ := reflect.TypeOf(q) + val := reflect.ValueOf(q) + + for i := 0; i < typ.NumField(); i++ { + field := typ.Field(i) + name := field.Tag.Get("target") + if name == "" { + name = field.Name + } + if name == "-" { + continue + } + value := val.Field(i) + qMap[name] = value.Int() + } + return qMap +} + type QuotaSettings struct { Enabled bool Org *OrgQuota From 1ad10914ce614b261bd6059dbcea01dac33eace8 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 15 Sep 2015 18:19:47 +0800 Subject: [PATCH 13/16] add quota middleware unittests --- pkg/middleware/quota_test.go | 144 +++++++++++++++++++++++++++++++++++ 1 file changed, 144 insertions(+) create mode 100644 pkg/middleware/quota_test.go diff --git a/pkg/middleware/quota_test.go b/pkg/middleware/quota_test.go new file mode 100644 index 00000000000..ae6b414b248 --- /dev/null +++ b/pkg/middleware/quota_test.go @@ -0,0 +1,144 @@ +package middleware + +import ( + "github.com/grafana/grafana/pkg/bus" + m "github.com/grafana/grafana/pkg/models" + "github.com/grafana/grafana/pkg/setting" + . "github.com/smartystreets/goconvey/convey" + "testing" +) + +func TestMiddlewareQuota(t *testing.T) { + + Convey("Given the grafana quota middleware", t, func() { + setting.Quota = setting.QuotaSettings{ + Enabled: true, + Org: &setting.OrgQuota{ + User: 5, + Dashboard: 5, + DataSource: 5, + ApiKey: 5, + }, + User: &setting.UserQuota{ + Org: 5, + }, + Global: &setting.GlobalQuota{ + Org: 5, + User: 5, + Dashboard: 5, + DataSource: 5, + ApiKey: 5, + Session: 5, + }, + } + + middlewareScenario("with user not logged in", func(sc *scenarioContext) { + bus.AddHandler("globalQuota", func(query *m.GetGlobalQuotaByTargetQuery) error { + query.Result = &m.GlobalQuotaDTO{ + Target: query.Target, + Limit: query.Default, + Used: 4, + } + return nil + }) + Convey("global quota not reached", func() { + sc.m.Get("/user", Quota("user"), sc.defaultHandler) + sc.fakeReq("GET", "/user").exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + Convey("global quota reached", func() { + setting.Quota.Global.User = 4 + sc.m.Get("/user", Quota("user"), sc.defaultHandler) + sc.fakeReq("GET", "/user").exec() + So(sc.resp.Code, ShouldEqual, 403) + }) + Convey("global session quota not reached", func() { + setting.Quota.Global.Session = 10 + sc.m.Get("/user", Quota("session"), sc.defaultHandler) + sc.fakeReq("GET", "/user").exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + Convey("global session quota reached", func() { + setting.Quota.Global.Session = 1 + sc.m.Get("/user", Quota("session"), sc.defaultHandler) + sc.fakeReq("GET", "/user").exec() + So(sc.resp.Code, ShouldEqual, 403) + }) + }) + + middlewareScenario("with user logged in", func(sc *scenarioContext) { + // log us in, so we have a user_id and org_id in the context + sc.fakeReq("GET", "/").handler(func(c *Context) { + c.Session.Set(SESS_KEY_USERID, int64(12)) + }).exec() + + bus.AddHandler("test", func(query *m.GetSignedInUserQuery) error { + query.Result = &m.SignedInUser{OrgId: 2, UserId: 12} + return nil + }) + bus.AddHandler("globalQuota", func(query *m.GetGlobalQuotaByTargetQuery) error { + query.Result = &m.GlobalQuotaDTO{ + Target: query.Target, + Limit: query.Default, + Used: 4, + } + return nil + }) + bus.AddHandler("userQuota", func(query *m.GetUserQuotaByTargetQuery) error { + query.Result = &m.UserQuotaDTO{ + Target: query.Target, + Limit: query.Default, + Used: 4, + } + return nil + }) + bus.AddHandler("orgQuota", func(query *m.GetOrgQuotaByTargetQuery) error { + query.Result = &m.OrgQuotaDTO{ + Target: query.Target, + Limit: query.Default, + Used: 4, + } + return nil + }) + Convey("global datasource quota reached", func() { + setting.Quota.Global.DataSource = 4 + sc.m.Get("/ds", Quota("data_source"), sc.defaultHandler) + sc.fakeReq("GET", "/ds").exec() + So(sc.resp.Code, ShouldEqual, 403) + }) + Convey("user Org quota not reached", func() { + setting.Quota.User.Org = 5 + sc.m.Get("/org", Quota("org"), sc.defaultHandler) + sc.fakeReq("GET", "/org").exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + Convey("user Org quota reached", func() { + setting.Quota.User.Org = 4 + sc.m.Get("/org", Quota("org"), sc.defaultHandler) + sc.fakeReq("GET", "/org").exec() + So(sc.resp.Code, ShouldEqual, 403) + }) + Convey("org dashboard quota not reached", func() { + setting.Quota.Org.Dashboard = 10 + sc.m.Get("/dashboard", Quota("dashboard"), sc.defaultHandler) + sc.fakeReq("GET", "/dashboard").exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + Convey("org dashboard quota reached", func() { + setting.Quota.Org.Dashboard = 4 + sc.m.Get("/dashboard", Quota("dashboard"), sc.defaultHandler) + sc.fakeReq("GET", "/dashboard").exec() + So(sc.resp.Code, ShouldEqual, 403) + }) + Convey("org dashboard quota reached but quotas disabled", func() { + setting.Quota.Org.Dashboard = 4 + setting.Quota.Enabled = false + sc.m.Get("/dashboard", Quota("dashboard"), sc.defaultHandler) + sc.fakeReq("GET", "/dashboard").exec() + So(sc.resp.Code, ShouldEqual, 200) + }) + + }) + + }) +} From 3926226417f98955090bf361ebf5c063926eb273 Mon Sep 17 00:00:00 2001 From: woodsaj Date: Tue, 15 Sep 2015 20:31:58 +0800 Subject: [PATCH 14/16] fix getting default quota as map[string]int64 --- pkg/api/quota.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pkg/api/quota.go b/pkg/api/quota.go index 490a8e90729..d8585435430 100644 --- a/pkg/api/quota.go +++ b/pkg/api/quota.go @@ -27,7 +27,7 @@ func UpdateOrgQuota(c *middleware.Context, cmd m.UpdateOrgQuotaCmd) Response { cmd.OrgId = c.ParamsInt64(":orgId") cmd.Target = c.Params(":target") - if _, ok := m.QuotaToMap(setting.Quota.Org)[cmd.Target]; !ok { + if _, ok := setting.Quota.Org.ToMap()[cmd.Target]; !ok { return ApiError(404, "Invalid quota target", nil) } @@ -57,7 +57,7 @@ func UpdateUserQuota(c *middleware.Context, cmd m.UpdateUserQuotaCmd) Response { cmd.UserId = c.ParamsInt64(":id") cmd.Target = c.Params(":target") - if _, ok := m.QuotaToMap(setting.Quota.User)[cmd.Target]; !ok { + if _, ok := setting.Quota.User.ToMap()[cmd.Target]; !ok { return ApiError(404, "Invalid quota target", nil) } From 10c099a52db930152dde31aee9b742981cbc7815 Mon Sep 17 00:00:00 2001 From: Elliot Anderson Date: Wed, 16 Sep 2015 11:02:50 +1000 Subject: [PATCH 15/16] Fixed 404 error for robots.txt --- pkg/cmd/web.go | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pkg/cmd/web.go b/pkg/cmd/web.go index c94661a5f9a..69843b6b095 100644 --- a/pkg/cmd/web.go +++ b/pkg/cmd/web.go @@ -33,7 +33,7 @@ func newMacaron() *macaron.Macaron { mapStatic(m, "css", "css") mapStatic(m, "img", "img") mapStatic(m, "fonts", "fonts") - mapStatic(m, "robots.txt", "robots.txxt") + mapStatic(m, "robots.txt", "robots.txt") m.Use(macaron.Renderer(macaron.RenderOptions{ Directory: path.Join(setting.StaticRootPath, "views"), From 1f959272c5f719701f9371fb2a0f597558a2ef64 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Torkel=20=C3=96degaard?= Date: Wed, 16 Sep 2015 16:28:41 +0200 Subject: [PATCH 16/16] feat(migration): added back support to import old dashboard from from Elasticsearch --- .../datasource/elasticsearch/datasource.js | 36 +++++++++++++++++++ 1 file changed, 36 insertions(+) diff --git a/public/app/plugins/datasource/elasticsearch/datasource.js b/public/app/plugins/datasource/elasticsearch/datasource.js index 3b05bbdcbe9..eb6a8d63ea2 100644 --- a/public/app/plugins/datasource/elasticsearch/datasource.js +++ b/public/app/plugins/datasource/elasticsearch/datasource.js @@ -215,6 +215,42 @@ function (angular, _, moment, kbn, ElasticQueryBuilder, IndexPattern, ElasticRes }); }; + ElasticDatasource.prototype.getDashboard = function(id) { + return this._get('/dashboard/' + id) + .then(function(result) { + return angular.fromJson(result._source.dashboard); + }); + }; + + ElasticDatasource.prototype.searchDashboards = function() { + var query = { + query: { query_string: { query: '*' } }, + size: 10000, + sort: ["_uid"], + }; + + return this._post(this.index + '/dashboard/_search', query) + .then(function(results) { + if(_.isUndefined(results.hits)) { + return { dashboards: [], tags: [] }; + } + + var resultsHits = results.hits.hits; + var displayHits = { dashboards: [] }; + + for (var i = 0, len = resultsHits.length; i < len; i++) { + var hit = resultsHits[i]; + displayHits.dashboards.push({ + id: hit._id, + title: hit._source.title, + tags: hit._source.tags + }); + } + + return displayHits; + }); + }; + return ElasticDatasource; }); });