From a3af6023c3ecd216ab6066a0fa58c3e2f611f5a7 Mon Sep 17 00:00:00 2001 From: Mihai Turdean <6640685+mihai-turdean@users.noreply.github.com> Date: Thu, 10 Jul 2025 13:51:02 -0600 Subject: [PATCH] Zanzana: Support more cache configuration options (#107155) --- .../authz/zanzana/server/openfga_server.go | 16 ++++- pkg/services/authz/zanzana/server/server.go | 2 +- .../authz/zanzana/server/server_list.go | 2 +- pkg/setting/settings_zanzana.go | 72 +++++++++++++++++-- 4 files changed, 82 insertions(+), 10 deletions(-) diff --git a/pkg/services/authz/zanzana/server/openfga_server.go b/pkg/services/authz/zanzana/server/openfga_server.go index 3df2d944604..9baaafc017f 100644 --- a/pkg/services/authz/zanzana/server/openfga_server.go +++ b/pkg/services/authz/zanzana/server/openfga_server.go @@ -32,9 +32,21 @@ func NewOpenFGAServer(cfg setting.ZanzanaServerSettings, store storage.OpenFGADa opts := []server.OpenFGAServiceV1Option{ server.WithDatastore(store), server.WithLogger(zlogger.New(logger)), - server.WithCheckQueryCacheEnabled(cfg.CheckQueryCache), - server.WithCheckQueryCacheTTL(cfg.CheckQueryCacheTTL), + server.WithCheckCacheLimit(cfg.CacheSettings.CheckCacheLimit), + server.WithCacheControllerEnabled(cfg.CacheSettings.CacheControllerEnabled), + server.WithCacheControllerTTL(cfg.CacheSettings.CacheControllerTTL), + server.WithCheckQueryCacheEnabled(cfg.CacheSettings.CheckQueryCacheEnabled), + server.WithCheckQueryCacheTTL(cfg.CacheSettings.CheckQueryCacheTTL), + server.WithCheckIteratorCacheEnabled(cfg.CacheSettings.CheckIteratorCacheEnabled), + server.WithCheckIteratorCacheMaxResults(cfg.CacheSettings.CheckIteratorCacheMaxResults), + server.WithCheckIteratorCacheTTL(cfg.CacheSettings.CheckIteratorCacheTTL), server.WithListObjectsMaxResults(cfg.ListObjectsMaxResults), + server.WithListObjectsIteratorCacheEnabled(cfg.CacheSettings.ListObjectsIteratorCacheEnabled), + server.WithListObjectsIteratorCacheMaxResults(cfg.CacheSettings.ListObjectsIteratorCacheMaxResults), + server.WithListObjectsIteratorCacheTTL(cfg.CacheSettings.ListObjectsIteratorCacheTTL), + server.WithSharedIteratorEnabled(cfg.CacheSettings.SharedIteratorEnabled), + server.WithSharedIteratorLimit(cfg.CacheSettings.SharedIteratorLimit), + server.WithSharedIteratorTTL(cfg.CacheSettings.SharedIteratorTTL), server.WithListObjectsDeadline(cfg.ListObjectsDeadline), } diff --git a/pkg/services/authz/zanzana/server/server.go b/pkg/services/authz/zanzana/server/server.go index 6803fdb594f..c495f0504ac 100644 --- a/pkg/services/authz/zanzana/server/server.go +++ b/pkg/services/authz/zanzana/server/server.go @@ -62,7 +62,7 @@ func NewServer(cfg setting.ZanzanaServerSettings, openfga OpenFGAServer, logger storesMU: &sync.Mutex{}, stores: make(map[string]storeInfo), cfg: cfg, - cache: localcache.New(cfg.CheckQueryCacheTTL, cacheCleanInterval), + cache: localcache.New(cfg.CacheSettings.CheckQueryCacheTTL, cacheCleanInterval), logger: logger, tracer: tracer, } diff --git a/pkg/services/authz/zanzana/server/server_list.go b/pkg/services/authz/zanzana/server/server_list.go index ee08be863f7..d81c9a9f1f1 100644 --- a/pkg/services/authz/zanzana/server/server_list.go +++ b/pkg/services/authz/zanzana/server/server_list.go @@ -178,7 +178,7 @@ func (s *Server) listObjects(ctx context.Context, req *openfgav1.ListObjectsRequ fn = s.streamedListObjects } - if s.cfg.CheckQueryCache { + if s.cfg.CacheSettings.CheckQueryCacheEnabled { return s.listObjectCached(ctx, req, fn) } diff --git a/pkg/setting/settings_zanzana.go b/pkg/setting/settings_zanzana.go index 3b4fff83b78..be7121f3b0d 100644 --- a/pkg/setting/settings_zanzana.go +++ b/pkg/setting/settings_zanzana.go @@ -33,10 +33,8 @@ type ZanzanaServerSettings struct { // OpenFGA http server address which allows to connect with fga cli. // Can only be used in dev mode. OpenFGAHttpAddr string - // Enable cache for Check() requests - CheckQueryCache bool - // TTL for cached requests. Default is 10 seconds. - CheckQueryCacheTTL time.Duration + // Cache settings + CacheSettings OpenFgaCacheSettings // Max number of results returned by ListObjects() query. Default is 1000. ListObjectsMaxResults uint32 // Deadline for the ListObjects() query. Default is 3 seconds. @@ -50,6 +48,54 @@ type ZanzanaServerSettings struct { AllowInsecure bool } +// Parameters to configure OpenFGA cache. +type OpenFgaCacheSettings struct { + // Number of items that will be kept in the in-memory cache used to resolve Check queries. + // If CheckQueryCacheEnabled or CheckIteratorCacheEnabled, this is the size limit of the cache + CheckCacheLimit uint32 + // When enabled, cache controller will verify whether check subproblem cache and check + // iterator cache needs to be invalidated when there is a check or list objects API request. + // The invalidation determination is based on whether there are recent write or deletes for + // the store. This feature allows a larger check-query-cache-ttl and check-iterator-cache-ttl + // at the expense of additional datastore queries for recent writes and deletes. + CacheControllerEnabled bool + // Specifies how frequently the cache controller checks for Writes occurring. While the + // cache controller result is cached, the server will not read the datastore to check + // whether subproblem cache and iterator cache needs to be invalidated. + CacheControllerTTL time.Duration + // Enables in-memory caching of Check subproblems. For example, if you have a relation + // `define viewer: owner or editor`, and the query is `Check(user:anne, viewer, doc:1)`, + // we'll evaluate the `owner` relation and the `editor` relation and cache both results: + // `(user:anne, viewer, doc:1) -> allowed=true` and `(user:anne, owner, doc:1) -> allowed=true`. + CheckQueryCacheEnabled bool + // Time that items will be kept in the cache of Check subproblems + CheckQueryCacheTTL time.Duration + // Enables in-memory caching of database iterators. Each iterator is the result of a database + // query, for example, usersets related to a specific object, or objects related to a specific + // user, up to a certain number of tuples per iterator + CheckIteratorCacheEnabled bool + // Number of tuples that will be stored for each database iterator + CheckIteratorCacheMaxResults uint32 + // Time that items will be kept in the cache of database iterators + CheckIteratorCacheTTL time.Duration + // Enables in-memory caching of database iterators for ListObjects. Each iterator is the result + // of a database query, for example, usersets related to a specific object, or objects related + // to a specific user, up to a certain number of tuples per iterator + ListObjectsIteratorCacheEnabled bool + // Number of tuples that will be stored for each ListObjects database iterator + ListObjectsIteratorCacheMaxResults uint32 + // Time that items will be kept in the cache of ListObjects database iterators + ListObjectsIteratorCacheTTL time.Duration + // Enables sharing of datastore iterators with different consumers. Each iterator is the result + // of a database query, for example usersets related to a specific object, or objects related + // to a specific user, up to a certain number of tuples per iterator + SharedIteratorEnabled bool + // Limit of the number of iterators that can be shared when shared iterator is enabled + SharedIteratorLimit uint32 + // Time that shared iterators will be kept in the cache + SharedIteratorTTL time.Duration +} + func (cfg *Cfg) readZanzanaSettings() { zc := ZanzanaClientSettings{} clientSec := cfg.SectionWithEnvOverrides("zanzana.client") @@ -73,13 +119,27 @@ func (cfg *Cfg) readZanzanaSettings() { serverSec := cfg.SectionWithEnvOverrides("zanzana.server") zs.OpenFGAHttpAddr = serverSec.Key("http_addr").MustString("127.0.0.1:8080") - zs.CheckQueryCache = serverSec.Key("check_query_cache").MustBool(true) - zs.CheckQueryCacheTTL = serverSec.Key("check_query_cache_ttl").MustDuration(10 * time.Second) zs.ListObjectsDeadline = serverSec.Key("list_objects_deadline").MustDuration(3 * time.Second) zs.ListObjectsMaxResults = uint32(serverSec.Key("list_objects_max_results").MustUint(1000)) zs.UseStreamedListObjects = serverSec.Key("use_streamed_list_objects").MustBool(false) zs.SigningKeysURL = serverSec.Key("signing_keys_url").MustString("") zs.AllowInsecure = serverSec.Key("allow_insecure").MustBool(false) + // Cache settings + zs.CacheSettings.CheckCacheLimit = uint32(serverSec.Key("check_cache_limit").MustUint(10000)) + zs.CacheSettings.CacheControllerEnabled = serverSec.Key("cache_controller_enabled").MustBool(false) + zs.CacheSettings.CacheControllerTTL = serverSec.Key("cache_controller_ttl").MustDuration(10 * time.Second) + zs.CacheSettings.CheckQueryCacheEnabled = serverSec.Key("check_query_cache_enabled").MustBool(true) + zs.CacheSettings.CheckQueryCacheTTL = serverSec.Key("check_query_cache_ttl").MustDuration(10 * time.Second) + zs.CacheSettings.CheckIteratorCacheEnabled = serverSec.Key("check_iterator_cache_enabled").MustBool(false) + zs.CacheSettings.CheckIteratorCacheMaxResults = uint32(serverSec.Key("check_iterator_cache_max_results").MustUint(1000)) + zs.CacheSettings.CheckIteratorCacheTTL = serverSec.Key("check_iterator_cache_ttl").MustDuration(10 * time.Second) + zs.CacheSettings.ListObjectsIteratorCacheEnabled = serverSec.Key("list_objects_iterator_cache_enabled").MustBool(false) + zs.CacheSettings.ListObjectsIteratorCacheMaxResults = uint32(serverSec.Key("list_objects_iterator_cache_max_results").MustUint(1000)) + zs.CacheSettings.ListObjectsIteratorCacheTTL = serverSec.Key("list_objects_iterator_cache_ttl").MustDuration(10 * time.Second) + zs.CacheSettings.SharedIteratorEnabled = serverSec.Key("shared_iterator_enabled").MustBool(false) + zs.CacheSettings.SharedIteratorLimit = uint32(serverSec.Key("shared_iterator_limit").MustUint(1000)) + zs.CacheSettings.SharedIteratorTTL = serverSec.Key("shared_iterator_ttl").MustDuration(10 * time.Second) + cfg.ZanzanaServer = zs }