mirror of https://github.com/grafana/loki
Invalidate caches on delete (#5661)
* Generate cache invalidation numbers in the delete store * Get cache generation numbers from the store on request * changlog * rename tombstones to something more meaningful * User invisible module * query frontend relies on a compactor to get the cache generation number * fix serialization * source -> name * lint errors * lint errors * log non-200 responses * add jsonnet changes * lint * review feedback * review feedback * client renamepull/5719/head^2
Travis Patterson
4 years ago
committed by
GitHub
parent
3fa6cc9fde
commit
9cef86b162
@ -0,0 +1,81 @@ |
||||
package generationnumber |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
"net/http" |
||||
"net/url" |
||||
|
||||
"github.com/go-kit/log/level" |
||||
|
||||
"github.com/grafana/loki/pkg/util/log" |
||||
) |
||||
|
||||
const ( |
||||
orgHeaderKey = "X-Scope-OrgID" |
||||
cacheGenNumPath = "/loki/api/v1/cache/generation_numbers" |
||||
) |
||||
|
||||
type CacheGenClient interface { |
||||
GetCacheGenerationNumber(ctx context.Context, userID string) (string, error) |
||||
Name() string |
||||
} |
||||
|
||||
type genNumberClient struct { |
||||
url string |
||||
httpClient doer |
||||
} |
||||
|
||||
type doer interface { |
||||
Do(*http.Request) (*http.Response, error) |
||||
} |
||||
|
||||
func NewGenNumberClient(addr string, c doer) (CacheGenClient, error) { |
||||
u, err := url.Parse(addr) |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error parsing url", "err", err) |
||||
return nil, err |
||||
} |
||||
u.Path = cacheGenNumPath |
||||
|
||||
return &genNumberClient{ |
||||
url: u.String(), |
||||
httpClient: c, |
||||
}, nil |
||||
} |
||||
|
||||
func (c *genNumberClient) Name() string { |
||||
return "gen_number_client" |
||||
} |
||||
|
||||
func (c *genNumberClient) GetCacheGenerationNumber(ctx context.Context, userID string) (string, error) { |
||||
req, err := http.NewRequestWithContext(ctx, http.MethodGet, c.url, nil) |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error getting cache gen numbers from the store", "err", err) |
||||
return "", err |
||||
} |
||||
|
||||
req.Header.Set(orgHeaderKey, userID) |
||||
|
||||
resp, err := c.httpClient.Do(req) |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error getting cache gen numbers from the store", "err", err) |
||||
return "", err |
||||
} |
||||
defer resp.Body.Close() |
||||
|
||||
if resp.StatusCode < 200 || resp.StatusCode >= 300 { |
||||
err := fmt.Errorf("unexpected status code: %d", resp.StatusCode) |
||||
level.Error(log.Logger).Log("msg", "error getting cache gen numbers from the store", "err", err) |
||||
return "", err |
||||
} |
||||
|
||||
var genNumber string |
||||
if err := json.NewDecoder(resp.Body).Decode(&genNumber); err != nil { |
||||
level.Error(log.Logger).Log("msg", "error marshalling response", "err", err) |
||||
return "", err |
||||
} |
||||
|
||||
return genNumber, err |
||||
} |
||||
@ -0,0 +1,40 @@ |
||||
package generationnumber |
||||
|
||||
import ( |
||||
"context" |
||||
"io" |
||||
"net/http" |
||||
"strings" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestGetCacheGenNumberForUser(t *testing.T) { |
||||
httpClient := &mockHTTPClient{ret: `"42"`} |
||||
client, err := NewGenNumberClient("http://test-server", httpClient) |
||||
require.Nil(t, err) |
||||
|
||||
cacheGenNumber, err := client.GetCacheGenerationNumber(context.Background(), "userID") |
||||
require.Nil(t, err) |
||||
|
||||
require.Equal(t, "42", cacheGenNumber) |
||||
|
||||
require.Equal(t, "http://test-server/loki/api/v1/cache/generation_numbers", httpClient.req.URL.String()) |
||||
require.Equal(t, http.MethodGet, httpClient.req.Method) |
||||
require.Equal(t, "userID", httpClient.req.Header.Get("X-Scope-OrgID")) |
||||
} |
||||
|
||||
type mockHTTPClient struct { |
||||
ret string |
||||
req *http.Request |
||||
} |
||||
|
||||
func (c *mockHTTPClient) Do(req *http.Request) (*http.Response, error) { |
||||
c.req = req |
||||
|
||||
return &http.Response{ |
||||
StatusCode: 200, |
||||
Body: io.NopCloser(strings.NewReader(c.ret)), |
||||
}, nil |
||||
} |
||||
@ -0,0 +1,153 @@ |
||||
package generationnumber |
||||
|
||||
import ( |
||||
"context" |
||||
"fmt" |
||||
"strconv" |
||||
"sync" |
||||
"time" |
||||
|
||||
"github.com/go-kit/log/level" |
||||
|
||||
"github.com/grafana/loki/pkg/util/log" |
||||
|
||||
"github.com/prometheus/client_golang/prometheus" |
||||
) |
||||
|
||||
const reloadDuration = 5 * time.Minute |
||||
|
||||
type GenNumberLoader struct { |
||||
numberGetter CacheGenClient |
||||
numbers map[string]string |
||||
quit chan struct{} |
||||
lock sync.RWMutex |
||||
metrics *genLoaderMetrics |
||||
} |
||||
|
||||
func NewGenNumberLoader(g CacheGenClient, registerer prometheus.Registerer) *GenNumberLoader { |
||||
if g == nil { |
||||
g = &noopNumberGetter{} |
||||
} |
||||
|
||||
l := &GenNumberLoader{ |
||||
numberGetter: g, |
||||
numbers: make(map[string]string), |
||||
metrics: newGenLoaderMetrics(registerer), |
||||
} |
||||
go l.loop() |
||||
|
||||
return l |
||||
} |
||||
|
||||
func (l *GenNumberLoader) loop() { |
||||
timer := time.NewTicker(reloadDuration) |
||||
for { |
||||
select { |
||||
case <-timer.C: |
||||
err := l.reload() |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error reloading generation numbers", "err", err) |
||||
} |
||||
case <-l.quit: |
||||
return |
||||
} |
||||
} |
||||
} |
||||
|
||||
func (l *GenNumberLoader) reload() error { |
||||
updatedGenNumbers, err := l.getUpdatedGenNumbers() |
||||
if err != nil { |
||||
return err |
||||
} |
||||
|
||||
l.lock.Lock() |
||||
defer l.lock.Unlock() |
||||
for userID, genNumber := range updatedGenNumbers { |
||||
l.numbers[userID] = genNumber |
||||
} |
||||
|
||||
return nil |
||||
} |
||||
|
||||
func (l *GenNumberLoader) getUpdatedGenNumbers() (map[string]string, error) { |
||||
l.lock.RLock() |
||||
defer l.lock.RUnlock() |
||||
|
||||
updatedGenNumbers := make(map[string]string) |
||||
for userID, oldGenNumber := range l.numbers { |
||||
genNumber, err := l.numberGetter.GetCacheGenerationNumber(context.Background(), userID) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
|
||||
if oldGenNumber != genNumber { |
||||
updatedGenNumbers[userID] = genNumber |
||||
} |
||||
} |
||||
|
||||
return updatedGenNumbers, nil |
||||
} |
||||
|
||||
func (l *GenNumberLoader) GetResultsCacheGenNumber(tenantIDs []string) string { |
||||
return l.getCacheGenNumbersPerTenants(tenantIDs) |
||||
} |
||||
|
||||
func (l *GenNumberLoader) getCacheGenNumbersPerTenants(tenantIDs []string) string { |
||||
var max int |
||||
for _, tenantID := range tenantIDs { |
||||
genNumber := l.getCacheGenNumber(tenantID) |
||||
if genNumber == "" { |
||||
continue |
||||
} |
||||
|
||||
number, err := strconv.Atoi(genNumber) |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error parsing resultsCacheGenNumber", "user", tenantID, "err", err) |
||||
} |
||||
|
||||
if number > max { |
||||
max = number |
||||
} |
||||
} |
||||
|
||||
if max == 0 { |
||||
return "" |
||||
} |
||||
return fmt.Sprint(max) |
||||
} |
||||
|
||||
func (l *GenNumberLoader) getCacheGenNumber(userID string) string { |
||||
l.lock.RLock() |
||||
if genNumber, ok := l.numbers[userID]; ok { |
||||
l.lock.RUnlock() |
||||
return genNumber |
||||
} |
||||
l.lock.RUnlock() |
||||
|
||||
genNumber, err := l.numberGetter.GetCacheGenerationNumber(context.Background(), userID) |
||||
if err != nil { |
||||
level.Error(log.Logger).Log("msg", "error loading cache generation numbers", "err", err) |
||||
l.metrics.cacheGenLoadFailures.WithLabelValues(l.numberGetter.Name()).Inc() |
||||
return "" |
||||
} |
||||
|
||||
l.lock.Lock() |
||||
defer l.lock.Unlock() |
||||
|
||||
l.numbers[userID] = genNumber |
||||
return genNumber |
||||
} |
||||
|
||||
func (l *GenNumberLoader) Stop() { |
||||
close(l.quit) |
||||
} |
||||
|
||||
type noopNumberGetter struct{} |
||||
|
||||
func (g *noopNumberGetter) GetCacheGenerationNumber(_ context.Context, _ string) (string, error) { |
||||
return "", nil |
||||
} |
||||
|
||||
func (g *noopNumberGetter) Name() string { |
||||
return "" |
||||
} |
||||
@ -0,0 +1,54 @@ |
||||
package generationnumber |
||||
|
||||
import ( |
||||
"context" |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestGetCacheGenNumber(t *testing.T) { |
||||
s := &mockGenNumberClient{ |
||||
genNumbers: map[string]string{ |
||||
"tenant-a": "1000", |
||||
"tenant-b": "1050", |
||||
}, |
||||
} |
||||
loader := NewGenNumberLoader(s, nil) |
||||
|
||||
for _, tc := range []struct { |
||||
name string |
||||
expectedResultsCacheGenNumber string |
||||
tenantIDs []string |
||||
}{ |
||||
{ |
||||
name: "single tenant with numeric values", |
||||
tenantIDs: []string{"tenant-a"}, |
||||
expectedResultsCacheGenNumber: "1000", |
||||
}, |
||||
{ |
||||
name: "multiple tenants with numeric values", |
||||
tenantIDs: []string{"tenant-a", "tenant-b"}, |
||||
expectedResultsCacheGenNumber: "1050", |
||||
}, |
||||
{ |
||||
name: "no tenants", // not really an expected call, edge case check to avoid any panics
|
||||
}, |
||||
} { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
assert.Equal(t, tc.expectedResultsCacheGenNumber, loader.GetResultsCacheGenNumber(tc.tenantIDs)) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
type mockGenNumberClient struct { |
||||
genNumbers map[string]string |
||||
} |
||||
|
||||
func (g *mockGenNumberClient) GetCacheGenerationNumber(ctx context.Context, userID string) (string, error) { |
||||
return g.genNumbers[userID], nil |
||||
} |
||||
|
||||
func (g *mockGenNumberClient) Name() string { |
||||
return "" |
||||
} |
||||
@ -0,0 +1,36 @@ |
||||
package generationnumber |
||||
|
||||
import ( |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
) |
||||
|
||||
// Make this package level because we want several instances of a loader to be able to report metrics
|
||||
var metrics *genLoaderMetrics |
||||
|
||||
type genLoaderMetrics struct { |
||||
cacheGenLoadFailures *prometheus.CounterVec |
||||
} |
||||
|
||||
func newGenLoaderMetrics(r prometheus.Registerer) *genLoaderMetrics { |
||||
if metrics != nil { |
||||
return metrics |
||||
} |
||||
|
||||
if r == nil { |
||||
return nil |
||||
} |
||||
|
||||
cacheGenLoadFailures := prometheus.NewCounterVec(prometheus.CounterOpts{ |
||||
Namespace: "loki", |
||||
Name: "delete_cache_gen_load_failures_total", |
||||
Help: "Total number of failures while loading cache generation number using gen number loader", |
||||
}, []string{"source"}) |
||||
|
||||
r.MustRegister(cacheGenLoadFailures) |
||||
|
||||
metrics = &genLoaderMetrics{ |
||||
cacheGenLoadFailures: cacheGenLoadFailures, |
||||
} |
||||
|
||||
return metrics |
||||
} |
||||
Loading…
Reference in new issue