Move `cortex/pkg/tenant` dependency to Loki. (#4944)

pull/4961/head
Kaviraj 4 years ago committed by GitHub
parent 6b277cd9de
commit 38c42a26c6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
  1. 1
      CHANGELOG.md
  2. 3
      clients/pkg/promtail/targets/lokipush/pushtarget.go
  3. 3
      cmd/migrate/main.go
  4. 3
      pkg/distributor/distributor.go
  5. 3
      pkg/distributor/http.go
  6. 2
      pkg/ingester/flush.go
  7. 3
      pkg/ingester/flush_test.go
  8. 3
      pkg/ingester/ingester.go
  9. 3
      pkg/ingester/ingester_test.go
  10. 3
      pkg/logql/engine.go
  11. 3
      pkg/lokifrontend/frontend/transport/handler.go
  12. 2
      pkg/lokifrontend/frontend/v1/frontend.go
  13. 2
      pkg/lokifrontend/frontend/v2/frontend.go
  14. 3
      pkg/querier/http.go
  15. 3
      pkg/querier/querier.go
  16. 3
      pkg/querier/queryrange/limits.go
  17. 3
      pkg/querier/queryrange/querysharding.go
  18. 3
      pkg/querier/queryrange/roundtrip.go
  19. 3
      pkg/querier/queryrange/split_by_interval.go
  20. 3
      pkg/scheduler/scheduler.go
  21. 3
      pkg/storage/chunk/storage/caching_index_client.go
  22. 3
      pkg/storage/store.go
  23. 3
      pkg/storage/stores/shipper/compactor/deletion/request_handler.go
  24. 158
      pkg/tenant/resolver.go
  25. 105
      pkg/tenant/tenant.go

@ -1,5 +1,6 @@
## Main
* [4944](https://github.com/grafana/loki/pull/4944) **kavirajk**: Move cortex/pkg/tenant dependency to Loki
* [4926](https://github.com/grafana/loki/pull/4926) **thejosephstevens**: Fix comment in Loki module loading for accuracy
* [4920](https://github.com/grafana/loki/pull/4920) **chaudum**: Add `-list-targets` command line flag to list all available run targets
* [4860](https://github.com/grafana/loki/pull/4860) **cyriltovena**: Add rate limiting and metrics to hedging

@ -9,7 +9,6 @@ import (
"strings"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
@ -20,6 +19,8 @@ import (
promql_parser "github.com/prometheus/prometheus/promql/parser"
"github.com/weaveworks/common/server"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/clients/pkg/promtail/api"
"github.com/grafana/loki/clients/pkg/promtail/scrapeconfig"
"github.com/grafana/loki/clients/pkg/promtail/targets/target"

@ -12,11 +12,12 @@ import (
"sync"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/common/model"
"github.com/grafana/loki/pkg/tenant"
"github.com/prometheus/prometheus/model/labels"
"github.com/weaveworks/common/user"

@ -7,7 +7,6 @@ import (
"time"
cortex_distributor "github.com/cortexproject/cortex/pkg/distributor"
"github.com/cortexproject/cortex/pkg/tenant"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/grafana/dskit/limiter"
"github.com/grafana/dskit/ring"
@ -23,6 +22,8 @@ import (
"go.uber.org/atomic"
"google.golang.org/grpc/health/grpc_health_v1"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/ingester/client"
"github.com/grafana/loki/pkg/logproto"
"github.com/grafana/loki/pkg/logql"

@ -4,11 +4,12 @@ import (
"net/http"
"strings"
"github.com/cortexproject/cortex/pkg/tenant"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log/level"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/loghttp/push"
serverutil "github.com/grafana/loki/pkg/util/server"
)

@ -19,7 +19,7 @@ import (
"github.com/cortexproject/cortex/pkg/util"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/chunkenc"
"github.com/grafana/loki/pkg/storage/chunk"

@ -10,7 +10,6 @@ import (
"testing"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
gokitlog "github.com/go-kit/log"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/kv"
@ -22,6 +21,8 @@ import (
"github.com/weaveworks/common/user"
"golang.org/x/net/context"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/chunkenc"
"github.com/grafana/loki/pkg/ingester/client"
"github.com/grafana/loki/pkg/iter"

@ -10,7 +10,6 @@ import (
"sync"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log/level"
@ -23,6 +22,8 @@ import (
"github.com/prometheus/prometheus/model/labels"
"google.golang.org/grpc/health/grpc_health_v1"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/chunkenc"
"github.com/grafana/loki/pkg/ingester/client"
"github.com/grafana/loki/pkg/ingester/index"

@ -7,7 +7,6 @@ import (
"testing"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/grafana/dskit/flagext"
"github.com/grafana/dskit/services"
"github.com/prometheus/common/model"
@ -19,6 +18,8 @@ import (
"google.golang.org/grpc"
"google.golang.org/grpc/metadata"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/chunkenc"
"github.com/grafana/loki/pkg/ingester/client"
"github.com/grafana/loki/pkg/ingester/index"

@ -8,7 +8,6 @@ import (
"sort"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util/spanlogger"
"github.com/go-kit/log/level"
"github.com/prometheus/client_golang/prometheus"
@ -17,6 +16,8 @@ import (
"github.com/prometheus/prometheus/promql"
promql_parser "github.com/prometheus/prometheus/promql/parser"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/iter"
"github.com/grafana/loki/pkg/logproto"
"github.com/grafana/loki/pkg/logqlmodel"

@ -14,7 +14,6 @@ import (
"time"
querier_stats "github.com/cortexproject/cortex/pkg/querier/stats"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log"
@ -22,6 +21,8 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/grafana/loki/pkg/tenant"
serverutil "github.com/grafana/loki/pkg/util/server"
)

@ -10,7 +10,6 @@ import (
"github.com/cortexproject/cortex/pkg/frontend/v1/frontendv1pb"
"github.com/cortexproject/cortex/pkg/querier/stats"
"github.com/cortexproject/cortex/pkg/scheduler/queue"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/validation"
"github.com/go-kit/log"
@ -22,6 +21,7 @@ import (
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
lokigrpc "github.com/grafana/loki/pkg/util/httpgrpc"
)

@ -11,7 +11,6 @@ import (
"github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb"
"github.com/cortexproject/cortex/pkg/querier/stats"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/dskit/flagext"
@ -25,6 +24,7 @@ import (
"github.com/weaveworks/common/httpgrpc"
"go.uber.org/atomic"
"github.com/grafana/loki/pkg/tenant"
lokigrpc "github.com/grafana/loki/pkg/util/httpgrpc"
)

@ -5,7 +5,6 @@ import (
"net/http"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log/level"
"github.com/gorilla/websocket"
@ -13,6 +12,8 @@ import (
"github.com/prometheus/prometheus/promql/parser"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/loghttp"
loghttp_legacy "github.com/grafana/loki/pkg/loghttp/legacy"
"github.com/grafana/loki/pkg/logql"

@ -10,11 +10,12 @@ import (
"github.com/weaveworks/common/httpgrpc"
"google.golang.org/grpc/health/grpc_health_v1"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util/spanlogger"
cortex_validation "github.com/cortexproject/cortex/pkg/util/validation"
"github.com/go-kit/log/level"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/iter"
"github.com/grafana/loki/pkg/loghttp"
"github.com/grafana/loki/pkg/logproto"

@ -9,7 +9,6 @@ import (
"github.com/cortexproject/cortex/pkg/cortexpb"
"github.com/cortexproject/cortex/pkg/querier/queryrange"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/spanlogger"
"github.com/cortexproject/cortex/pkg/util/validation"
@ -19,6 +18,8 @@ import (
"github.com/weaveworks/common/httpgrpc"
"github.com/weaveworks/common/user"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/logql"
)

@ -8,7 +8,6 @@ import (
"github.com/cortexproject/cortex/pkg/querier/astmapper"
"github.com/cortexproject/cortex/pkg/querier/queryrange"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log"
@ -17,6 +16,8 @@ import (
"github.com/prometheus/prometheus/promql/parser"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/loghttp"
"github.com/grafana/loki/pkg/logql"
"github.com/grafana/loki/pkg/logqlmodel"

@ -8,12 +8,13 @@ import (
"time"
"github.com/cortexproject/cortex/pkg/querier/queryrange"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/go-kit/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/prometheus/model/labels"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/loghttp"
"github.com/grafana/loki/pkg/logql"
"github.com/grafana/loki/pkg/storage/chunk"

@ -6,13 +6,14 @@ import (
"time"
"github.com/cortexproject/cortex/pkg/querier/queryrange"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/opentracing/opentracing-go"
otlog "github.com/opentracing/opentracing-go/log"
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/weaveworks/common/httpgrpc"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/logproto"
)

@ -13,7 +13,6 @@ import (
"github.com/cortexproject/cortex/pkg/frontend/v2/frontendv2pb"
"github.com/cortexproject/cortex/pkg/scheduler/queue"
"github.com/cortexproject/cortex/pkg/scheduler/schedulerpb"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
"github.com/cortexproject/cortex/pkg/util/validation"
"github.com/go-kit/log"
@ -33,6 +32,8 @@ import (
"go.uber.org/atomic"
"google.golang.org/grpc"
"github.com/grafana/loki/pkg/tenant"
lokiutil "github.com/grafana/loki/pkg/util"
lokigrpc "github.com/grafana/loki/pkg/util/httpgrpc"
)

@ -11,9 +11,10 @@ import (
"github.com/prometheus/client_golang/prometheus"
"github.com/prometheus/client_golang/prometheus/promauto"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util/spanlogger"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/storage/chunk"
"github.com/grafana/loki/pkg/storage/chunk/cache"
chunk_util "github.com/grafana/loki/pkg/storage/chunk/util"

@ -8,7 +8,6 @@ import (
"time"
"github.com/cortexproject/cortex/pkg/querier/astmapper"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/go-kit/log"
"github.com/go-kit/log/level"
"github.com/grafana/dskit/flagext"
@ -16,6 +15,8 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/model/labels"
"github.com/grafana/loki/pkg/tenant"
"github.com/grafana/loki/pkg/iter"
"github.com/grafana/loki/pkg/logproto"
"github.com/grafana/loki/pkg/logql"

@ -5,7 +5,6 @@ import (
"net/http"
"time"
"github.com/cortexproject/cortex/pkg/tenant"
"github.com/cortexproject/cortex/pkg/util"
util_log "github.com/cortexproject/cortex/pkg/util/log"
"github.com/go-kit/log/level"
@ -13,6 +12,8 @@ import (
"github.com/prometheus/common/model"
"github.com/prometheus/prometheus/promql/parser"
"github.com/grafana/loki/pkg/tenant"
serverutil "github.com/grafana/loki/pkg/util/server"
)

@ -0,0 +1,158 @@
package tenant
import (
"context"
"errors"
"net/http"
"strings"
"github.com/weaveworks/common/user"
)
var defaultResolver Resolver = NewSingleResolver()
// WithDefaultResolver updates the resolver used for the package methods.
func WithDefaultResolver(r Resolver) {
defaultResolver = r
}
// TenantID returns exactly a single tenant ID from the context. It should be
// used when a certain endpoint should only support exactly a single
// tenant ID. It returns an error user.ErrNoOrgID if there is no tenant ID
// supplied or user.ErrTooManyOrgIDs if there are multiple tenant IDs present.
//
// ignore stutter warning
//nolint:golint,revive
func TenantID(ctx context.Context) (string, error) {
return defaultResolver.TenantID(ctx)
}
// TenantIDs returns all tenant IDs from the context. It should return
// normalized list of ordered and distinct tenant IDs (as produced by
// NormalizeTenantIDs).
//
// ignore stutter warning
//nolint:golint,revive
func TenantIDs(ctx context.Context) ([]string, error) {
return defaultResolver.TenantIDs(ctx)
}
type Resolver interface {
// TenantID returns exactly a single tenant ID from the context. It should be
// used when a certain endpoint should only support exactly a single
// tenant ID. It returns an error user.ErrNoOrgID if there is no tenant ID
// supplied or user.ErrTooManyOrgIDs if there are multiple tenant IDs present.
TenantID(context.Context) (string, error)
// TenantIDs returns all tenant IDs from the context. It should return
// normalized list of ordered and distinct tenant IDs (as produced by
// NormalizeTenantIDs).
TenantIDs(context.Context) ([]string, error)
}
// NewSingleResolver creates a tenant resolver, which restricts all requests to
// be using a single tenant only. This allows a wider set of characters to be
// used within the tenant ID and should not impose a breaking change.
func NewSingleResolver() *SingleResolver {
return &SingleResolver{}
}
type SingleResolver struct {
}
// containsUnsafePathSegments will return true if the string is a directory
// reference like `.` and `..` or if any path separator character like `/` and
// `\` can be found.
func containsUnsafePathSegments(id string) bool {
// handle the relative reference to current and parent path.
if id == "." || id == ".." {
return true
}
return strings.ContainsAny(id, "\\/")
}
var errInvalidTenantID = errors.New("invalid tenant ID")
func (t *SingleResolver) TenantID(ctx context.Context) (string, error) {
//lint:ignore faillint wrapper around upstream method
id, err := user.ExtractOrgID(ctx)
if err != nil {
return "", err
}
if containsUnsafePathSegments(id) {
return "", errInvalidTenantID
}
return id, nil
}
func (t *SingleResolver) TenantIDs(ctx context.Context) ([]string, error) {
orgID, err := t.TenantID(ctx)
if err != nil {
return nil, err
}
return []string{orgID}, err
}
type MultiResolver struct {
}
// NewMultiResolver creates a tenant resolver, which allows request to have
// multiple tenant ids submitted separated by a '|' character. This enforces
// further limits on the character set allowed within tenants as detailed here:
// https://cortexmetrics.io/docs/guides/limitations/#tenant-id-naming)
func NewMultiResolver() *MultiResolver {
return &MultiResolver{}
}
func (t *MultiResolver) TenantID(ctx context.Context) (string, error) {
orgIDs, err := t.TenantIDs(ctx)
if err != nil {
return "", err
}
if len(orgIDs) > 1 {
return "", user.ErrTooManyOrgIDs
}
return orgIDs[0], nil
}
func (t *MultiResolver) TenantIDs(ctx context.Context) ([]string, error) {
//lint:ignore faillint wrapper around upstream method
orgID, err := user.ExtractOrgID(ctx)
if err != nil {
return nil, err
}
orgIDs := strings.Split(orgID, tenantIDsLabelSeparator)
for _, orgID := range orgIDs {
if err := ValidTenantID(orgID); err != nil {
return nil, err
}
if containsUnsafePathSegments(orgID) {
return nil, errInvalidTenantID
}
}
return NormalizeTenantIDs(orgIDs), nil
}
// ExtractTenantIDFromHTTPRequest extracts a single TenantID through a given
// resolver directly from a HTTP request.
func ExtractTenantIDFromHTTPRequest(req *http.Request) (string, context.Context, error) {
//lint:ignore faillint wrapper around upstream method
_, ctx, err := user.ExtractOrgIDFromHTTPRequest(req)
if err != nil {
return "", nil, err
}
tenantID, err := defaultResolver.TenantID(ctx)
if err != nil {
return "", nil, err
}
return tenantID, ctx, nil
}

@ -0,0 +1,105 @@
package tenant
import (
"context"
"errors"
"fmt"
"sort"
"strings"
"github.com/weaveworks/common/user"
)
var (
errTenantIDTooLong = errors.New("tenant ID is too long: max 150 characters")
)
type errTenantIDUnsupportedCharacter struct {
pos int
tenantID string
}
func (e *errTenantIDUnsupportedCharacter) Error() string {
return fmt.Sprintf(
"tenant ID '%s' contains unsupported character '%c'",
e.tenantID,
e.tenantID[e.pos],
)
}
const tenantIDsLabelSeparator = "|"
// NormalizeTenantIDs is creating a normalized form by sortiing and de-duplicating the list of tenantIDs
func NormalizeTenantIDs(tenantIDs []string) []string {
sort.Strings(tenantIDs)
count := len(tenantIDs)
if count <= 1 {
return tenantIDs
}
posOut := 1
for posIn := 1; posIn < count; posIn++ {
if tenantIDs[posIn] != tenantIDs[posIn-1] {
tenantIDs[posOut] = tenantIDs[posIn]
posOut++
}
}
return tenantIDs[0:posOut]
}
// ValidTenantID
func ValidTenantID(s string) error {
// check if it contains invalid runes
for pos, r := range s {
if !isSupported(r) {
return &errTenantIDUnsupportedCharacter{
tenantID: s,
pos: pos,
}
}
}
if len(s) > 150 {
return errTenantIDTooLong
}
return nil
}
func JoinTenantIDs(tenantIDs []string) string {
return strings.Join(tenantIDs, tenantIDsLabelSeparator)
}
// this checks if a rune is supported in tenant IDs (according to
// https://cortexmetrics.io/docs/guides/limitations/#tenant-id-naming)
func isSupported(c rune) bool {
// characters
if ('a' <= c && c <= 'z') || ('A' <= c && c <= 'Z') {
return true
}
// digits
if '0' <= c && c <= '9' {
return true
}
// special
return c == '!' ||
c == '-' ||
c == '_' ||
c == '.' ||
c == '*' ||
c == '\'' ||
c == '(' ||
c == ')'
}
// TenantIDsFromOrgID extracts different tenants from an orgID string value
//
// ignore stutter warning
//nolint:golint,revive
func TenantIDsFromOrgID(orgID string) ([]string, error) {
return TenantIDs(user.InjectOrgID(context.TODO(), orgID))
}
Loading…
Cancel
Save