mirror of https://github.com/grafana/loki
Add a Ring to IndexGateway (#5358)
* Begin to add a Ring to IndexGateway Signed-off-by: Jordan Rushing <jordan.rushing@grafana.com> * Implement missing methods for IndexGateway. - Implement missing methods for the IndexGateway to be used as a BasicLifecyclerDelegate. The methods are stored in a lifecycle file and follow the same approach of the Ruler ring - Make Gateway public and add missing parameters to the IndexGateway's initialization method * Fix failing linter Signed-off-by: Jordan Rushing <jordan.rushing@grafana.com> * Implement IndexGateway support for dynamic configs. - Add memberlist as a dependency of the IndexGateway - Add IndexGateway support for the common configuration section - Modify Loki to register IndexGateway flags. This fixes the default flag values for the IndexGateway - Make IndexGateway configurations public. Without this, we can't use them. * Implement NewBasicService for the IndexGateway. - If IndexGateway is running with a ring, it uses the IdleService. Otherwise, it uses the BasicService - Implement IndexGateway support to handle ring HTTP pages * Test IndexGateway dynamic configuration. * Implement new IndexGatewayGRPCPool entity. * Make IndexGateway attributes public. * Implement IndexGatewayRing reader. - Implement new IndexGatewayRing service - Add IndexGatewayRing as a Store dependency - Modify store to pass IndexGatewayRing as a parameter - Implement IndexGatewayClient ring mode * Implement Ring mode in the IndexGatewayClient. - Moves IndexGateway configuration to the IndexGatewayClient struct within the boltdb-shipper - Reuse the IndexGatewayClient everywhere - Implement IndexGateway gRPC pool * Add new ring index gateway parameter to new Store calls. * Use errors.Wrap instead of fmt.Errorf. * Extract tenantID from context instead of iterating on queries. * Remove indexGateway ring param. - Instead, add it to the store configuration and bubble it down to deeper modules. * Split IndexGateway server from client implementation. * Fix imports order. * Remove ring as parameter from IndexGateway-related funcs. * Fix default flag value and IndexQuerier type. * Remove additional mode field and reuse it from cfg. * Remove redundant service init. * Add sanity check for IndexGateway client constructor. * Move mode assigning to initStore method. * Reorder IndexGateway constructor. * Rewrite indexClient chunk.IndexClient as querier Index.Querier. * Fix flag registration for IndexGateway server. - In a previous PR I changed how it works and forgot to update how the flag registration occurs. * Fix flag registration for test. * Keep only one reference to indexQuerier. * Add guard-clause on IndexGatewayRing service. - Without this, we would be initializing ring-related things even if the index gateway is in simple mode. * Move IndexGatewayClientCfg to gateway_client file. * Update CHANGELOG.md for `IndexGateway` support for `RingMode` Signed-off-by: JordanRushing <rushing.jordan@gmail.com> * Update GatewayClient to use dskit tenant package Signed-off-by: JordanRushing <rushing.jordan@gmail.com> * Add listenport configuration for IndexGateway and Ring Signed-off-by: JordanRushing <rushing.jordan@gmail.com> * Make IndexGateway replication factor configurable. - Add `replication_factor` flag to the IndexGateway config struct - Modify Index Gateway ring config struct to be inline with a new struct, since it doesn't expose a replication factor config - Modify dynamic config wrapper to reuse a common replication factor on the Index Gateway ring * Randomize replication set access. - If we don't randomize access, we'll always access same Index Gateway instances in same order for the same tenant * Remove unwanted merge HEAD tags. * Move away from stores/chunk package. * Pass util_log in factory. * Change index gateway client ring to ignore replicas. - This is accomplished by using `NewIgnoreUnhealthyInstancesReplicationStrategy` - It is already done by the server * Refactor where the common replication factor is applied. * Housekeeping config_wrapper IndexGateway configs. - Remove unnecessary/wrong comments - Only set replication factor at a single place Co-authored-by: Dylan Guedes <djmgguedes@gmail.com>pull/5900/head
JordanRushing
3 years ago
committed by
GitHub
parent
e5ed1578c8
commit
1dc0d978e4
@ -0,0 +1,36 @@ |
||||
package shipper |
||||
|
||||
import ( |
||||
"io" |
||||
|
||||
"github.com/pkg/errors" |
||||
"google.golang.org/grpc" |
||||
"google.golang.org/grpc/health/grpc_health_v1" |
||||
|
||||
"github.com/grafana/loki/pkg/storage/stores/shipper/indexgateway/indexgatewaypb" |
||||
) |
||||
|
||||
// IndexGatewayGRPCPool represents a pool of gRPC connections to different index gateway instances.
|
||||
//
|
||||
// Only used when Index Gateway is configured to run in ring mode.
|
||||
type IndexGatewayGRPCPool struct { |
||||
grpc_health_v1.HealthClient |
||||
indexgatewaypb.IndexGatewayClient |
||||
io.Closer |
||||
} |
||||
|
||||
// NewIndexGatewayGRPCPool instantiates a new pool of IndexGateway GRPC connections.
|
||||
//
|
||||
// Internally, it also instantiates a protobuf index gateway client and a health client.
|
||||
func NewIndexGatewayGRPCPool(address string, opts []grpc.DialOption) (*IndexGatewayGRPCPool, error) { |
||||
conn, err := grpc.Dial(address, opts...) |
||||
if err != nil { |
||||
return nil, errors.Wrap(err, "shipper new grpc pool dial") |
||||
} |
||||
|
||||
return &IndexGatewayGRPCPool{ |
||||
Closer: conn, |
||||
HealthClient: grpc_health_v1.NewHealthClient(conn), |
||||
IndexGatewayClient: indexgatewaypb.NewIndexGatewayClient(conn), |
||||
}, nil |
||||
} |
||||
@ -0,0 +1,85 @@ |
||||
package indexgateway |
||||
|
||||
import ( |
||||
"flag" |
||||
"fmt" |
||||
|
||||
loki_util "github.com/grafana/loki/pkg/util" |
||||
) |
||||
|
||||
// Mode represents in which mode an Index Gateway instance is running.
|
||||
//
|
||||
// Right now, two modes are supported: simple mode (default) and ring mode.
|
||||
type Mode string |
||||
|
||||
// Set implements a flag interface, and is necessary to use the IndexGatewayClientMode as a flag.
|
||||
func (i Mode) Set(v string) error { |
||||
switch v { |
||||
case string(SimpleMode): |
||||
// nolint:ineffassign
|
||||
i = SimpleMode |
||||
case string(RingMode): |
||||
// nolint:ineffassign
|
||||
i = RingMode |
||||
default: |
||||
return fmt.Errorf("mode %s not supported. list of supported modes: simple (default), ring", v) |
||||
} |
||||
return nil |
||||
} |
||||
|
||||
// String implements a flag interface, and is necessary to use the IndexGatewayClientMode as a flag.
|
||||
func (i Mode) String() string { |
||||
switch i { |
||||
case RingMode: |
||||
return string(RingMode) |
||||
default: |
||||
return string(SimpleMode) |
||||
} |
||||
} |
||||
|
||||
const ( |
||||
// SimpleMode is a mode where an Index Gateway instance solely handle all the work.
|
||||
SimpleMode Mode = "simple" |
||||
|
||||
// RingMode is a mode where different Index Gateway instances are assigned to handle different tenants.
|
||||
//
|
||||
// It is more horizontally scalable than the simple mode, but requires running a key-value store ring.
|
||||
RingMode Mode = "ring" |
||||
) |
||||
|
||||
// RingCfg is a wrapper for our Index Gateway ring configuration plus the replication factor.
|
||||
type RingCfg struct { |
||||
// InternalRingCfg configures the Index Gateway ring.
|
||||
loki_util.RingConfig `yaml:",inline"` |
||||
|
||||
// ReplicationFactor defines how many Index Gateway instances are assigned to each tenant.
|
||||
//
|
||||
// Whenever the store queries the ring key-value store for the Index Gateway instance responsible for tenant X,
|
||||
// multiple Index Gateway instances are expected to be returned as Index Gateway might be busy/locked for specific
|
||||
// reasons (this is assured by the spikey behavior of Index Gateway latencies).
|
||||
ReplicationFactor int `yaml:"replication_factor"` |
||||
} |
||||
|
||||
// RegisterFlagsWithPrefix register all Index Gateway flags related to its ring but with a proper store prefix to avoid conflicts.
|
||||
func (cfg *RingCfg) RegisterFlags(prefix, storePrefix string, f *flag.FlagSet) { |
||||
cfg.RegisterFlagsWithPrefix(prefix, storePrefix, f) |
||||
f.IntVar(&cfg.ReplicationFactor, "replication-factor", 3, "how many index gateway instances are assigned to each tenant") |
||||
} |
||||
|
||||
// Config configures an Index Gateway server.
|
||||
type Config struct { |
||||
// Mode configures in which mode the client will be running when querying and communicating with an Index Gateway instance.
|
||||
Mode Mode `yaml:"mode"` |
||||
|
||||
// Ring configures the ring key-value store used to save and retrieve the different Index Gateway instances.
|
||||
//
|
||||
// In case it isn't explicitly set, it follows the same behavior of the other rings (ex: using the common configuration
|
||||
// section and the ingester configuration by default).
|
||||
Ring RingCfg `yaml:"ring,omitempty"` |
||||
} |
||||
|
||||
// RegisterFlags register all IndexGatewayClientConfig flags and all the flags of its subconfigs but with a prefix (ex: shipper).
|
||||
func (cfg *Config) RegisterFlags(f *flag.FlagSet) { |
||||
cfg.Ring.RegisterFlags("index-gateway.", "collectors/", f) |
||||
f.StringVar((*string)(&cfg.Mode), "index-gateway.mode", SimpleMode.String(), "mode in which the index gateway client will be running") |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
package indexgateway |
||||
|
||||
import ( |
||||
"github.com/grafana/dskit/ring" |
||||
) |
||||
|
||||
func (g *Gateway) OnRingInstanceRegister(_ *ring.BasicLifecycler, ringDesc ring.Desc, instanceExists bool, instanceID string, instanceDesc ring.InstanceDesc) (ring.InstanceState, ring.Tokens) { |
||||
// When we initialize the index gateway instance in the ring we want to start from
|
||||
// a clean situation, so whatever is the state we set it JOINING, while we keep existing
|
||||
// tokens (if any) or the ones loaded from file.
|
||||
var tokens []uint32 |
||||
if instanceExists { |
||||
tokens = instanceDesc.GetTokens() |
||||
} |
||||
|
||||
takenTokens := ringDesc.GetTokens() |
||||
newTokens := ring.GenerateTokens(ringNumTokens-len(tokens), takenTokens) |
||||
|
||||
// Tokens sorting will be enforced by the parent caller.
|
||||
tokens = append(tokens, newTokens...) |
||||
|
||||
return ring.JOINING, tokens |
||||
} |
||||
|
||||
func (g *Gateway) OnRingInstanceTokens(_ *ring.BasicLifecycler, _ ring.Tokens) {} |
||||
func (g *Gateway) OnRingInstanceStopping(_ *ring.BasicLifecycler) {} |
||||
func (g *Gateway) OnRingInstanceHeartbeat(_ *ring.BasicLifecycler, _ *ring.Desc, _ *ring.InstanceDesc) { |
||||
} |
||||
Loading…
Reference in new issue