apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Move promtail StreamLagLabels config to upper level config.Config (#5686)

Browse Source 
* Move promtail StreamLagLabels config to upper level config.Config

* Fix promtail config load test

* Fix promtail configuration reference

* Add options promtail config section

* Kick GitHub CI

Signed-off-by: Danny Kopping <danny.kopping@grafana.com>

Co-authored-by: Danny Kopping <danny.kopping@grafana.com>
pull/5699/head
Susana Ferreira 3 years ago committed by GitHub
parent dec695f112
commit 1d1461824d
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
6 changed files with 80 additions and 78 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 2
      clients/cmd/promtail/main.go
  2. 11
      clients/pkg/promtail/client/config.go
  3. 16
      clients/pkg/promtail/config/config.go
  4. 95
      clients/pkg/promtail/config/config_test.go
  5. 4
      clients/pkg/promtail/promtail.go
  6. 30
      docs/sources/clients/promtail/configuration.md

2
clients/cmd/promtail/main.go
Unescape View File

@ -111,7 +111,7 @@ func main() {
}
}
clientMetrics := client.NewMetrics(prometheus.DefaultRegisterer, config.Config.ClientConfigs.StreamLagLabels)
clientMetrics := client.NewMetrics(prometheus.DefaultRegisterer, config.Config.Options.StreamLagLabels)
p, err := promtail.New(config.Config, clientMetrics, config.dryRun)
if err != nil {
level.Error(util_log.Logger).Log("msg", "error creating promtail", "error", err)

11
clients/pkg/promtail/client/config.go
Unescape View File

@ -6,7 +6,6 @@ import (
"github.com/grafana/dskit/backoff"
"github.com/grafana/dskit/flagext"
dskit_flagext "github.com/grafana/dskit/flagext"
"github.com/prometheus/common/config"
lokiflag "github.com/grafana/loki/pkg/util/flagext"
@ -22,12 +21,7 @@ const (
Timeout = 10 * time.Second
)
type Configs struct {
StreamLagLabels dskit_flagext.StringSliceCSV `yaml:"stream_lag_labels,omitempty"`
Configs []Config `yaml:"configs"`
}
// Config describes configuration for a HTTP pusher client.
// Config describes configuration for an HTTP pusher client.
type Config struct {
Name string `yaml:"name,omitempty"`
URL flagext.URLValue
@ -44,6 +38,9 @@ type Config struct {
// The tenant ID to use when pushing logs to Loki (empty string means
// single tenant mode)
TenantID string `yaml:"tenant_id"`
// deprecated use StreamLagLabels from config.Config instead
StreamLagLabels flagext.StringSliceCSV `yaml:"stream_lag_labels"`
}
// RegisterFlags with prefix registers flags where every name is prefixed by

16
clients/pkg/promtail/config/config.go
Unescape View File

@ -4,6 +4,8 @@ import (
"flag"
"fmt"
dskit_flagext "github.com/grafana/dskit/flagext"
yaml "gopkg.in/yaml.v2"
"github.com/grafana/loki/clients/pkg/promtail/client"
@ -16,16 +18,22 @@ import (
"github.com/grafana/loki/pkg/util/flagext"
)
// Options contains cross-cutting promtail configurations
type Options struct {
StreamLagLabels dskit_flagext.StringSliceCSV `yaml:"stream_lag_labels,omitempty"`
}
// Config for promtail, describing what files to watch.
type Config struct {
ServerConfig server.Config `yaml:"server,omitempty"`
// deprecated use ClientConfigs instead
ClientConfig client.Config `yaml:"client,omitempty"`
ClientConfigs client.Configs `yaml:"clients,omitempty"`
ClientConfigs []client.Config `yaml:"clients,omitempty"`
PositionsConfig positions.Config `yaml:"positions,omitempty"`
ScrapeConfig []scrapeconfig.Config `yaml:"scrape_configs,omitempty"`
TargetConfig file.Config `yaml:"target_config,omitempty"`
LimitConfig limit.Config `yaml:"limit_config,omitempty"`
Options Options `yaml:"options,omitempty"`
}
// RegisterFlags with prefix registers flags where every name is prefixed by
@ -54,7 +62,7 @@ func (c Config) String() string {
func (c *Config) Setup() {
if c.ClientConfig.URL.URL != nil {
// if a single client config is used we add it to the multiple client config for backward compatibility
c.ClientConfigs.Configs = append(c.ClientConfigs.Configs, c.ClientConfig)
c.ClientConfigs = append(c.ClientConfigs, c.ClientConfig)
}
// This is a bit crude but if the Loki Push API target is specified,
@ -73,8 +81,8 @@ func (c *Config) Setup() {
// not typically the order of precedence, the assumption here is someone providing a specific config in
// yaml is doing so explicitly to make a key specific to a client.
if len(c.ClientConfig.ExternalLabels.LabelSet) > 0 {
for i := range c.ClientConfigs.Configs {
c.ClientConfigs.Configs[i].ExternalLabels = flagext.LabelSet{LabelSet: c.ClientConfig.ExternalLabels.LabelSet.Merge(c.ClientConfigs.Configs[i].ExternalLabels.LabelSet)}
for i := range c.ClientConfigs {
c.ClientConfigs[i].ExternalLabels = flagext.LabelSet{LabelSet: c.ClientConfig.ExternalLabels.LabelSet.Merge(c.ClientConfigs[i].ExternalLabels.LabelSet)}
}
}
}

95
clients/pkg/promtail/config/config_test.go
Unescape View File

@ -17,13 +17,12 @@ import (
const testFile = `
clients:
clients:
- external_labels:
cluster: dev1
url: https://1:shh@example.com/loki/api/v1/push
- external_labels:
cluster: prod1
url: https://1:shh@example.com/loki/api/v1/push
- external_labels:
cluster: dev1
url: https://1:shh@example.com/loki/api/v1/push
- external_labels:
cluster: prod1
url: https://1:shh@example.com/loki/api/v1/push
scrape_configs:
- job_name: kubernetes-pods-name
kubernetes_sd_configs:
@ -37,6 +36,8 @@ scrape_configs:
limit_config:
readline_rate: 100
readline_burst: 200
options:
stream_lag_labels: foo
`
func Test_Load(t *testing.T) {
@ -64,32 +65,32 @@ func TestConfig_Setup(t *testing.T) {
ClientConfig: client.Config{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
},
ClientConfigs: client.Configs{
StreamLagLabels: []string{},
Configs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2"}},
},
ClientConfigs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2"}},
},
},
Options: Options{
StreamLagLabels: []string{},
},
},
Config{
ClientConfig: client.Config{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
},
ClientConfigs: client.Configs{
StreamLagLabels: []string{},
Configs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2", "foo": "bar"}},
},
ClientConfigs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2", "foo": "bar"}},
},
},
Options: Options{
StreamLagLabels: []string{},
},
},
},
@ -99,16 +100,16 @@ func TestConfig_Setup(t *testing.T) {
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
URL: dskitflagext.URLValue{URL: mustURL("http://foo")},
},
ClientConfigs: client.Configs{
StreamLagLabels: []string{},
Configs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2"}},
},
ClientConfigs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2"}},
},
},
Options: Options{
StreamLagLabels: []string{},
},
},
Config{
@ -116,20 +117,20 @@ func TestConfig_Setup(t *testing.T) {
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
URL: dskitflagext.URLValue{URL: mustURL("http://foo")},
},
ClientConfigs: client.Configs{
StreamLagLabels: []string{},
Configs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
URL: dskitflagext.URLValue{URL: mustURL("http://foo")},
},
ClientConfigs: []client.Config{
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client1": "1", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"client2": "2", "foo": "bar"}},
},
{
ExternalLabels: flagext.LabelSet{LabelSet: model.LabelSet{"foo": "bar"}},
URL: dskitflagext.URLValue{URL: mustURL("http://foo")},
},
},
Options: Options{
StreamLagLabels: []string{},
},
},
},

4
clients/pkg/promtail/promtail.go
Unescape View File

@ -68,13 +68,13 @@ func New(cfg config.Config, metrics *client.Metrics, dryRun bool, opts ...Option
}
var err error
if dryRun {
promtail.client, err = client.NewLogger(metrics, cfg.ClientConfigs.StreamLagLabels, promtail.logger, cfg.ClientConfigs.Configs...)
promtail.client, err = client.NewLogger(metrics, cfg.Options.StreamLagLabels, promtail.logger, cfg.ClientConfigs...)
if err != nil {
return nil, err
}
cfg.PositionsConfig.ReadOnly = true
} else {
promtail.client, err = client.NewMulti(metrics, cfg.ClientConfigs.StreamLagLabels, promtail.logger, cfg.ClientConfigs.Configs...)
promtail.client, err = client.NewMulti(metrics, cfg.Options.StreamLagLabels, promtail.logger, cfg.ClientConfigs...)
if err != nil {
return nil, err
}

30
docs/sources/clients/promtail/configuration.md
Unescape View File

@ -97,6 +97,9 @@ scrape_configs:
# Configures how tailed targets will be watched.
[target_config: <target_config>]
# Configures additional promtail configurations.
[options: <options_config>]
```
## server
@ -159,23 +162,6 @@ The `server` block configures Promtail's behavior as an HTTP server:
The `clients` block configures how Promtail connects to instances of
Loki:
```yaml
# A comma-separated list of labels to include in the stream lag metric `promtail_stream_lag_seconds`.
# The default value is "filename". A "host" label is always included.
# The stream lag metric indicates which streams are falling behind on writes to Loki;
# be mindful about using too many labels, as it can increase cardinality.
[stream_lag_labels: <string> | default = "filename"]
configs:
- <client_config>
```
### client
The `client` block configures how an individual Promtail client connects
to instances of Loki:
```yaml
# The URL where Loki is listening, denoted in Loki as http_listen_address and
# http_listen_port. If Loki is running in microservices mode, this is the HTTP
@ -1782,6 +1768,16 @@ targets.
sync_period: "10s"
```
## options_config
```yaml
# A comma-separated list of labels to include in the stream lag metric `promtail_stream_lag_seconds`.
# The default value is "filename". A "host" label is always included.
# The stream lag metric indicates which streams are falling behind on writes to Loki;
# be mindful about using too many labels, as it can increase cardinality.
[stream_lag_labels: <string> | default = "filename"]
```
## Example Docker Config
It's fairly difficult to tail Docker files on a standalone machine because they are in different locations for every OS. We recommend the [Docker logging driver](../../docker-driver/) for local Docker installs or Docker Compose.

Write Preview
Loading…
Cancel
Save