Like Prometheus, but for logs.
You can not select more than 25 topics Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
loki/cmd/fluent-bit/config_test.go

208 lines
6.3 KiB

fluent-bit shared object go plugin (#847) * Import fluent-bit-go-loki plugin sources * Add fluent-bit-go package constraint * Ensure dependencies with dep ``` $ dep ensure ``` * Update .gitignore * Move fluent-bit-go-loki to fluent-bit/fluent-bit-go-loki directory * Add documentation for fluent-bit-go-loki * Fix for lint * Fix for goimports errors * Display fluent-bit-go-loki plugin version * Use more descriptive description for fluent-bit Loki plugin * Remove needless newlines * Rectify fluent-bit-go-loki makefile Remove needless parts and rename meaningless target * Use more descriptive description in README * refactor(fluent-bit): move make targets into root Makefile * Organize imports * Reorder imports with goimports $ goimports -w out_loki.go * Remove needless LICENSE file Because it is the same for Loki repository. * Rely on Loki client retry mechanism * Make descriptions more clearly * Change default unit * BatchWait: msec -> sec * BatchSize: KiB -> Byte * Use logql to parse Loki native labels format * Use logger instead of fmt.Printf * Update documentation to reflect recent changes * Use prometheus version module to print version * Dump error message with logger * Use DYN_GO_FLAGS instead of custom FLAGS for fluent-bit-plugin * Support RemoveKeys parameter * use logql.ParseMatchers() * refactor fluent-bit go plugin * add ci step for docker image * fix lint issues * fix a failing url test * fix a failing url test * add build images for drone ci * Adding labelmap file to complex record to labels Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add helm chart and fix a bug with labelmap * New push path * dep sync * Move fluent-bit folder and use jsonnet for drone * fix build * Adds more documentation and include a basic config in the container Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * remove binaries * Some readme grammar fixes * docs: really nitpicky grammar change in clients README * Add helm documentation * fixes bad config * bump chart * fixes .gitignore
6 years ago
package main
import (
"io/ioutil"
"net/url"
"os"
"reflect"
"testing"
"time"
"github.com/prometheus/common/model"
"github.com/cortexproject/cortex/pkg/util/flagext"
"github.com/grafana/loki/pkg/promtail/client"
lokiflag "github.com/grafana/loki/pkg/util/flagext"
"github.com/weaveworks/common/logging"
)
type fakeConfig map[string]string
func (f fakeConfig) Get(key string) string {
return f[key]
}
func Test_parseConfig(t *testing.T) {
fileName := createTempLabelMap(t)
defer os.Remove(fileName)
tests := []struct {
name string
conf map[string]string
want *config
wantErr bool
}{
{"defaults",
map[string]string{},
&config{
lineFormat: jsonFormat,
clientConfig: client.Config{
URL: mustParseURL("http://localhost:3100/loki/api/v1/push"),
BatchSize: defaultClientCfg.BatchSize,
BatchWait: defaultClientCfg.BatchWait,
ExternalLabels: lokiflag.LabelSet{LabelSet: model.LabelSet{"job": "fluent-bit"}},
},
logLevel: mustParseLogLevel("info"),
dropSingleKey: true,
},
false},
{"setting values",
map[string]string{
"URL": "http://somewhere.com:3100/loki/api/v1/push",
"LineFormat": "key_value",
"LogLevel": "warn",
"Labels": `{app="foo"}`,
"BatchWait": "30",
"BatchSize": "100",
"RemoveKeys": "buzz,fuzz",
"LabelKeys": "foo,bar",
"DropSingleKey": "false",
},
&config{
lineFormat: kvPairFormat,
clientConfig: client.Config{
URL: mustParseURL("http://somewhere.com:3100/loki/api/v1/push"),
BatchSize: 100,
BatchWait: 30 * time.Second,
ExternalLabels: lokiflag.LabelSet{LabelSet: model.LabelSet{"app": "foo"}},
},
logLevel: mustParseLogLevel("warn"),
labelKeys: []string{"foo", "bar"},
removeKeys: []string{"buzz", "fuzz"},
dropSingleKey: false,
},
false},
{"with label map",
map[string]string{
"URL": "http://somewhere.com:3100/loki/api/v1/push",
"LineFormat": "key_value",
"LogLevel": "warn",
"Labels": `{app="foo"}`,
"BatchWait": "30",
"BatchSize": "100",
"RemoveKeys": "buzz,fuzz",
"LabelKeys": "foo,bar",
"DropSingleKey": "false",
"LabelMapPath": fileName,
},
&config{
lineFormat: kvPairFormat,
clientConfig: client.Config{
URL: mustParseURL("http://somewhere.com:3100/loki/api/v1/push"),
BatchSize: 100,
BatchWait: 30 * time.Second,
ExternalLabels: lokiflag.LabelSet{LabelSet: model.LabelSet{"app": "foo"}},
},
logLevel: mustParseLogLevel("warn"),
labelKeys: nil,
removeKeys: []string{"buzz", "fuzz"},
dropSingleKey: false,
fluent-bit: sorted JSON and properly convert []byte to string (#1310) * fluent-bit: fix variable spelling mistake Signed-off-by: Jens Erat <email@jenserat.de> * fluent-bit: sort JSON map While developing another fix, I stumbled upon #1309 as a newly written unit test (with multiple key-value pairs in a map) was flaky. While JSON does not [strictly define an order on records in a map][RFC8259], but practical operations with a logging tool pretty much require it (although of course grep for JSON is jq, not grep...). Also, the key value output format is already sorted. Switching to sorted output in jsoniter is pretty easy. As of today, it still has a [bug] though, for which I already provided a [fix]. I propose accepting that rare case where invalid types can occur from msgpack output (can this even happen?) and re-enable the failing test case as soon as the upstream PR is merged. [RFC8259]: https://tools.ietf.org/html/rfc8259#section-4 [bug]: https://github.com/json-iterator/go/issues/388 [fix]: https://github.com/json-iterator/go/issues/422 Signed-off-by: Jens Erat <email@jenserat.de> * fluent-bit: properly convert []byte to string Recently, a regression was introduced that no longer ran a deep conversion of `[]byte` to `string` unless a label map was supplied. This commit fixes this by running the string conversion recursively, also removing the need of applying the conversion function again during label map stage. This change has two minor side effects: - Some test cases had to be moved, as string conversion happens much earlier now. - Invalid characters do not result in the whole label being ignored any more, but are replaced by the unicode placeholder character now. I'd consider this an improvement, making both debugging much easier ("why is that value suddenly missing?") and preserving as much information as possible. Signed-off-by: Jens Erat <email@jenserat.de>
6 years ago
labelMap: map[string]interface{}{
fluent-bit shared object go plugin (#847) * Import fluent-bit-go-loki plugin sources * Add fluent-bit-go package constraint * Ensure dependencies with dep ``` $ dep ensure ``` * Update .gitignore * Move fluent-bit-go-loki to fluent-bit/fluent-bit-go-loki directory * Add documentation for fluent-bit-go-loki * Fix for lint * Fix for goimports errors * Display fluent-bit-go-loki plugin version * Use more descriptive description for fluent-bit Loki plugin * Remove needless newlines * Rectify fluent-bit-go-loki makefile Remove needless parts and rename meaningless target * Use more descriptive description in README * refactor(fluent-bit): move make targets into root Makefile * Organize imports * Reorder imports with goimports $ goimports -w out_loki.go * Remove needless LICENSE file Because it is the same for Loki repository. * Rely on Loki client retry mechanism * Make descriptions more clearly * Change default unit * BatchWait: msec -> sec * BatchSize: KiB -> Byte * Use logql to parse Loki native labels format * Use logger instead of fmt.Printf * Update documentation to reflect recent changes * Use prometheus version module to print version * Dump error message with logger * Use DYN_GO_FLAGS instead of custom FLAGS for fluent-bit-plugin * Support RemoveKeys parameter * use logql.ParseMatchers() * refactor fluent-bit go plugin * add ci step for docker image * fix lint issues * fix a failing url test * fix a failing url test * add build images for drone ci * Adding labelmap file to complex record to labels Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add helm chart and fix a bug with labelmap * New push path * dep sync * Move fluent-bit folder and use jsonnet for drone * fix build * Adds more documentation and include a basic config in the container Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * remove binaries * Some readme grammar fixes * docs: really nitpicky grammar change in clients README * Add helm documentation * fixes bad config * bump chart * fixes .gitignore
6 years ago
"kubernetes": map[string]interface{}{
"container_name": "container",
"host": "host",
"namespace_name": "namespace",
"pod_name": "instance",
"labels": map[string]interface{}{
"component": "component",
"tier": "tier",
},
},
"stream": "stream",
},
},
false},
{"bad url", map[string]string{"URL": "::doh.com"}, nil, true},
{"bad BatchWait", map[string]string{"BatchWait": "a"}, nil, true},
{"bad BatchSize", map[string]string{"BatchSize": "a"}, nil, true},
{"bad labels", map[string]string{"Labels": "a"}, nil, true},
{"bad format", map[string]string{"LineFormat": "a"}, nil, true},
{"bad log level", map[string]string{"LogLevel": "a"}, nil, true},
{"bad drop single key", map[string]string{"DropSingleKey": "a"}, nil, true},
{"bad labelmap file", map[string]string{"LabelMapPath": "a"}, nil, true},
}
for _, tt := range tests {
t.Run(tt.name, func(t *testing.T) {
got, err := parseConfig(fakeConfig(tt.conf))
if (err != nil) != tt.wantErr {
t.Errorf("parseConfig() error = %v, wantErr %v", err, tt.wantErr)
return
}
if !tt.wantErr {
assertConfig(t, got, tt.want)
}
})
}
}
func assertConfig(t *testing.T, expected, actual *config) {
if expected.clientConfig.BatchSize != actual.clientConfig.BatchSize {
t.Errorf("incorrect batch size want:%v got:%v", expected.clientConfig.BatchSize, actual.clientConfig.BatchSize)
}
if !reflect.DeepEqual(expected.clientConfig.ExternalLabels, actual.clientConfig.ExternalLabels) {
t.Errorf("incorrect labels want:%v got:%v", expected.clientConfig.ExternalLabels, actual.clientConfig.ExternalLabels)
}
if expected.clientConfig.BatchWait != actual.clientConfig.BatchWait {
t.Errorf("incorrect batch wait want:%v got:%v", expected.clientConfig.BatchWait, actual.clientConfig.BatchWait)
}
if !reflect.DeepEqual(expected.clientConfig.URL, actual.clientConfig.URL) {
t.Errorf("incorrect URL want:%v got:%v", expected.clientConfig.URL, actual.clientConfig.URL)
}
if !reflect.DeepEqual(expected.lineFormat, actual.lineFormat) {
t.Errorf("incorrect lineFormat want:%v got:%v", expected.lineFormat, actual.lineFormat)
}
if !reflect.DeepEqual(expected.removeKeys, actual.removeKeys) {
t.Errorf("incorrect removeKeys want:%v got:%v", expected.removeKeys, actual.removeKeys)
}
if !reflect.DeepEqual(expected.labelKeys, actual.labelKeys) {
t.Errorf("incorrect labelKeys want:%v got:%v", expected.labelKeys, actual.labelKeys)
}
if expected.logLevel.String() != actual.logLevel.String() {
t.Errorf("incorrect logLevel want:%v got:%v", expected.logLevel.String(), actual.logLevel.String())
}
fluent-bit: sorted JSON and properly convert []byte to string (#1310) * fluent-bit: fix variable spelling mistake Signed-off-by: Jens Erat <email@jenserat.de> * fluent-bit: sort JSON map While developing another fix, I stumbled upon #1309 as a newly written unit test (with multiple key-value pairs in a map) was flaky. While JSON does not [strictly define an order on records in a map][RFC8259], but practical operations with a logging tool pretty much require it (although of course grep for JSON is jq, not grep...). Also, the key value output format is already sorted. Switching to sorted output in jsoniter is pretty easy. As of today, it still has a [bug] though, for which I already provided a [fix]. I propose accepting that rare case where invalid types can occur from msgpack output (can this even happen?) and re-enable the failing test case as soon as the upstream PR is merged. [RFC8259]: https://tools.ietf.org/html/rfc8259#section-4 [bug]: https://github.com/json-iterator/go/issues/388 [fix]: https://github.com/json-iterator/go/issues/422 Signed-off-by: Jens Erat <email@jenserat.de> * fluent-bit: properly convert []byte to string Recently, a regression was introduced that no longer ran a deep conversion of `[]byte` to `string` unless a label map was supplied. This commit fixes this by running the string conversion recursively, also removing the need of applying the conversion function again during label map stage. This change has two minor side effects: - Some test cases had to be moved, as string conversion happens much earlier now. - Invalid characters do not result in the whole label being ignored any more, but are replaced by the unicode placeholder character now. I'd consider this an improvement, making both debugging much easier ("why is that value suddenly missing?") and preserving as much information as possible. Signed-off-by: Jens Erat <email@jenserat.de>
6 years ago
if !reflect.DeepEqual(expected.labelMap, actual.labelMap) {
t.Errorf("incorrect labelMap want:%v got:%v", expected.labelMap, actual.labelMap)
fluent-bit shared object go plugin (#847) * Import fluent-bit-go-loki plugin sources * Add fluent-bit-go package constraint * Ensure dependencies with dep ``` $ dep ensure ``` * Update .gitignore * Move fluent-bit-go-loki to fluent-bit/fluent-bit-go-loki directory * Add documentation for fluent-bit-go-loki * Fix for lint * Fix for goimports errors * Display fluent-bit-go-loki plugin version * Use more descriptive description for fluent-bit Loki plugin * Remove needless newlines * Rectify fluent-bit-go-loki makefile Remove needless parts and rename meaningless target * Use more descriptive description in README * refactor(fluent-bit): move make targets into root Makefile * Organize imports * Reorder imports with goimports $ goimports -w out_loki.go * Remove needless LICENSE file Because it is the same for Loki repository. * Rely on Loki client retry mechanism * Make descriptions more clearly * Change default unit * BatchWait: msec -> sec * BatchSize: KiB -> Byte * Use logql to parse Loki native labels format * Use logger instead of fmt.Printf * Update documentation to reflect recent changes * Use prometheus version module to print version * Dump error message with logger * Use DYN_GO_FLAGS instead of custom FLAGS for fluent-bit-plugin * Support RemoveKeys parameter * use logql.ParseMatchers() * refactor fluent-bit go plugin * add ci step for docker image * fix lint issues * fix a failing url test * fix a failing url test * add build images for drone ci * Adding labelmap file to complex record to labels Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * Add helm chart and fix a bug with labelmap * New push path * dep sync * Move fluent-bit folder and use jsonnet for drone * fix build * Adds more documentation and include a basic config in the container Signed-off-by: Cyril Tovena <cyril.tovena@gmail.com> * remove binaries * Some readme grammar fixes * docs: really nitpicky grammar change in clients README * Add helm documentation * fixes bad config * bump chart * fixes .gitignore
6 years ago
}
}
func mustParseURL(u string) flagext.URLValue {
parsed, err := url.Parse(u)
if err != nil {
panic(err)
}
return flagext.URLValue{URL: parsed}
}
func mustParseLogLevel(l string) logging.Level {
level := logging.Level{}
err := level.Set(l)
if err != nil {
panic(err)
}
return level
}
func createTempLabelMap(t *testing.T) string {
file, err := ioutil.TempFile("", "labelmap")
if err != nil {
t.Fatal(err)
}
_, err = file.WriteString(`{
"kubernetes": {
"namespace_name": "namespace",
"labels": {
"component": "component",
"tier": "tier"
},
"host": "host",
"container_name": "container",
"pod_name": "instance"
},
"stream": "stream"
}`)
if err != nil {
t.Fatal(err)
}
return file.Name()
}