From 2139dbf55f6857a7c53947ad31566bbb2f6a506f Mon Sep 17 00:00:00 2001 From: Christian Haudum Date: Mon, 6 Feb 2023 15:07:59 +0100 Subject: [PATCH] [LogCLI] Load tenant-specific schema config file when using `--remote-schema` (#8413) Signed-off-by: Christian Haudum --- CHANGELOG.md | 6 ++++ docs/sources/tools/logcli.md | 3 -- pkg/logcli/query/query.go | 54 ++++++++++++++++++++++------------ pkg/logcli/query/query_test.go | 17 ++++++++--- 4 files changed, 55 insertions(+), 25 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 1c343c50ff..1877d5bcc4 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -65,6 +65,12 @@ ##### Changes +#### LogCLI + +##### Enhancement + +* [8413](https://github.com/grafana/loki/pull/8413) **chaudum**: Try to load tenant-specific `schemaconfig-{orgID}.yaml` when using `--remote-schema` argument and fallback to global `schemaconfig.yaml`. + #### Fluent Bit #### Loki Canary diff --git a/docs/sources/tools/logcli.md b/docs/sources/tools/logcli.md index 06e0245a4a..7eee81ed39 100644 --- a/docs/sources/tools/logcli.md +++ b/docs/sources/tools/logcli.md @@ -372,9 +372,6 @@ Flags: --remote-schema Execute the current query using a remote schema retrieved using the configured storage in the given Loki configuration file. - --remote-schema Execute the current query using a remote schema - retrieved using the configured storage in the given - Loki configuration file. --colored-output Show output with colored labels -t, --tail Tail the logs -f, --follow Alias for --tail diff --git a/pkg/logcli/query/query.go b/pkg/logcli/query/query.go index 77430d3f92..d9b7bd90f5 100644 --- a/pkg/logcli/query/query.go +++ b/pkg/logcli/query/query.go @@ -18,6 +18,7 @@ import ( "github.com/weaveworks/common/user" "gopkg.in/yaml.v2" + "github.com/grafana/dskit/multierror" "github.com/grafana/loki/pkg/logcli/client" "github.com/grafana/loki/pkg/logcli/output" "github.com/grafana/loki/pkg/loghttp" @@ -36,7 +37,7 @@ import ( "github.com/grafana/loki/pkg/validation" ) -const SchemaConfigFilename = "schemaconfig.yaml" +const schemaConfigFilename = "schemaconfig" type streamEntryPair struct { entry loghttp.Entry @@ -197,7 +198,11 @@ func (q *Query) DoLocalQuery(out output.LogOutput, statistics bool, orgID string return err } - loadedSchema, err := LoadSchemaUsingObjectClient(client, SchemaConfigFilename) + objects := []string{ + fmt.Sprintf("%s-%s.yaml", orgID, schemaConfigFilename), // schemaconfig-tenant.yaml + fmt.Sprintf("%s.yaml", schemaConfigFilename), // schemaconfig.yaml for backwards compatibility + } + loadedSchema, err := LoadSchemaUsingObjectClient(client, objects...) if err != nil { return err } @@ -284,24 +289,37 @@ type schemaConfigSection struct { config.SchemaConfig `yaml:"schema_config"` } -func LoadSchemaUsingObjectClient(oc chunk.ObjectClient, name string) (*config.SchemaConfig, error) { - ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) - defer cancel() - rdr, _, err := oc.GetObject(ctx, name) - if err != nil { - return nil, err - } - defer rdr.Close() +// LoadSchemaUsingObjectClient returns the loaded schema from the first found object +func LoadSchemaUsingObjectClient(oc chunk.ObjectClient, names ...string) (*config.SchemaConfig, error) { + errors := multierror.New() + for _, name := range names { + schema, err := func(name string) (*config.SchemaConfig, error) { + ctx, cancel := context.WithDeadline(context.Background(), time.Now().Add(5*time.Second)) + defer cancel() + rdr, _, err := oc.GetObject(ctx, name) + if err != nil { + return nil, err + } + defer rdr.Close() - decoder := yaml.NewDecoder(rdr) - decoder.SetStrict(true) - section := schemaConfigSection{} - err = decoder.Decode(§ion) - if err != nil { - return nil, err - } + decoder := yaml.NewDecoder(rdr) + decoder.SetStrict(true) + section := schemaConfigSection{} + err = decoder.Decode(§ion) + if err != nil { + return nil, err + } + + return §ion.SchemaConfig, nil + }(name) - return §ion.SchemaConfig, nil + if err != nil { + errors = append(errors, err) + continue + } + return schema, nil + } + return nil, errors.Err() } // SetInstant makes the Query an instant type diff --git a/pkg/logcli/query/query_test.go b/pkg/logcli/query/query_test.go index 8429b06f54..3de2537a25 100644 --- a/pkg/logcli/query/query_test.go +++ b/pkg/logcli/query/query_test.go @@ -614,19 +614,28 @@ func TestLoadFromURL(t *testing.T) { require.NoError(t, err) require.NotNil(t, client) - // Missing schema.config file should error - schemaConfig, err := LoadSchemaUsingObjectClient(client, SchemaConfigFilename) + filename := "schemaconfig.yaml" + + // Missing schemaconfig.yaml file should error + schemaConfig, err := LoadSchemaUsingObjectClient(client, filename) require.Error(t, err) require.Nil(t, schemaConfig) err = os.WriteFile( - filepath.Join(tmpDir, SchemaConfigFilename), + filepath.Join(tmpDir, filename), []byte(schemaConfigContents), 0666, ) require.NoError(t, err) - schemaConfig, err = LoadSchemaUsingObjectClient(client, SchemaConfigFilename) + // Load single schemaconfig.yaml + schemaConfig, err = LoadSchemaUsingObjectClient(client, filename) + + require.NoError(t, err) + require.NotNil(t, schemaConfig) + + // Load multiple schemaconfig files + schemaConfig, err = LoadSchemaUsingObjectClient(client, "foo.yaml", filename, "bar.yaml") require.NoError(t, err) require.NotNil(t, schemaConfig)