From 00a3c5b026c0c1724d822207a3d439fa4351f2b4 Mon Sep 17 00:00:00 2001 From: Vladyslav Diachenko <82767850+vlad-diachenko@users.noreply.github.com> Date: Mon, 4 Sep 2023 14:03:20 +0300 Subject: [PATCH] renamed `non indexed labels` feature to `structured metadata` (#10432) Loki team has made a decision to rename the feature because the new name is more precise. --------- Signed-off-by: Vladyslav Diachenko Co-authored-by: Salva Corts --- CHANGELOG.md | 4 +- .../logentry/stages/nonindexedlabels_test.go | 132 ---------- clients/pkg/logentry/stages/stage.go | 57 ++--- ...indexedlabels.go => structuredmetadata.go} | 14 +- .../stages/structuredmetadata_test.go | 146 +++++++++++ clients/pkg/promtail/client/batch.go | 8 +- clients/pkg/promtail/client/client_test.go | 2 +- docs/sources/get-started/architecture.md | 82 +++---- docs/sources/get-started/labels/_index.md | 4 +- docs/sources/get-started/labels/bp-labels.md | 2 +- ...dexed-labels.md => structured-metadata.md} | 30 +-- docs/sources/reference/api.md | 4 +- .../send-data/promtail/stages/_index.md | 2 +- ...dexed_labels.md => structured_metadata.md} | 21 +- integration/client/client.go | 20 +- .../loki_micro_services_delete_test.go | 20 +- integration/loki_micro_services_test.go | 8 +- pkg/chunkenc/memchunk.go | 124 +++++----- pkg/chunkenc/memchunk_test.go | 192 +++++++-------- pkg/chunkenc/symbols.go | 10 +- pkg/chunkenc/unordered.go | 84 +++---- pkg/chunkenc/unordered_test.go | 88 +++---- pkg/chunkenc/util_test.go | 10 +- pkg/ingester/chunk_test.go | 2 +- pkg/ingester/encoding_test.go | 4 +- pkg/ingester/flush_test.go | 2 +- pkg/ingester/recovery_test.go | 14 +- pkg/ingester/wal/encoding.go | 26 +- pkg/ingester/wal/encoding_test.go | 44 ++-- pkg/iter/entry_iterator.go | 6 +- pkg/loghttp/entry.go | 26 +- pkg/loghttp/push/push.go | 34 +-- pkg/loghttp/push/push_test.go | 48 ++-- pkg/loghttp/query.go | 8 +- pkg/loghttp/query_test.go | 14 +- pkg/logql/log/metrics_extraction.go | 28 +-- pkg/logql/log/metrics_extraction_test.go | 126 +++++----- pkg/logql/log/pipeline.go | 32 +-- pkg/logql/log/pipeline_test.go | 60 ++--- pkg/logql/metrics.go | 2 +- pkg/logqlmodel/stats/context.go | 16 +- pkg/logqlmodel/stats/stats.pb.go | 226 +++++++++--------- pkg/logqlmodel/stats/stats.proto | 12 +- pkg/push/go.sum | 3 +- pkg/push/push.pb.go | 116 ++++----- pkg/push/push.proto | 4 +- pkg/push/types.go | 28 +-- pkg/push/types_test.go | 16 +- pkg/querier/http_test.go | 10 +- pkg/querier/queryrange/codec_test.go | 10 +- pkg/querier/queryrange/prometheus_test.go | 10 +- pkg/storage/chunk/cache/cache_test.go | 2 +- .../chunk/client/grpc/grpc_client_test.go | 2 +- .../chunk/client/testutils/testutils.go | 2 +- pkg/storage/chunk/fetcher/fetcher_test.go | 2 +- pkg/storage/hack/main.go | 2 +- .../compactor/deletion/delete_request.go | 4 +- .../compactor/deletion/delete_request_test.go | 50 ++-- .../deletion/delete_requests_manager.go | 4 +- .../deletion/delete_requests_manager_test.go | 62 ++--- .../compactor/retention/retention.go | 4 +- .../compactor/retention/retention_test.go | 26 +- pkg/storage/util_test.go | 2 +- pkg/util/filter/filter_function.go | 2 +- pkg/util/marshal/legacy/marshal_test.go | 18 +- pkg/util/marshal/marshal_test.go | 34 +-- pkg/util/marshal/query.go | 4 +- pkg/util/unmarshal/legacy/unmarshal_test.go | 4 +- pkg/util/unmarshal/unmarshal_test.go | 2 +- .../grafana/loki/pkg/push/push.pb.go | 116 ++++----- .../grafana/loki/pkg/push/push.proto | 4 +- .../github.com/grafana/loki/pkg/push/types.go | 28 +-- 72 files changed, 1191 insertions(+), 1174 deletions(-) delete mode 100644 clients/pkg/logentry/stages/nonindexedlabels_test.go rename clients/pkg/logentry/stages/{nonindexedlabels.go => structuredmetadata.go} (59%) create mode 100644 clients/pkg/logentry/stages/structuredmetadata_test.go rename docs/sources/get-started/labels/{non-indexed-labels.md => structured-metadata.md} (59%) rename docs/sources/send-data/promtail/stages/{non_indexed_labels.md => structured_metadata.md} (53%) diff --git a/CHANGELOG.md b/CHANGELOG.md index b1a1426f3f..06f1994e8a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -64,7 +64,7 @@ * [10344](https://github.com/grafana/loki/pull/10344) **ashwanthgoli** Compactor: deprecate `-boltdb.shipper.compactor.` prefix in favor of `-compactor.`. * [10373](https://github.com/grafana/loki/pull/10373) **jeschkies** Loki: Shard `avg_over_time` range aggregations. * [10377](https://github.com/grafana/loki/pull/10377) **shantanualsi** Remove deprecated config `-s3.sse-encryption` in favor or `-s3.sse.*` settings. -* [10073](https://github.com/grafana/loki/pull/10073) **sandeepsukhani,salvacorts,vlad-diachenko** Support attaching non-indexed labels to log lines. +* [10073](https://github.com/grafana/loki/pull/10073) **sandeepsukhani,salvacorts,vlad-diachenko** Support attaching structured metadata to log lines. * [10378](https://github.com/grafana/loki/pull/10378) **shantanualsi** Remove deprecated `ruler.wal-cleaer.period` * [10380](https://github.com/grafana/loki/pull/10380) **shantanualsi** Remove `experimental.ruler.enable-api` in favour of `ruler.enable-api` * [10395](https://github.com/grafana/loki/pull/10395/) **shantanualshi** Remove deprecated `split_queries_by_interval` and `forward_headers_list` configuration options in the `query_range` section @@ -103,7 +103,7 @@ * [8474](https://github.com/grafana/loki/pull/8787) **andriikushch**: Promtail: Add a new target for the Azure Event Hubs * [8874](https://github.com/grafana/loki/pull/8874) **rfratto**: Promtail: Support expoential backoff when polling unchanged files for logs. * [9508](https://github.com/grafana/loki/pull/9508) **farodin91**: Promtail: improve behavior of partial lines. -* [9986](https://github.com/grafana/loki/pull/9986) **vlad-diachenko**: Promtail: Add `non_indexed_labels` stage to attach non-indexed labels to each log line. +* [9986](https://github.com/grafana/loki/pull/9986) **vlad-diachenko**: Promtail: Add `structured_metadata` stage to attach metadata to each log line. ##### Fixes diff --git a/clients/pkg/logentry/stages/nonindexedlabels_test.go b/clients/pkg/logentry/stages/nonindexedlabels_test.go deleted file mode 100644 index 6542f1d9eb..0000000000 --- a/clients/pkg/logentry/stages/nonindexedlabels_test.go +++ /dev/null @@ -1,132 +0,0 @@ -package stages - -import ( - "testing" - "time" - - "github.com/prometheus/client_golang/prometheus" - "github.com/prometheus/common/model" - "github.com/stretchr/testify/require" - - "github.com/grafana/loki/pkg/push" - util_log "github.com/grafana/loki/pkg/util/log" -) - -var pipelineStagesNonIndexedLabelsUsingMatch = ` -pipeline_stages: -- match: - selector: '{source="test"}' - stages: - - logfmt: - mapping: - app: - - non_indexed_labels: - app: -` - -var pipelineStagesNonIndexedLabelsFromLogfmt = ` -pipeline_stages: -- logfmt: - mapping: - app: -- non_indexed_labels: - app: -` - -var pipelineStagesNonIndexedLabelsFromJSON = ` -pipeline_stages: -- json: - expressions: - app: -- non_indexed_labels: - app: -` - -var pipelineStagesNonIndexedLabelsWithRegexParser = ` -pipeline_stages: -- regex: - expression: "^(?s)(?P