diff --git a/pkg/tsdb/loki/framing_test.go b/pkg/tsdb/loki/framing_test.go index 077444c0f82..fff9f446d88 100644 --- a/pkg/tsdb/loki/framing_test.go +++ b/pkg/tsdb/loki/framing_test.go @@ -53,6 +53,7 @@ func TestSuccessResponse(t *testing.T) { {name: "parse an empty response", filepath: "empty", query: matrixQuery}, {name: "parse structured metadata", filepath: "streams_structured_metadata", query: streamsQuery}, + {name: "parse structured metadata different labels each log line", filepath: "streams_structured_metadata_2", query: streamsQuery}, } runTest := func(folder string, path string, query lokiQuery, responseOpts ResponseOpts) { diff --git a/pkg/tsdb/loki/testdata/streams_structured_metadata_2.golden.jsonc b/pkg/tsdb/loki/testdata/streams_structured_metadata_2.golden.jsonc new file mode 100644 index 00000000000..2f65417a5af --- /dev/null +++ b/pkg/tsdb/loki/testdata/streams_structured_metadata_2.golden.jsonc @@ -0,0 +1,380 @@ +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +// +// Frame[0] { +// "typeVersion": [ +// 0, +// 0 +// ], +// "custom": { +// "frameType": "LabeledTimeValues" +// }, +// "stats": [ +// { +// "displayName": "Summary: bytes processed per second", +// "unit": "Bps", +// "value": 3507022 +// }, +// { +// "displayName": "Summary: lines processed per second", +// "value": 24818 +// }, +// { +// "displayName": "Summary: total bytes processed", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Summary: total lines processed", +// "value": 55 +// }, +// { +// "displayName": "Summary: exec time", +// "unit": "s", +// "value": 0.002216125 +// }, +// { +// "displayName": "Store: total chunks ref", +// "value": 2 +// }, +// { +// "displayName": "Store: total chunks downloaded", +// "value": 3 +// }, +// { +// "displayName": "Store: chunks download time", +// "unit": "s", +// "value": 0.000390958 +// }, +// { +// "displayName": "Store: head chunk bytes", +// "unit": "decbytes", +// "value": 4 +// }, +// { +// "displayName": "Store: head chunk lines", +// "value": 5 +// }, +// { +// "displayName": "Store: decompressed bytes", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Store: decompressed lines", +// "value": 55 +// }, +// { +// "displayName": "Store: compressed bytes", +// "unit": "decbytes", +// "value": 31432 +// }, +// { +// "displayName": "Store: total duplicates", +// "value": 6 +// }, +// { +// "displayName": "Ingester: total reached", +// "value": 7 +// }, +// { +// "displayName": "Ingester: total chunks matched", +// "value": 8 +// }, +// { +// "displayName": "Ingester: total batches", +// "value": 9 +// }, +// { +// "displayName": "Ingester: total lines sent", +// "value": 10 +// }, +// { +// "displayName": "Ingester: head chunk bytes", +// "unit": "decbytes", +// "value": 11 +// }, +// { +// "displayName": "Ingester: head chunk lines", +// "value": 12 +// }, +// { +// "displayName": "Ingester: decompressed bytes", +// "unit": "decbytes", +// "value": 13 +// }, +// { +// "displayName": "Ingester: decompressed lines", +// "value": 14 +// }, +// { +// "displayName": "Ingester: compressed bytes", +// "unit": "decbytes", +// "value": 15 +// }, +// { +// "displayName": "Ingester: total duplicates", +// "value": 16 +// } +// ], +// "executedQueryString": "Expr: query1" +// } +// Name: +// Dimensions: 6 Fields by 3 Rows +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// | Name: labels | Name: Time | Name: Line | Name: tsNs | Name: labelTypes | Name: id | +// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | Type: []string | +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// | {"code":"\",two","field2":"two","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000 | {"code":"I","field2":"P","location":"I"} | 1704895296244577000_194597ad | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000 | {"code":"I","field1":"P","location":"I"} | 1704895267503906000_90781cdf | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000 | {"code":"I","field1":"P","location":"I"} | 1704895245190222000_90781cdf | +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// +// +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +{ + "status": 200, + "frames": [ + { + "schema": { + "meta": { + "typeVersion": [ + 0, + 0 + ], + "custom": { + "frameType": "LabeledTimeValues" + }, + "stats": [ + { + "displayName": "Summary: bytes processed per second", + "unit": "Bps", + "value": 3507022 + }, + { + "displayName": "Summary: lines processed per second", + "value": 24818 + }, + { + "displayName": "Summary: total bytes processed", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Summary: total lines processed", + "value": 55 + }, + { + "displayName": "Summary: exec time", + "unit": "s", + "value": 0.002216125 + }, + { + "displayName": "Store: total chunks ref", + "value": 2 + }, + { + "displayName": "Store: total chunks downloaded", + "value": 3 + }, + { + "displayName": "Store: chunks download time", + "unit": "s", + "value": 0.000390958 + }, + { + "displayName": "Store: head chunk bytes", + "unit": "decbytes", + "value": 4 + }, + { + "displayName": "Store: head chunk lines", + "value": 5 + }, + { + "displayName": "Store: decompressed bytes", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Store: decompressed lines", + "value": 55 + }, + { + "displayName": "Store: compressed bytes", + "unit": "decbytes", + "value": 31432 + }, + { + "displayName": "Store: total duplicates", + "value": 6 + }, + { + "displayName": "Ingester: total reached", + "value": 7 + }, + { + "displayName": "Ingester: total chunks matched", + "value": 8 + }, + { + "displayName": "Ingester: total batches", + "value": 9 + }, + { + "displayName": "Ingester: total lines sent", + "value": 10 + }, + { + "displayName": "Ingester: head chunk bytes", + "unit": "decbytes", + "value": 11 + }, + { + "displayName": "Ingester: head chunk lines", + "value": 12 + }, + { + "displayName": "Ingester: decompressed bytes", + "unit": "decbytes", + "value": 13 + }, + { + "displayName": "Ingester: decompressed lines", + "value": 14 + }, + { + "displayName": "Ingester: compressed bytes", + "unit": "decbytes", + "value": 15 + }, + { + "displayName": "Ingester: total duplicates", + "value": 16 + } + ], + "executedQueryString": "Expr: query1" + }, + "fields": [ + { + "name": "labels", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + } + }, + { + "name": "Time", + "type": "time", + "typeInfo": { + "frame": "time.Time" + } + }, + { + "name": "Line", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "tsNs", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "labelTypes", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + }, + "config": { + "custom": { + "hidden": true + } + } + }, + { + "name": "id", + "type": "string", + "typeInfo": { + "frame": "string" + } + } + ] + }, + "data": { + "values": [ + [ + { + "code": "\",two", + "field2": "two", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + } + ], + [ + 1704895296244, + 1704895267503, + 1704895245190 + ], + [ + "{\"field2\":\"two\"}", + "{\"field1\":\"one\"}", + "{\"field1\":\"one\"}" + ], + [ + "1704895296244577000", + "1704895267503906000", + "1704895245190222000" + ], + [ + { + "code": "I", + "field2": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + } + ], + [ + "1704895296244577000_194597ad", + "1704895267503906000_90781cdf", + "1704895245190222000_90781cdf" + ] + ], + "nanos": [ + null, + [ + 577000, + 906000, + 222000 + ], + null, + null, + null, + null + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata/streams_structured_metadata_2.json b/pkg/tsdb/loki/testdata/streams_structured_metadata_2.json new file mode 100644 index 00000000000..45dc560f354 --- /dev/null +++ b/pkg/tsdb/loki/testdata/streams_structured_metadata_2.json @@ -0,0 +1,78 @@ +{ + "status": "success", + "data": { + "encodingFlags": [ + "categorize-labels" + ], + "resultType": "streams", + "result": [ + { + "stream": { + "code": "\",two", + "location": "moon๐ŸŒ™" + }, + "values": [ + [ + "1704895296244577000", + "{\"field2\":\"two\"}", + { + "parsed": { + "field2": "two" + } + } + ], + [ + "1704895267503906000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ], + [ + "1704895245190222000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 3507022, + "linesProcessedPerSecond": 24818, + "totalBytesProcessed": 7772, + "totalLinesProcessed": 55, + "execTime": 0.002216125 + }, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 3, + "chunksDownloadTime": 0.000390958, + "headChunkBytes": 4, + "headChunkLines": 5, + "decompressedBytes": 7772, + "decompressedLines": 55, + "compressedBytes": 31432, + "totalDuplicates": 6 + }, + "ingester": { + "totalReached": 7, + "totalChunksMatched": 8, + "totalBatches": 9, + "totalLinesSent": 10, + "headChunkBytes": 11, + "headChunkLines": 12, + "decompressedBytes": 13, + "decompressedLines": 14, + "compressedBytes": 15, + "totalDuplicates": 16 + } + } + } +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.golden.jsonc b/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.golden.jsonc new file mode 100644 index 00000000000..b2ea873dc1f --- /dev/null +++ b/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.golden.jsonc @@ -0,0 +1,363 @@ +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +// +// Frame[0] { +// "type": "log-lines", +// "typeVersion": [ +// 0, +// 0 +// ], +// "stats": [ +// { +// "displayName": "Summary: bytes processed per second", +// "unit": "Bps", +// "value": 3507022 +// }, +// { +// "displayName": "Summary: lines processed per second", +// "value": 24818 +// }, +// { +// "displayName": "Summary: total bytes processed", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Summary: total lines processed", +// "value": 55 +// }, +// { +// "displayName": "Summary: exec time", +// "unit": "s", +// "value": 0.002216125 +// }, +// { +// "displayName": "Store: total chunks ref", +// "value": 2 +// }, +// { +// "displayName": "Store: total chunks downloaded", +// "value": 3 +// }, +// { +// "displayName": "Store: chunks download time", +// "unit": "s", +// "value": 0.000390958 +// }, +// { +// "displayName": "Store: head chunk bytes", +// "unit": "decbytes", +// "value": 4 +// }, +// { +// "displayName": "Store: head chunk lines", +// "value": 5 +// }, +// { +// "displayName": "Store: decompressed bytes", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Store: decompressed lines", +// "value": 55 +// }, +// { +// "displayName": "Store: compressed bytes", +// "unit": "decbytes", +// "value": 31432 +// }, +// { +// "displayName": "Store: total duplicates", +// "value": 6 +// }, +// { +// "displayName": "Ingester: total reached", +// "value": 7 +// }, +// { +// "displayName": "Ingester: total chunks matched", +// "value": 8 +// }, +// { +// "displayName": "Ingester: total batches", +// "value": 9 +// }, +// { +// "displayName": "Ingester: total lines sent", +// "value": 10 +// }, +// { +// "displayName": "Ingester: head chunk bytes", +// "unit": "decbytes", +// "value": 11 +// }, +// { +// "displayName": "Ingester: head chunk lines", +// "value": 12 +// }, +// { +// "displayName": "Ingester: decompressed bytes", +// "unit": "decbytes", +// "value": 13 +// }, +// { +// "displayName": "Ingester: decompressed lines", +// "value": 14 +// }, +// { +// "displayName": "Ingester: compressed bytes", +// "unit": "decbytes", +// "value": 15 +// }, +// { +// "displayName": "Ingester: total duplicates", +// "value": 16 +// } +// ], +// "executedQueryString": "Expr: query1" +// } +// Name: +// Dimensions: 5 Fields by 3 Rows +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// | Name: labels | Name: timestamp | Name: body | Name: id | Name: labelTypes | +// | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// | {"code":"\",two","field2":"two","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000_194597ad | {"code":"I","field2":"P","location":"I"} | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000_90781cdf | {"code":"I","field1":"P","location":"I"} | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000_90781cdf | {"code":"I","field1":"P","location":"I"} | +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// +// +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +{ + "status": 200, + "frames": [ + { + "schema": { + "meta": { + "type": "log-lines", + "typeVersion": [ + 0, + 0 + ], + "stats": [ + { + "displayName": "Summary: bytes processed per second", + "unit": "Bps", + "value": 3507022 + }, + { + "displayName": "Summary: lines processed per second", + "value": 24818 + }, + { + "displayName": "Summary: total bytes processed", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Summary: total lines processed", + "value": 55 + }, + { + "displayName": "Summary: exec time", + "unit": "s", + "value": 0.002216125 + }, + { + "displayName": "Store: total chunks ref", + "value": 2 + }, + { + "displayName": "Store: total chunks downloaded", + "value": 3 + }, + { + "displayName": "Store: chunks download time", + "unit": "s", + "value": 0.000390958 + }, + { + "displayName": "Store: head chunk bytes", + "unit": "decbytes", + "value": 4 + }, + { + "displayName": "Store: head chunk lines", + "value": 5 + }, + { + "displayName": "Store: decompressed bytes", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Store: decompressed lines", + "value": 55 + }, + { + "displayName": "Store: compressed bytes", + "unit": "decbytes", + "value": 31432 + }, + { + "displayName": "Store: total duplicates", + "value": 6 + }, + { + "displayName": "Ingester: total reached", + "value": 7 + }, + { + "displayName": "Ingester: total chunks matched", + "value": 8 + }, + { + "displayName": "Ingester: total batches", + "value": 9 + }, + { + "displayName": "Ingester: total lines sent", + "value": 10 + }, + { + "displayName": "Ingester: head chunk bytes", + "unit": "decbytes", + "value": 11 + }, + { + "displayName": "Ingester: head chunk lines", + "value": 12 + }, + { + "displayName": "Ingester: decompressed bytes", + "unit": "decbytes", + "value": 13 + }, + { + "displayName": "Ingester: decompressed lines", + "value": 14 + }, + { + "displayName": "Ingester: compressed bytes", + "unit": "decbytes", + "value": 15 + }, + { + "displayName": "Ingester: total duplicates", + "value": 16 + } + ], + "executedQueryString": "Expr: query1" + }, + "fields": [ + { + "name": "labels", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + } + }, + { + "name": "timestamp", + "type": "time", + "typeInfo": { + "frame": "time.Time" + } + }, + { + "name": "body", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "id", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "labelTypes", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + }, + "config": { + "custom": { + "hidden": true + } + } + } + ] + }, + "data": { + "values": [ + [ + { + "code": "\",two", + "field2": "two", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + } + ], + [ + 1704895296244, + 1704895267503, + 1704895245190 + ], + [ + "{\"field2\":\"two\"}", + "{\"field1\":\"one\"}", + "{\"field1\":\"one\"}" + ], + [ + "1704895296244577000_194597ad", + "1704895267503906000_90781cdf", + "1704895245190222000_90781cdf" + ], + [ + { + "code": "I", + "field2": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + } + ] + ], + "nanos": [ + null, + [ + 577000, + 906000, + 222000 + ], + null, + null, + null + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.json b/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.json new file mode 100644 index 00000000000..45dc560f354 --- /dev/null +++ b/pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.json @@ -0,0 +1,78 @@ +{ + "status": "success", + "data": { + "encodingFlags": [ + "categorize-labels" + ], + "resultType": "streams", + "result": [ + { + "stream": { + "code": "\",two", + "location": "moon๐ŸŒ™" + }, + "values": [ + [ + "1704895296244577000", + "{\"field2\":\"two\"}", + { + "parsed": { + "field2": "two" + } + } + ], + [ + "1704895267503906000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ], + [ + "1704895245190222000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 3507022, + "linesProcessedPerSecond": 24818, + "totalBytesProcessed": 7772, + "totalLinesProcessed": 55, + "execTime": 0.002216125 + }, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 3, + "chunksDownloadTime": 0.000390958, + "headChunkBytes": 4, + "headChunkLines": 5, + "decompressedBytes": 7772, + "decompressedLines": 55, + "compressedBytes": 31432, + "totalDuplicates": 6 + }, + "ingester": { + "totalReached": 7, + "totalChunksMatched": 8, + "totalBatches": 9, + "totalLinesSent": 10, + "headChunkBytes": 11, + "headChunkLines": 12, + "decompressedBytes": 13, + "decompressedLines": 14, + "compressedBytes": 15, + "totalDuplicates": 16 + } + } + } +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.golden.jsonc b/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.golden.jsonc new file mode 100644 index 00000000000..b2ea873dc1f --- /dev/null +++ b/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.golden.jsonc @@ -0,0 +1,363 @@ +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +// +// Frame[0] { +// "type": "log-lines", +// "typeVersion": [ +// 0, +// 0 +// ], +// "stats": [ +// { +// "displayName": "Summary: bytes processed per second", +// "unit": "Bps", +// "value": 3507022 +// }, +// { +// "displayName": "Summary: lines processed per second", +// "value": 24818 +// }, +// { +// "displayName": "Summary: total bytes processed", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Summary: total lines processed", +// "value": 55 +// }, +// { +// "displayName": "Summary: exec time", +// "unit": "s", +// "value": 0.002216125 +// }, +// { +// "displayName": "Store: total chunks ref", +// "value": 2 +// }, +// { +// "displayName": "Store: total chunks downloaded", +// "value": 3 +// }, +// { +// "displayName": "Store: chunks download time", +// "unit": "s", +// "value": 0.000390958 +// }, +// { +// "displayName": "Store: head chunk bytes", +// "unit": "decbytes", +// "value": 4 +// }, +// { +// "displayName": "Store: head chunk lines", +// "value": 5 +// }, +// { +// "displayName": "Store: decompressed bytes", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Store: decompressed lines", +// "value": 55 +// }, +// { +// "displayName": "Store: compressed bytes", +// "unit": "decbytes", +// "value": 31432 +// }, +// { +// "displayName": "Store: total duplicates", +// "value": 6 +// }, +// { +// "displayName": "Ingester: total reached", +// "value": 7 +// }, +// { +// "displayName": "Ingester: total chunks matched", +// "value": 8 +// }, +// { +// "displayName": "Ingester: total batches", +// "value": 9 +// }, +// { +// "displayName": "Ingester: total lines sent", +// "value": 10 +// }, +// { +// "displayName": "Ingester: head chunk bytes", +// "unit": "decbytes", +// "value": 11 +// }, +// { +// "displayName": "Ingester: head chunk lines", +// "value": 12 +// }, +// { +// "displayName": "Ingester: decompressed bytes", +// "unit": "decbytes", +// "value": 13 +// }, +// { +// "displayName": "Ingester: decompressed lines", +// "value": 14 +// }, +// { +// "displayName": "Ingester: compressed bytes", +// "unit": "decbytes", +// "value": 15 +// }, +// { +// "displayName": "Ingester: total duplicates", +// "value": 16 +// } +// ], +// "executedQueryString": "Expr: query1" +// } +// Name: +// Dimensions: 5 Fields by 3 Rows +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// | Name: labels | Name: timestamp | Name: body | Name: id | Name: labelTypes | +// | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// | {"code":"\",two","field2":"two","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000_194597ad | {"code":"I","field2":"P","location":"I"} | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000_90781cdf | {"code":"I","field1":"P","location":"I"} | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000_90781cdf | {"code":"I","field1":"P","location":"I"} | +// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+ +// +// +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +{ + "status": 200, + "frames": [ + { + "schema": { + "meta": { + "type": "log-lines", + "typeVersion": [ + 0, + 0 + ], + "stats": [ + { + "displayName": "Summary: bytes processed per second", + "unit": "Bps", + "value": 3507022 + }, + { + "displayName": "Summary: lines processed per second", + "value": 24818 + }, + { + "displayName": "Summary: total bytes processed", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Summary: total lines processed", + "value": 55 + }, + { + "displayName": "Summary: exec time", + "unit": "s", + "value": 0.002216125 + }, + { + "displayName": "Store: total chunks ref", + "value": 2 + }, + { + "displayName": "Store: total chunks downloaded", + "value": 3 + }, + { + "displayName": "Store: chunks download time", + "unit": "s", + "value": 0.000390958 + }, + { + "displayName": "Store: head chunk bytes", + "unit": "decbytes", + "value": 4 + }, + { + "displayName": "Store: head chunk lines", + "value": 5 + }, + { + "displayName": "Store: decompressed bytes", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Store: decompressed lines", + "value": 55 + }, + { + "displayName": "Store: compressed bytes", + "unit": "decbytes", + "value": 31432 + }, + { + "displayName": "Store: total duplicates", + "value": 6 + }, + { + "displayName": "Ingester: total reached", + "value": 7 + }, + { + "displayName": "Ingester: total chunks matched", + "value": 8 + }, + { + "displayName": "Ingester: total batches", + "value": 9 + }, + { + "displayName": "Ingester: total lines sent", + "value": 10 + }, + { + "displayName": "Ingester: head chunk bytes", + "unit": "decbytes", + "value": 11 + }, + { + "displayName": "Ingester: head chunk lines", + "value": 12 + }, + { + "displayName": "Ingester: decompressed bytes", + "unit": "decbytes", + "value": 13 + }, + { + "displayName": "Ingester: decompressed lines", + "value": 14 + }, + { + "displayName": "Ingester: compressed bytes", + "unit": "decbytes", + "value": 15 + }, + { + "displayName": "Ingester: total duplicates", + "value": 16 + } + ], + "executedQueryString": "Expr: query1" + }, + "fields": [ + { + "name": "labels", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + } + }, + { + "name": "timestamp", + "type": "time", + "typeInfo": { + "frame": "time.Time" + } + }, + { + "name": "body", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "id", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "labelTypes", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + }, + "config": { + "custom": { + "hidden": true + } + } + } + ] + }, + "data": { + "values": [ + [ + { + "code": "\",two", + "field2": "two", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + } + ], + [ + 1704895296244, + 1704895267503, + 1704895245190 + ], + [ + "{\"field2\":\"two\"}", + "{\"field1\":\"one\"}", + "{\"field1\":\"one\"}" + ], + [ + "1704895296244577000_194597ad", + "1704895267503906000_90781cdf", + "1704895245190222000_90781cdf" + ], + [ + { + "code": "I", + "field2": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + } + ] + ], + "nanos": [ + null, + [ + 577000, + 906000, + 222000 + ], + null, + null, + null + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.json b/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.json new file mode 100644 index 00000000000..45dc560f354 --- /dev/null +++ b/pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.json @@ -0,0 +1,78 @@ +{ + "status": "success", + "data": { + "encodingFlags": [ + "categorize-labels" + ], + "resultType": "streams", + "result": [ + { + "stream": { + "code": "\",two", + "location": "moon๐ŸŒ™" + }, + "values": [ + [ + "1704895296244577000", + "{\"field2\":\"two\"}", + { + "parsed": { + "field2": "two" + } + } + ], + [ + "1704895267503906000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ], + [ + "1704895245190222000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 3507022, + "linesProcessedPerSecond": 24818, + "totalBytesProcessed": 7772, + "totalLinesProcessed": 55, + "execTime": 0.002216125 + }, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 3, + "chunksDownloadTime": 0.000390958, + "headChunkBytes": 4, + "headChunkLines": 5, + "decompressedBytes": 7772, + "decompressedLines": 55, + "compressedBytes": 31432, + "totalDuplicates": 6 + }, + "ingester": { + "totalReached": 7, + "totalChunksMatched": 8, + "totalBatches": 9, + "totalLinesSent": 10, + "headChunkBytes": 11, + "headChunkLines": 12, + "decompressedBytes": 13, + "decompressedLines": 14, + "compressedBytes": 15, + "totalDuplicates": 16 + } + } + } +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.golden.jsonc b/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.golden.jsonc new file mode 100644 index 00000000000..2f65417a5af --- /dev/null +++ b/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.golden.jsonc @@ -0,0 +1,380 @@ +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +// +// Frame[0] { +// "typeVersion": [ +// 0, +// 0 +// ], +// "custom": { +// "frameType": "LabeledTimeValues" +// }, +// "stats": [ +// { +// "displayName": "Summary: bytes processed per second", +// "unit": "Bps", +// "value": 3507022 +// }, +// { +// "displayName": "Summary: lines processed per second", +// "value": 24818 +// }, +// { +// "displayName": "Summary: total bytes processed", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Summary: total lines processed", +// "value": 55 +// }, +// { +// "displayName": "Summary: exec time", +// "unit": "s", +// "value": 0.002216125 +// }, +// { +// "displayName": "Store: total chunks ref", +// "value": 2 +// }, +// { +// "displayName": "Store: total chunks downloaded", +// "value": 3 +// }, +// { +// "displayName": "Store: chunks download time", +// "unit": "s", +// "value": 0.000390958 +// }, +// { +// "displayName": "Store: head chunk bytes", +// "unit": "decbytes", +// "value": 4 +// }, +// { +// "displayName": "Store: head chunk lines", +// "value": 5 +// }, +// { +// "displayName": "Store: decompressed bytes", +// "unit": "decbytes", +// "value": 7772 +// }, +// { +// "displayName": "Store: decompressed lines", +// "value": 55 +// }, +// { +// "displayName": "Store: compressed bytes", +// "unit": "decbytes", +// "value": 31432 +// }, +// { +// "displayName": "Store: total duplicates", +// "value": 6 +// }, +// { +// "displayName": "Ingester: total reached", +// "value": 7 +// }, +// { +// "displayName": "Ingester: total chunks matched", +// "value": 8 +// }, +// { +// "displayName": "Ingester: total batches", +// "value": 9 +// }, +// { +// "displayName": "Ingester: total lines sent", +// "value": 10 +// }, +// { +// "displayName": "Ingester: head chunk bytes", +// "unit": "decbytes", +// "value": 11 +// }, +// { +// "displayName": "Ingester: head chunk lines", +// "value": 12 +// }, +// { +// "displayName": "Ingester: decompressed bytes", +// "unit": "decbytes", +// "value": 13 +// }, +// { +// "displayName": "Ingester: decompressed lines", +// "value": 14 +// }, +// { +// "displayName": "Ingester: compressed bytes", +// "unit": "decbytes", +// "value": 15 +// }, +// { +// "displayName": "Ingester: total duplicates", +// "value": 16 +// } +// ], +// "executedQueryString": "Expr: query1" +// } +// Name: +// Dimensions: 6 Fields by 3 Rows +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// | Name: labels | Name: Time | Name: Line | Name: tsNs | Name: labelTypes | Name: id | +// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | +// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | Type: []string | +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// | {"code":"\",two","field2":"two","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000 | {"code":"I","field2":"P","location":"I"} | 1704895296244577000_194597ad | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000 | {"code":"I","field1":"P","location":"I"} | 1704895267503906000_90781cdf | +// | {"code":"\",two","field1":"one","location":"moon๐ŸŒ™"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000 | {"code":"I","field1":"P","location":"I"} | 1704895245190222000_90781cdf | +// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+ +// +// +// ๐ŸŒŸ This was machine generated. Do not edit. ๐ŸŒŸ +{ + "status": 200, + "frames": [ + { + "schema": { + "meta": { + "typeVersion": [ + 0, + 0 + ], + "custom": { + "frameType": "LabeledTimeValues" + }, + "stats": [ + { + "displayName": "Summary: bytes processed per second", + "unit": "Bps", + "value": 3507022 + }, + { + "displayName": "Summary: lines processed per second", + "value": 24818 + }, + { + "displayName": "Summary: total bytes processed", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Summary: total lines processed", + "value": 55 + }, + { + "displayName": "Summary: exec time", + "unit": "s", + "value": 0.002216125 + }, + { + "displayName": "Store: total chunks ref", + "value": 2 + }, + { + "displayName": "Store: total chunks downloaded", + "value": 3 + }, + { + "displayName": "Store: chunks download time", + "unit": "s", + "value": 0.000390958 + }, + { + "displayName": "Store: head chunk bytes", + "unit": "decbytes", + "value": 4 + }, + { + "displayName": "Store: head chunk lines", + "value": 5 + }, + { + "displayName": "Store: decompressed bytes", + "unit": "decbytes", + "value": 7772 + }, + { + "displayName": "Store: decompressed lines", + "value": 55 + }, + { + "displayName": "Store: compressed bytes", + "unit": "decbytes", + "value": 31432 + }, + { + "displayName": "Store: total duplicates", + "value": 6 + }, + { + "displayName": "Ingester: total reached", + "value": 7 + }, + { + "displayName": "Ingester: total chunks matched", + "value": 8 + }, + { + "displayName": "Ingester: total batches", + "value": 9 + }, + { + "displayName": "Ingester: total lines sent", + "value": 10 + }, + { + "displayName": "Ingester: head chunk bytes", + "unit": "decbytes", + "value": 11 + }, + { + "displayName": "Ingester: head chunk lines", + "value": 12 + }, + { + "displayName": "Ingester: decompressed bytes", + "unit": "decbytes", + "value": 13 + }, + { + "displayName": "Ingester: decompressed lines", + "value": 14 + }, + { + "displayName": "Ingester: compressed bytes", + "unit": "decbytes", + "value": 15 + }, + { + "displayName": "Ingester: total duplicates", + "value": 16 + } + ], + "executedQueryString": "Expr: query1" + }, + "fields": [ + { + "name": "labels", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + } + }, + { + "name": "Time", + "type": "time", + "typeInfo": { + "frame": "time.Time" + } + }, + { + "name": "Line", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "tsNs", + "type": "string", + "typeInfo": { + "frame": "string" + } + }, + { + "name": "labelTypes", + "type": "other", + "typeInfo": { + "frame": "json.RawMessage" + }, + "config": { + "custom": { + "hidden": true + } + } + }, + { + "name": "id", + "type": "string", + "typeInfo": { + "frame": "string" + } + } + ] + }, + "data": { + "values": [ + [ + { + "code": "\",two", + "field2": "two", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + }, + { + "code": "\",two", + "field1": "one", + "location": "moon๐ŸŒ™" + } + ], + [ + 1704895296244, + 1704895267503, + 1704895245190 + ], + [ + "{\"field2\":\"two\"}", + "{\"field1\":\"one\"}", + "{\"field1\":\"one\"}" + ], + [ + "1704895296244577000", + "1704895267503906000", + "1704895245190222000" + ], + [ + { + "code": "I", + "field2": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + }, + { + "code": "I", + "field1": "P", + "location": "I" + } + ], + [ + "1704895296244577000_194597ad", + "1704895267503906000_90781cdf", + "1704895245190222000_90781cdf" + ] + ], + "nanos": [ + null, + [ + 577000, + 906000, + 222000 + ], + null, + null, + null, + null + ] + } + } + ] +} \ No newline at end of file diff --git a/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.json b/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.json new file mode 100644 index 00000000000..45dc560f354 --- /dev/null +++ b/pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.json @@ -0,0 +1,78 @@ +{ + "status": "success", + "data": { + "encodingFlags": [ + "categorize-labels" + ], + "resultType": "streams", + "result": [ + { + "stream": { + "code": "\",two", + "location": "moon๐ŸŒ™" + }, + "values": [ + [ + "1704895296244577000", + "{\"field2\":\"two\"}", + { + "parsed": { + "field2": "two" + } + } + ], + [ + "1704895267503906000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ], + [ + "1704895245190222000", + "{\"field1\":\"one\"}", + { + "parsed": { + "field1": "one" + } + } + ] + ] + } + ], + "stats": { + "summary": { + "bytesProcessedPerSecond": 3507022, + "linesProcessedPerSecond": 24818, + "totalBytesProcessed": 7772, + "totalLinesProcessed": 55, + "execTime": 0.002216125 + }, + "store": { + "totalChunksRef": 2, + "totalChunksDownloaded": 3, + "chunksDownloadTime": 0.000390958, + "headChunkBytes": 4, + "headChunkLines": 5, + "decompressedBytes": 7772, + "decompressedLines": 55, + "compressedBytes": 31432, + "totalDuplicates": 6 + }, + "ingester": { + "totalReached": 7, + "totalChunksMatched": 8, + "totalBatches": 9, + "totalLinesSent": 10, + "headChunkBytes": 11, + "headChunkLines": 12, + "decompressedBytes": 13, + "decompressedLines": 14, + "compressedBytes": 15, + "totalDuplicates": 16 + } + } + } +} \ No newline at end of file diff --git a/pkg/util/converter/prom.go b/pkg/util/converter/prom.go index c74d7265592..fcdd6f8a23b 100644 --- a/pkg/util/converter/prom.go +++ b/pkg/util/converter/prom.go @@ -965,7 +965,7 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse { tsField := data.NewFieldFromFieldType(data.FieldTypeString, 0) tsField.Name = "TS" - labels := data.Labels{} + indexedLabels := data.Labels{} for more, err := iter.ReadArray(); more; more, err = iter.ReadArray() { if err != nil { @@ -981,8 +981,8 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse { case "stream": // we need to clear `labels`, because `iter.ReadVal` // only appends to it - labels = data.Labels{} - if err = iter.ReadVal(&labels); err != nil { + indexedLabels = data.Labels{} + if err = iter.ReadVal(&indexedLabels); err != nil { return rspErr(err) } @@ -1030,23 +1030,25 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse { } typeMap := data.Labels{} + clonedLabels := data.Labels{} - for k := range labels { + for k := range indexedLabels { typeMap[k] = "I" + clonedLabels[k] = indexedLabels[k] } // merge all labels (indexed, parsed, structuredMetadata) into one dataframe field for k, v := range structuredMetadataMap { - labels[k] = fmt.Sprintf("%s", v) + clonedLabels[k] = fmt.Sprintf("%s", v) typeMap[k] = "S" } for k, v := range parsedLabelsMap { - labels[k] = fmt.Sprintf("%s", v) + clonedLabels[k] = fmt.Sprintf("%s", v) typeMap[k] = "P" } - labelJson, err := labelsToRawJson(labels) + labelJson, err := labelsToRawJson(clonedLabels) if err != nil { return rspErr(err) }