apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Loki: Fix bug duplicating parsed labels across multiple log lines (#80292)

Browse Source
pull/80797/head
Sven Grossmann 1 year ago committed by GitHub
parent 1947919516
commit e1aa8a95d9
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
10 changed files with 1808 additions and 7 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 1
      pkg/tsdb/loki/framing_test.go
  2. 380
      pkg/tsdb/loki/testdata/streams_structured_metadata_2.golden.jsonc
  3. 78
      pkg/tsdb/loki/testdata/streams_structured_metadata_2.json
  4. 363
      pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.golden.jsonc
  5. 78
      pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.json
  6. 363
      pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.golden.jsonc
  7. 78
      pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.json
  8. 380
      pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.golden.jsonc
  9. 78
      pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.json
  10. 16
      pkg/util/converter/prom.go

1
pkg/tsdb/loki/framing_test.go
Unescape View File

@ -53,6 +53,7 @@ func TestSuccessResponse(t *testing.T) {
{name: "parse an empty response", filepath: "empty", query: matrixQuery},
{name: "parse structured metadata", filepath: "streams_structured_metadata", query: streamsQuery},
{name: "parse structured metadata different labels each log line", filepath: "streams_structured_metadata_2", query: streamsQuery},
}
runTest := func(folder string, path string, query lokiQuery, responseOpts ResponseOpts) {

380
pkg/tsdb/loki/testdata/streams_structured_metadata_2.golden.jsonc vendored
Unescape View File

@ -0,0 +1,380 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "custom": {
// "frameType": "LabeledTimeValues"
// },
// "stats": [
// {
// "displayName": "Summary: bytes processed per second",
// "unit": "Bps",
// "value": 3507022
// },
// {
// "displayName": "Summary: lines processed per second",
// "value": 24818
// },
// {
// "displayName": "Summary: total bytes processed",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Summary: total lines processed",
// "value": 55
// },
// {
// "displayName": "Summary: exec time",
// "unit": "s",
// "value": 0.002216125
// },
// {
// "displayName": "Store: total chunks ref",
// "value": 2
// },
// {
// "displayName": "Store: total chunks downloaded",
// "value": 3
// },
// {
// "displayName": "Store: chunks download time",
// "unit": "s",
// "value": 0.000390958
// },
// {
// "displayName": "Store: head chunk bytes",
// "unit": "decbytes",
// "value": 4
// },
// {
// "displayName": "Store: head chunk lines",
// "value": 5
// },
// {
// "displayName": "Store: decompressed bytes",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Store: decompressed lines",
// "value": 55
// },
// {
// "displayName": "Store: compressed bytes",
// "unit": "decbytes",
// "value": 31432
// },
// {
// "displayName": "Store: total duplicates",
// "value": 6
// },
// {
// "displayName": "Ingester: total reached",
// "value": 7
// },
// {
// "displayName": "Ingester: total chunks matched",
// "value": 8
// },
// {
// "displayName": "Ingester: total batches",
// "value": 9
// },
// {
// "displayName": "Ingester: total lines sent",
// "value": 10
// },
// {
// "displayName": "Ingester: head chunk bytes",
// "unit": "decbytes",
// "value": 11
// },
// {
// "displayName": "Ingester: head chunk lines",
// "value": 12
// },
// {
// "displayName": "Ingester: decompressed bytes",
// "unit": "decbytes",
// "value": 13
// },
// {
// "displayName": "Ingester: decompressed lines",
// "value": 14
// },
// {
// "displayName": "Ingester: compressed bytes",
// "unit": "decbytes",
// "value": 15
// },
// {
// "displayName": "Ingester: total duplicates",
// "value": 16
// }
// ],
// "executedQueryString": "Expr: query1"
// }
// Name:
// Dimensions: 6 Fields by 3 Rows
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
// | Name: labels | Name: Time | Name: Line | Name: tsNs | Name: labelTypes | Name: id |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | Type: []string |
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
// | {"code":"\",two","field2":"two","location":"moon🌙"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000 | {"code":"I","field2":"P","location":"I"} | 1704895296244577000_194597ad |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000 | {"code":"I","field1":"P","location":"I"} | 1704895267503906000_90781cdf |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000 | {"code":"I","field1":"P","location":"I"} | 1704895245190222000_90781cdf |
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"custom": {
"frameType": "LabeledTimeValues"
},
"stats": [
{
"displayName": "Summary: bytes processed per second",
"unit": "Bps",
"value": 3507022
},
{
"displayName": "Summary: lines processed per second",
"value": 24818
},
{
"displayName": "Summary: total bytes processed",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Summary: total lines processed",
"value": 55
},
{
"displayName": "Summary: exec time",
"unit": "s",
"value": 0.002216125
},
{
"displayName": "Store: total chunks ref",
"value": 2
},
{
"displayName": "Store: total chunks downloaded",
"value": 3
},
{
"displayName": "Store: chunks download time",
"unit": "s",
"value": 0.000390958
},
{
"displayName": "Store: head chunk bytes",
"unit": "decbytes",
"value": 4
},
{
"displayName": "Store: head chunk lines",
"value": 5
},
{
"displayName": "Store: decompressed bytes",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Store: decompressed lines",
"value": 55
},
{
"displayName": "Store: compressed bytes",
"unit": "decbytes",
"value": 31432
},
{
"displayName": "Store: total duplicates",
"value": 6
},
{
"displayName": "Ingester: total reached",
"value": 7
},
{
"displayName": "Ingester: total chunks matched",
"value": 8
},
{
"displayName": "Ingester: total batches",
"value": 9
},
{
"displayName": "Ingester: total lines sent",
"value": 10
},
{
"displayName": "Ingester: head chunk bytes",
"unit": "decbytes",
"value": 11
},
{
"displayName": "Ingester: head chunk lines",
"value": 12
},
{
"displayName": "Ingester: decompressed bytes",
"unit": "decbytes",
"value": 13
},
{
"displayName": "Ingester: decompressed lines",
"value": 14
},
{
"displayName": "Ingester: compressed bytes",
"unit": "decbytes",
"value": 15
},
{
"displayName": "Ingester: total duplicates",
"value": 16
}
],
"executedQueryString": "Expr: query1"
},
"fields": [
{
"name": "labels",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
}
},
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
}
},
{
"name": "Line",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "tsNs",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "labelTypes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
},
"config": {
"custom": {
"hidden": true
}
}
},
{
"name": "id",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
{
"code": "\",two",
"field2": "two",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
}
],
[
1704895296244,
1704895267503,
1704895245190
],
[
"{\"field2\":\"two\"}",
"{\"field1\":\"one\"}",
"{\"field1\":\"one\"}"
],
[
"1704895296244577000",
"1704895267503906000",
"1704895245190222000"
],
[
{
"code": "I",
"field2": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
}
],
[
"1704895296244577000_194597ad",
"1704895267503906000_90781cdf",
"1704895245190222000_90781cdf"
]
],
"nanos": [
null,
[
577000,
906000,
222000
],
null,
null,
null,
null
]
}
}
]
}

78
pkg/tsdb/loki/testdata/streams_structured_metadata_2.json vendored
Unescape View File

@ -0,0 +1,78 @@
{
"status": "success",
"data": {
"encodingFlags": [
"categorize-labels"
],
"resultType": "streams",
"result": [
{
"stream": {
"code": "\",two",
"location": "moon🌙"
},
"values": [
[
"1704895296244577000",
"{\"field2\":\"two\"}",
{
"parsed": {
"field2": "two"
}
}
],
[
"1704895267503906000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
],
[
"1704895245190222000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3507022,
"linesProcessedPerSecond": 24818,
"totalBytesProcessed": 7772,
"totalLinesProcessed": 55,
"execTime": 0.002216125
},
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 3,
"chunksDownloadTime": 0.000390958,
"headChunkBytes": 4,
"headChunkLines": 5,
"decompressedBytes": 7772,
"decompressedLines": 55,
"compressedBytes": 31432,
"totalDuplicates": 6
},
"ingester": {
"totalReached": 7,
"totalChunksMatched": 8,
"totalBatches": 9,
"totalLinesSent": 10,
"headChunkBytes": 11,
"headChunkLines": 12,
"decompressedBytes": 13,
"decompressedLines": 14,
"compressedBytes": 15,
"totalDuplicates": 16
}
}
}
}

363
pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.golden.jsonc
Unescape View File

@ -0,0 +1,363 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "type": "log-lines",
// "typeVersion": [
// 0,
// 0
// ],
// "stats": [
// {
// "displayName": "Summary: bytes processed per second",
// "unit": "Bps",
// "value": 3507022
// },
// {
// "displayName": "Summary: lines processed per second",
// "value": 24818
// },
// {
// "displayName": "Summary: total bytes processed",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Summary: total lines processed",
// "value": 55
// },
// {
// "displayName": "Summary: exec time",
// "unit": "s",
// "value": 0.002216125
// },
// {
// "displayName": "Store: total chunks ref",
// "value": 2
// },
// {
// "displayName": "Store: total chunks downloaded",
// "value": 3
// },
// {
// "displayName": "Store: chunks download time",
// "unit": "s",
// "value": 0.000390958
// },
// {
// "displayName": "Store: head chunk bytes",
// "unit": "decbytes",
// "value": 4
// },
// {
// "displayName": "Store: head chunk lines",
// "value": 5
// },
// {
// "displayName": "Store: decompressed bytes",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Store: decompressed lines",
// "value": 55
// },
// {
// "displayName": "Store: compressed bytes",
// "unit": "decbytes",
// "value": 31432
// },
// {
// "displayName": "Store: total duplicates",
// "value": 6
// },
// {
// "displayName": "Ingester: total reached",
// "value": 7
// },
// {
// "displayName": "Ingester: total chunks matched",
// "value": 8
// },
// {
// "displayName": "Ingester: total batches",
// "value": 9
// },
// {
// "displayName": "Ingester: total lines sent",
// "value": 10
// },
// {
// "displayName": "Ingester: head chunk bytes",
// "unit": "decbytes",
// "value": 11
// },
// {
// "displayName": "Ingester: head chunk lines",
// "value": 12
// },
// {
// "displayName": "Ingester: decompressed bytes",
// "unit": "decbytes",
// "value": 13
// },
// {
// "displayName": "Ingester: decompressed lines",
// "value": 14
// },
// {
// "displayName": "Ingester: compressed bytes",
// "unit": "decbytes",
// "value": 15
// },
// {
// "displayName": "Ingester: total duplicates",
// "value": 16
// }
// ],
// "executedQueryString": "Expr: query1"
// }
// Name:
// Dimensions: 5 Fields by 3 Rows
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
// | Name: labels | Name: timestamp | Name: body | Name: id | Name: labelTypes |
// | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage |
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
// | {"code":"\",two","field2":"two","location":"moon🌙"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000_194597ad | {"code":"I","field2":"P","location":"I"} |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000_90781cdf | {"code":"I","field1":"P","location":"I"} |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000_90781cdf | {"code":"I","field1":"P","location":"I"} |
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"type": "log-lines",
"typeVersion": [
0,
0
],
"stats": [
{
"displayName": "Summary: bytes processed per second",
"unit": "Bps",
"value": 3507022
},
{
"displayName": "Summary: lines processed per second",
"value": 24818
},
{
"displayName": "Summary: total bytes processed",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Summary: total lines processed",
"value": 55
},
{
"displayName": "Summary: exec time",
"unit": "s",
"value": 0.002216125
},
{
"displayName": "Store: total chunks ref",
"value": 2
},
{
"displayName": "Store: total chunks downloaded",
"value": 3
},
{
"displayName": "Store: chunks download time",
"unit": "s",
"value": 0.000390958
},
{
"displayName": "Store: head chunk bytes",
"unit": "decbytes",
"value": 4
},
{
"displayName": "Store: head chunk lines",
"value": 5
},
{
"displayName": "Store: decompressed bytes",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Store: decompressed lines",
"value": 55
},
{
"displayName": "Store: compressed bytes",
"unit": "decbytes",
"value": 31432
},
{
"displayName": "Store: total duplicates",
"value": 6
},
{
"displayName": "Ingester: total reached",
"value": 7
},
{
"displayName": "Ingester: total chunks matched",
"value": 8
},
{
"displayName": "Ingester: total batches",
"value": 9
},
{
"displayName": "Ingester: total lines sent",
"value": 10
},
{
"displayName": "Ingester: head chunk bytes",
"unit": "decbytes",
"value": 11
},
{
"displayName": "Ingester: head chunk lines",
"value": 12
},
{
"displayName": "Ingester: decompressed bytes",
"unit": "decbytes",
"value": 13
},
{
"displayName": "Ingester: decompressed lines",
"value": 14
},
{
"displayName": "Ingester: compressed bytes",
"unit": "decbytes",
"value": 15
},
{
"displayName": "Ingester: total duplicates",
"value": 16
}
],
"executedQueryString": "Expr: query1"
},
"fields": [
{
"name": "labels",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
}
},
{
"name": "timestamp",
"type": "time",
"typeInfo": {
"frame": "time.Time"
}
},
{
"name": "body",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "id",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "labelTypes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
},
"config": {
"custom": {
"hidden": true
}
}
}
]
},
"data": {
"values": [
[
{
"code": "\",two",
"field2": "two",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
}
],
[
1704895296244,
1704895267503,
1704895245190
],
[
"{\"field2\":\"two\"}",
"{\"field1\":\"one\"}",
"{\"field1\":\"one\"}"
],
[
"1704895296244577000_194597ad",
"1704895267503906000_90781cdf",
"1704895245190222000_90781cdf"
],
[
{
"code": "I",
"field2": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
}
]
],
"nanos": [
null,
[
577000,
906000,
222000
],
null,
null,
null
]
}
}
]
}

78
pkg/tsdb/loki/testdata_dataplane/streams_structured_metadata_2.json
Unescape View File

@ -0,0 +1,78 @@
{
"status": "success",
"data": {
"encodingFlags": [
"categorize-labels"
],
"resultType": "streams",
"result": [
{
"stream": {
"code": "\",two",
"location": "moon🌙"
},
"values": [
[
"1704895296244577000",
"{\"field2\":\"two\"}",
{
"parsed": {
"field2": "two"
}
}
],
[
"1704895267503906000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
],
[
"1704895245190222000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3507022,
"linesProcessedPerSecond": 24818,
"totalBytesProcessed": 7772,
"totalLinesProcessed": 55,
"execTime": 0.002216125
},
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 3,
"chunksDownloadTime": 0.000390958,
"headChunkBytes": 4,
"headChunkLines": 5,
"decompressedBytes": 7772,
"decompressedLines": 55,
"compressedBytes": 31432,
"totalDuplicates": 6
},
"ingester": {
"totalReached": 7,
"totalChunksMatched": 8,
"totalBatches": 9,
"totalLinesSent": 10,
"headChunkBytes": 11,
"headChunkLines": 12,
"decompressedBytes": 13,
"decompressedLines": 14,
"compressedBytes": 15,
"totalDuplicates": 16
}
}
}
}

363
pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.golden.jsonc
Unescape View File

@ -0,0 +1,363 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "type": "log-lines",
// "typeVersion": [
// 0,
// 0
// ],
// "stats": [
// {
// "displayName": "Summary: bytes processed per second",
// "unit": "Bps",
// "value": 3507022
// },
// {
// "displayName": "Summary: lines processed per second",
// "value": 24818
// },
// {
// "displayName": "Summary: total bytes processed",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Summary: total lines processed",
// "value": 55
// },
// {
// "displayName": "Summary: exec time",
// "unit": "s",
// "value": 0.002216125
// },
// {
// "displayName": "Store: total chunks ref",
// "value": 2
// },
// {
// "displayName": "Store: total chunks downloaded",
// "value": 3
// },
// {
// "displayName": "Store: chunks download time",
// "unit": "s",
// "value": 0.000390958
// },
// {
// "displayName": "Store: head chunk bytes",
// "unit": "decbytes",
// "value": 4
// },
// {
// "displayName": "Store: head chunk lines",
// "value": 5
// },
// {
// "displayName": "Store: decompressed bytes",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Store: decompressed lines",
// "value": 55
// },
// {
// "displayName": "Store: compressed bytes",
// "unit": "decbytes",
// "value": 31432
// },
// {
// "displayName": "Store: total duplicates",
// "value": 6
// },
// {
// "displayName": "Ingester: total reached",
// "value": 7
// },
// {
// "displayName": "Ingester: total chunks matched",
// "value": 8
// },
// {
// "displayName": "Ingester: total batches",
// "value": 9
// },
// {
// "displayName": "Ingester: total lines sent",
// "value": 10
// },
// {
// "displayName": "Ingester: head chunk bytes",
// "unit": "decbytes",
// "value": 11
// },
// {
// "displayName": "Ingester: head chunk lines",
// "value": 12
// },
// {
// "displayName": "Ingester: decompressed bytes",
// "unit": "decbytes",
// "value": 13
// },
// {
// "displayName": "Ingester: decompressed lines",
// "value": 14
// },
// {
// "displayName": "Ingester: compressed bytes",
// "unit": "decbytes",
// "value": 15
// },
// {
// "displayName": "Ingester: total duplicates",
// "value": 16
// }
// ],
// "executedQueryString": "Expr: query1"
// }
// Name:
// Dimensions: 5 Fields by 3 Rows
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
// | Name: labels | Name: timestamp | Name: body | Name: id | Name: labelTypes |
// | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage |
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
// | {"code":"\",two","field2":"two","location":"moon🌙"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000_194597ad | {"code":"I","field2":"P","location":"I"} |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000_90781cdf | {"code":"I","field1":"P","location":"I"} |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000_90781cdf | {"code":"I","field1":"P","location":"I"} |
// +------------------------------------------------------+--------------------------------------+------------------+------------------------------+------------------------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"type": "log-lines",
"typeVersion": [
0,
0
],
"stats": [
{
"displayName": "Summary: bytes processed per second",
"unit": "Bps",
"value": 3507022
},
{
"displayName": "Summary: lines processed per second",
"value": 24818
},
{
"displayName": "Summary: total bytes processed",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Summary: total lines processed",
"value": 55
},
{
"displayName": "Summary: exec time",
"unit": "s",
"value": 0.002216125
},
{
"displayName": "Store: total chunks ref",
"value": 2
},
{
"displayName": "Store: total chunks downloaded",
"value": 3
},
{
"displayName": "Store: chunks download time",
"unit": "s",
"value": 0.000390958
},
{
"displayName": "Store: head chunk bytes",
"unit": "decbytes",
"value": 4
},
{
"displayName": "Store: head chunk lines",
"value": 5
},
{
"displayName": "Store: decompressed bytes",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Store: decompressed lines",
"value": 55
},
{
"displayName": "Store: compressed bytes",
"unit": "decbytes",
"value": 31432
},
{
"displayName": "Store: total duplicates",
"value": 6
},
{
"displayName": "Ingester: total reached",
"value": 7
},
{
"displayName": "Ingester: total chunks matched",
"value": 8
},
{
"displayName": "Ingester: total batches",
"value": 9
},
{
"displayName": "Ingester: total lines sent",
"value": 10
},
{
"displayName": "Ingester: head chunk bytes",
"unit": "decbytes",
"value": 11
},
{
"displayName": "Ingester: head chunk lines",
"value": 12
},
{
"displayName": "Ingester: decompressed bytes",
"unit": "decbytes",
"value": 13
},
{
"displayName": "Ingester: decompressed lines",
"value": 14
},
{
"displayName": "Ingester: compressed bytes",
"unit": "decbytes",
"value": 15
},
{
"displayName": "Ingester: total duplicates",
"value": 16
}
],
"executedQueryString": "Expr: query1"
},
"fields": [
{
"name": "labels",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
}
},
{
"name": "timestamp",
"type": "time",
"typeInfo": {
"frame": "time.Time"
}
},
{
"name": "body",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "id",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "labelTypes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
},
"config": {
"custom": {
"hidden": true
}
}
}
]
},
"data": {
"values": [
[
{
"code": "\",two",
"field2": "two",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
}
],
[
1704895296244,
1704895267503,
1704895245190
],
[
"{\"field2\":\"two\"}",
"{\"field1\":\"one\"}",
"{\"field1\":\"one\"}"
],
[
"1704895296244577000_194597ad",
"1704895267503906000_90781cdf",
"1704895245190222000_90781cdf"
],
[
{
"code": "I",
"field2": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
}
]
],
"nanos": [
null,
[
577000,
906000,
222000
],
null,
null,
null
]
}
}
]
}

78
pkg/tsdb/loki/testdata_logs_dataplane/streams_structured_metadata_2.json
Unescape View File

@ -0,0 +1,78 @@
{
"status": "success",
"data": {
"encodingFlags": [
"categorize-labels"
],
"resultType": "streams",
"result": [
{
"stream": {
"code": "\",two",
"location": "moon🌙"
},
"values": [
[
"1704895296244577000",
"{\"field2\":\"two\"}",
{
"parsed": {
"field2": "two"
}
}
],
[
"1704895267503906000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
],
[
"1704895245190222000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3507022,
"linesProcessedPerSecond": 24818,
"totalBytesProcessed": 7772,
"totalLinesProcessed": 55,
"execTime": 0.002216125
},
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 3,
"chunksDownloadTime": 0.000390958,
"headChunkBytes": 4,
"headChunkLines": 5,
"decompressedBytes": 7772,
"decompressedLines": 55,
"compressedBytes": 31432,
"totalDuplicates": 6
},
"ingester": {
"totalReached": 7,
"totalChunksMatched": 8,
"totalBatches": 9,
"totalLinesSent": 10,
"headChunkBytes": 11,
"headChunkLines": 12,
"decompressedBytes": 13,
"decompressedLines": 14,
"compressedBytes": 15,
"totalDuplicates": 16
}
}
}
}

380
pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.golden.jsonc
Unescape View File

@ -0,0 +1,380 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "custom": {
// "frameType": "LabeledTimeValues"
// },
// "stats": [
// {
// "displayName": "Summary: bytes processed per second",
// "unit": "Bps",
// "value": 3507022
// },
// {
// "displayName": "Summary: lines processed per second",
// "value": 24818
// },
// {
// "displayName": "Summary: total bytes processed",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Summary: total lines processed",
// "value": 55
// },
// {
// "displayName": "Summary: exec time",
// "unit": "s",
// "value": 0.002216125
// },
// {
// "displayName": "Store: total chunks ref",
// "value": 2
// },
// {
// "displayName": "Store: total chunks downloaded",
// "value": 3
// },
// {
// "displayName": "Store: chunks download time",
// "unit": "s",
// "value": 0.000390958
// },
// {
// "displayName": "Store: head chunk bytes",
// "unit": "decbytes",
// "value": 4
// },
// {
// "displayName": "Store: head chunk lines",
// "value": 5
// },
// {
// "displayName": "Store: decompressed bytes",
// "unit": "decbytes",
// "value": 7772
// },
// {
// "displayName": "Store: decompressed lines",
// "value": 55
// },
// {
// "displayName": "Store: compressed bytes",
// "unit": "decbytes",
// "value": 31432
// },
// {
// "displayName": "Store: total duplicates",
// "value": 6
// },
// {
// "displayName": "Ingester: total reached",
// "value": 7
// },
// {
// "displayName": "Ingester: total chunks matched",
// "value": 8
// },
// {
// "displayName": "Ingester: total batches",
// "value": 9
// },
// {
// "displayName": "Ingester: total lines sent",
// "value": 10
// },
// {
// "displayName": "Ingester: head chunk bytes",
// "unit": "decbytes",
// "value": 11
// },
// {
// "displayName": "Ingester: head chunk lines",
// "value": 12
// },
// {
// "displayName": "Ingester: decompressed bytes",
// "unit": "decbytes",
// "value": 13
// },
// {
// "displayName": "Ingester: decompressed lines",
// "value": 14
// },
// {
// "displayName": "Ingester: compressed bytes",
// "unit": "decbytes",
// "value": 15
// },
// {
// "displayName": "Ingester: total duplicates",
// "value": 16
// }
// ],
// "executedQueryString": "Expr: query1"
// }
// Name:
// Dimensions: 6 Fields by 3 Rows
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
// | Name: labels | Name: Time | Name: Line | Name: tsNs | Name: labelTypes | Name: id |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []json.RawMessage | Type: []time.Time | Type: []string | Type: []string | Type: []json.RawMessage | Type: []string |
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
// | {"code":"\",two","field2":"two","location":"moon🌙"} | 2024-01-10 14:01:36.244577 +0000 UTC | {"field2":"two"} | 1704895296244577000 | {"code":"I","field2":"P","location":"I"} | 1704895296244577000_194597ad |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:01:07.503906 +0000 UTC | {"field1":"one"} | 1704895267503906000 | {"code":"I","field1":"P","location":"I"} | 1704895267503906000_90781cdf |
// | {"code":"\",two","field1":"one","location":"moon🌙"} | 2024-01-10 14:00:45.190222 +0000 UTC | {"field1":"one"} | 1704895245190222000 | {"code":"I","field1":"P","location":"I"} | 1704895245190222000_90781cdf |
// +------------------------------------------------------+--------------------------------------+------------------+---------------------+------------------------------------------+------------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"custom": {
"frameType": "LabeledTimeValues"
},
"stats": [
{
"displayName": "Summary: bytes processed per second",
"unit": "Bps",
"value": 3507022
},
{
"displayName": "Summary: lines processed per second",
"value": 24818
},
{
"displayName": "Summary: total bytes processed",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Summary: total lines processed",
"value": 55
},
{
"displayName": "Summary: exec time",
"unit": "s",
"value": 0.002216125
},
{
"displayName": "Store: total chunks ref",
"value": 2
},
{
"displayName": "Store: total chunks downloaded",
"value": 3
},
{
"displayName": "Store: chunks download time",
"unit": "s",
"value": 0.000390958
},
{
"displayName": "Store: head chunk bytes",
"unit": "decbytes",
"value": 4
},
{
"displayName": "Store: head chunk lines",
"value": 5
},
{
"displayName": "Store: decompressed bytes",
"unit": "decbytes",
"value": 7772
},
{
"displayName": "Store: decompressed lines",
"value": 55
},
{
"displayName": "Store: compressed bytes",
"unit": "decbytes",
"value": 31432
},
{
"displayName": "Store: total duplicates",
"value": 6
},
{
"displayName": "Ingester: total reached",
"value": 7
},
{
"displayName": "Ingester: total chunks matched",
"value": 8
},
{
"displayName": "Ingester: total batches",
"value": 9
},
{
"displayName": "Ingester: total lines sent",
"value": 10
},
{
"displayName": "Ingester: head chunk bytes",
"unit": "decbytes",
"value": 11
},
{
"displayName": "Ingester: head chunk lines",
"value": 12
},
{
"displayName": "Ingester: decompressed bytes",
"unit": "decbytes",
"value": 13
},
{
"displayName": "Ingester: decompressed lines",
"value": 14
},
{
"displayName": "Ingester: compressed bytes",
"unit": "decbytes",
"value": 15
},
{
"displayName": "Ingester: total duplicates",
"value": 16
}
],
"executedQueryString": "Expr: query1"
},
"fields": [
{
"name": "labels",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
}
},
{
"name": "Time",
"type": "time",
"typeInfo": {
"frame": "time.Time"
}
},
{
"name": "Line",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "tsNs",
"type": "string",
"typeInfo": {
"frame": "string"
}
},
{
"name": "labelTypes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage"
},
"config": {
"custom": {
"hidden": true
}
}
},
{
"name": "id",
"type": "string",
"typeInfo": {
"frame": "string"
}
}
]
},
"data": {
"values": [
[
{
"code": "\",two",
"field2": "two",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
},
{
"code": "\",two",
"field1": "one",
"location": "moon🌙"
}
],
[
1704895296244,
1704895267503,
1704895245190
],
[
"{\"field2\":\"two\"}",
"{\"field1\":\"one\"}",
"{\"field1\":\"one\"}"
],
[
"1704895296244577000",
"1704895267503906000",
"1704895245190222000"
],
[
{
"code": "I",
"field2": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
},
{
"code": "I",
"field1": "P",
"location": "I"
}
],
[
"1704895296244577000_194597ad",
"1704895267503906000_90781cdf",
"1704895245190222000_90781cdf"
]
],
"nanos": [
null,
[
577000,
906000,
222000
],
null,
null,
null,
null
]
}
}
]
}

78
pkg/tsdb/loki/testdata_metric_dataplane/streams_structured_metadata_2.json
Unescape View File

@ -0,0 +1,78 @@
{
"status": "success",
"data": {
"encodingFlags": [
"categorize-labels"
],
"resultType": "streams",
"result": [
{
"stream": {
"code": "\",two",
"location": "moon🌙"
},
"values": [
[
"1704895296244577000",
"{\"field2\":\"two\"}",
{
"parsed": {
"field2": "two"
}
}
],
[
"1704895267503906000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
],
[
"1704895245190222000",
"{\"field1\":\"one\"}",
{
"parsed": {
"field1": "one"
}
}
]
]
}
],
"stats": {
"summary": {
"bytesProcessedPerSecond": 3507022,
"linesProcessedPerSecond": 24818,
"totalBytesProcessed": 7772,
"totalLinesProcessed": 55,
"execTime": 0.002216125
},
"store": {
"totalChunksRef": 2,
"totalChunksDownloaded": 3,
"chunksDownloadTime": 0.000390958,
"headChunkBytes": 4,
"headChunkLines": 5,
"decompressedBytes": 7772,
"decompressedLines": 55,
"compressedBytes": 31432,
"totalDuplicates": 6
},
"ingester": {
"totalReached": 7,
"totalChunksMatched": 8,
"totalBatches": 9,
"totalLinesSent": 10,
"headChunkBytes": 11,
"headChunkLines": 12,
"decompressedBytes": 13,
"decompressedLines": 14,
"compressedBytes": 15,
"totalDuplicates": 16
}
}
}
}

16
pkg/util/converter/prom.go
Unescape View File

@ -965,7 +965,7 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse {
tsField := data.NewFieldFromFieldType(data.FieldTypeString, 0)
tsField.Name = "TS"
labels := data.Labels{}
indexedLabels := data.Labels{}
for more, err := iter.ReadArray(); more; more, err = iter.ReadArray() {
if err != nil {
@ -981,8 +981,8 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse {
case "stream":
// we need to clear `labels`, because `iter.ReadVal`
// only appends to it
labels = data.Labels{}
if err = iter.ReadVal(&labels); err != nil {
indexedLabels = data.Labels{}
if err = iter.ReadVal(&indexedLabels); err != nil {
return rspErr(err)
}
@ -1030,23 +1030,25 @@ func readCategorizedStream(iter *jsonitere.Iterator) backend.DataResponse {
}
typeMap := data.Labels{}
clonedLabels := data.Labels{}
for k := range labels {
for k := range indexedLabels {
typeMap[k] = "I"
clonedLabels[k] = indexedLabels[k]
}
// merge all labels (indexed, parsed, structuredMetadata) into one dataframe field
for k, v := range structuredMetadataMap {
labels[k] = fmt.Sprintf("%s", v)
clonedLabels[k] = fmt.Sprintf("%s", v)
typeMap[k] = "S"
}
for k, v := range parsedLabelsMap {
labels[k] = fmt.Sprintf("%s", v)
clonedLabels[k] = fmt.Sprintf("%s", v)
typeMap[k] = "P"
}
labelJson, err := labelsToRawJson(labels)
labelJson, err := labelsToRawJson(clonedLabels)
if err != nil {
return rspErr(err)
}

Write Preview
Loading…
Cancel
Save