apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Elasticsearch: Implement processing of logs query results in backend (#63647)

Browse Source 
* Elasticsearch: Add processing of logs query to backend

* Add and fix tests

* Add snapshot tests

* Fix test in ES client

* Small updates, remove redundant logic

* Refactor setPreferredVisType to improve readability
pull/63803/head^2
Ivana Huckova 2 years ago committed by GitHub
parent 8ac92aab19
commit d258c8ef8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1060 additions and 53 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 40
      pkg/tsdb/elasticsearch/client/client.go
  2. 8
      pkg/tsdb/elasticsearch/client/client_test.go
  3. 18
      pkg/tsdb/elasticsearch/elasticsearch.go
  4. 13
      pkg/tsdb/elasticsearch/querydata_test.go
  5. 113
      pkg/tsdb/elasticsearch/response_parser.go
  6. 129
      pkg/tsdb/elasticsearch/response_parser_test.go
  7. 1
      pkg/tsdb/elasticsearch/snapshot_test.go
  8. 534
      pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc
  9. 23
      pkg/tsdb/elasticsearch/testdata_response/logs.queries.json
  10. 208
      pkg/tsdb/elasticsearch/testdata_response/logs.response.json
  11. 4
      pkg/tsdb/elasticsearch/time_series_query.go
  12. 22
      pkg/tsdb/elasticsearch/time_series_query_test.go

40
pkg/tsdb/elasticsearch/client/client.go
Unescape View File

@ -25,7 +25,7 @@ type DatasourceInfo struct {
URL string
Database string
ESVersion *semver.Version
TimeField string
ConfiguredFields ConfiguredFields
Interval string
TimeInterval string
MaxConcurrentShardRequests int64
@ -33,11 +33,17 @@ type DatasourceInfo struct {
XPack bool
}
type ConfiguredFields struct {
TimeField string
LogMessageField string
LogLevelField string
}
const loggerName = "tsdb.elasticsearch.client"
// Client represents a client which can interact with elasticsearch api
type Client interface {
GetTimeField() string
GetConfiguredFields() ConfiguredFields
GetMinInterval(queryInterval string) (time.Duration, error)
ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error)
MultiSearch() *MultiSearchRequestBuilder
@ -56,29 +62,29 @@ var NewClient = func(ctx context.Context, ds *DatasourceInfo, timeRange backend.
}
logger := log.New(loggerName).FromContext(ctx)
logger.Debug("Creating new client", "version", ds.ESVersion, "timeField", ds.TimeField, "indices", strings.Join(indices, ", "))
logger.Debug("Creating new client", "version", ds.ESVersion, "configuredFields", fmt.Sprintf("%#v", ds.ConfiguredFields), "indices", strings.Join(indices, ", "))
return &baseClientImpl{
logger: logger,
ctx: ctx,
ds: ds,
timeField: ds.TimeField,
indices: indices,
timeRange: timeRange,
logger: logger,
ctx: ctx,
ds: ds,
configuredFields: ds.ConfiguredFields,
indices: indices,
timeRange: timeRange,
}, nil
}
type baseClientImpl struct {
ctx context.Context
ds *DatasourceInfo
timeField string
indices []string
timeRange backend.TimeRange
logger log.Logger
ctx context.Context
ds *DatasourceInfo
configuredFields ConfiguredFields
indices []string
timeRange backend.TimeRange
logger log.Logger
}
func (c *baseClientImpl) GetTimeField() string {
return c.timeField
func (c *baseClientImpl) GetConfiguredFields() ConfiguredFields {
return c.configuredFields
}
func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) {

8
pkg/tsdb/elasticsearch/client/client_test.go
Unescape View File

@ -46,12 +46,18 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
version, err := semver.NewVersion("8.0.0")
require.NoError(t, err)
configuredFields := ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
ds := DatasourceInfo{
URL: ts.URL,
HTTPClient: ts.Client(),
Database: "[metrics-]YYYY.MM.DD",
ESVersion: version,
TimeField: "@timestamp",
ConfiguredFields: configuredFields,
Interval: "Daily",
MaxConcurrentShardRequests: 6,
IncludeFrozen: true,

18
pkg/tsdb/elasticsearch/elasticsearch.go
Unescape View File

@ -98,6 +98,16 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
return nil, errors.New("elasticsearch time field name is required")
}
logLevelField, ok := jsonData["logLevelField"].(string)
if !ok {
logLevelField = ""
}
logMessageField, ok := jsonData["logMessageField"].(string)
if !ok {
logMessageField = ""
}
interval, ok := jsonData["interval"].(string)
if !ok {
interval = ""
@ -132,6 +142,12 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
xpack = false
}
configuredFields := es.ConfiguredFields{
TimeField: timeField,
LogLevelField: logLevelField,
LogMessageField: logMessageField,
}
model := es.DatasourceInfo{
ID: settings.ID,
URL: settings.URL,
@ -139,7 +155,7 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
Database: settings.Database,
MaxConcurrentShardRequests: int64(maxConcurrentShardRequests),
ESVersion: version,
TimeField: timeField,
ConfiguredFields: configuredFields,
Interval: interval,
TimeInterval: timeInterval,
IncludeFrozen: includeFrozen,

13
pkg/tsdb/elasticsearch/querydata_test.go
Unescape View File

@ -37,15 +37,22 @@ func (rt *queryDataTestRoundTripper) RoundTrip(req *http.Request) (*http.Respons
}
// we setup a fake datasource-info
func newFlowTestDsInfo(body []byte, statusCode int, reuestCallback func(req *http.Request) error) *es.DatasourceInfo {
func newFlowTestDsInfo(body []byte, statusCode int, requestCallback func(req *http.Request) error) *es.DatasourceInfo {
client := http.Client{
Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: reuestCallback},
Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: requestCallback},
}
configuredFields := es.ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &es.DatasourceInfo{
ESVersion: semver.MustParse("8.5.0"),
Interval: "Daily",
Database: "[testdb-]YYYY.MM.DD",
TimeField: "testtime",
ConfiguredFields: configuredFields,
TimeInterval: "1s",
URL: "http://localhost:9200",
HTTPClient: &client,

113
pkg/tsdb/elasticsearch/response_parser.go
Unescape View File

@ -36,7 +36,7 @@ const (
logsType = "logs"
)
func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField string) (*backend.QueryDataResponse, error) {
func parseResponse(responses []*es.SearchResponse, targets []*Query, configuredFields es.ConfiguredFields) (*backend.QueryDataResponse, error) {
result := backend.QueryDataResponse{
Responses: backend.Responses{},
}
@ -58,12 +58,19 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
queryRes := backend.DataResponse{}
if isDocumentQuery(target) {
err := processDocumentResponse(res, target, timeField, &queryRes)
err := processDocumentResponse(res, target, configuredFields, &queryRes)
if err != nil {
return &backend.QueryDataResponse{}, err
}
result.Responses[target.RefID] = queryRes
} else if isLogsQuery(target) {
err := processLogsResponse(res, target, configuredFields, &queryRes)
if err != nil {
return &backend.QueryDataResponse{}, err
}
result.Responses[target.RefID] = queryRes
} else {
// Process as metric query result
props := make(map[string]string)
err := processBuckets(res.Aggregations, target, &queryRes, props, 0)
if err != nil {
@ -78,9 +85,56 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
return &result, nil
}
func processDocumentResponse(res *es.SearchResponse, target *Query, timeField string, queryRes *backend.DataResponse) error {
func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{}
if hit["_source"] != nil {
flattened = flatten(hit["_source"].(map[string]interface{}))
}
doc := map[string]interface{}{
"_id": hit["_id"],
"_type": hit["_type"],
"_index": hit["_index"],
"sort": hit["sort"],
"highlight": hit["highlight"],
"_source": flattened,
}
for k, v := range flattened {
if configuredFields.LogLevelField != "" && k == configuredFields.LogLevelField {
doc["level"] = v
} else {
doc[k] = v
}
}
for key := range doc {
propNames[key] = true
}
// TODO: Implement highlighting
docs[hitIdx] = doc
}
sortedPropNames := sortPropNames(propNames, configuredFields, true)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
setPreferredVisType(frame, "logs")
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocumentResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{}
@ -108,18 +162,28 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
docs[hitIdx] = doc
}
sortedPropNames := sortPropNames(propNames, configuredFields, false)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocsToDataFrameFields(docs []map[string]interface{}, propNames []string, configuredFields es.ConfiguredFields) []*data.Field {
size := len(docs)
isFilterable := true
allFields := make([]*data.Field, len(propNames))
sortedPropNames := sortPropNames(propNames, timeField)
for propNameIdx, propName := range sortedPropNames {
for propNameIdx, propName := range propNames {
// Special handling for time field
if propName == timeField {
if propName == configuredFields.TimeField {
timeVector := make([]*time.Time, size)
for i, doc := range docs {
timeString, ok := doc[timeField].(string)
timeString, ok := doc[configuredFields.TimeField].(string)
if !ok {
continue
}
@ -131,7 +195,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
timeVector[i] = &timeValue
}
}
field := data.NewField(timeField, nil, timeVector)
field := data.NewField(configuredFields.TimeField, nil, timeVector)
field.Config = &data.FieldConfig{Filterable: &isFilterable}
allFields[propNameIdx] = field
continue
@ -166,12 +230,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
}
}
frames := data.Frames{}
frame := data.NewFrame("", allFields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
return allFields
}
func processBuckets(aggs map[string]interface{}, target *Query,
@ -894,14 +953,18 @@ func flatten(target map[string]interface{}) map[string]interface{} {
return output
}
// sortPropNames orders propNames so that timeField is first (if it exists) and rest of propNames are ordered alphabetically
func sortPropNames(propNames map[string]bool, timeField string) []string {
// sortPropNames orders propNames so that timeField is first (if it exists), log message field is second
// if shouldSortLogMessageField is true, and rest of propNames are ordered alphabetically
func sortPropNames(propNames map[string]bool, configuredFields es.ConfiguredFields, shouldSortLogMessageField bool) []string {
hasTimeField := false
hasLogMessageField := false
var sortedPropNames []string
for k := range propNames {
if k == timeField {
if configuredFields.TimeField != "" && k == configuredFields.TimeField {
hasTimeField = true
} else if shouldSortLogMessageField && configuredFields.LogMessageField != "" && k == configuredFields.LogMessageField {
hasLogMessageField = true
} else {
sortedPropNames = append(sortedPropNames, k)
}
@ -909,8 +972,12 @@ func sortPropNames(propNames map[string]bool, timeField string) []string {
sort.Strings(sortedPropNames)
if hasLogMessageField {
sortedPropNames = append([]string{configuredFields.LogMessageField}, sortedPropNames...)
}
if hasTimeField {
sortedPropNames = append([]string{timeField}, sortedPropNames...)
sortedPropNames = append([]string{configuredFields.TimeField}, sortedPropNames...)
}
return sortedPropNames
@ -939,3 +1006,11 @@ func createFieldOfType[T int | float64 | bool | string](docs []map[string]interf
field.Config = &data.FieldConfig{Filterable: &isFilterable}
return field
}
func setPreferredVisType(frame *data.Frame, visType data.VisType) {
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.PreferredVisualization = visType
}

129
pkg/tsdb/elasticsearch/response_parser_test.go
Unescape View File

@ -1043,6 +1043,127 @@ func TestResponseParser(t *testing.T) {
require.Nil(t, frame.Fields[1].Config)
})
t.Run("Log query", func(t *testing.T) {
targets := map[string]string{
"A": `{
"metrics": [{ "type": "logs" }]
}`,
}
response := `{
"responses":[
{
"hits":{
"total":{
"value":109,
"relation":"eq"
},
"max_score":null,
"hits":[
{
"_index":"logs-2023.02.08",
"_id":"GB2UMYYBfCQ-FCMjayJa",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:55.830Z",
"line":"log text [479231733]",
"counter":"109",
"float":58.253758485091,
"label":"val1",
"lvl":"info",
"location":"17.089705232090438, 41.62861966340297",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": null
},
"sort":[
1675869055830,
4
]
},
{
"_index":"logs-2023.02.08",
"_id":"Fx2UMYYBfCQ-FCMjZyJ_",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:54.835Z",
"line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]",
"counter":"108",
"float":54.5977098233944,
"label":"val1",
"lvl":"info",
"location":"19.766305918490463, 40.42639175509792",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": "def"
},
"sort":[
1675869054835,
7
]
}
]
},
"status":200
}
]
}`
result, err := parseTestResponse(targets, response)
require.NoError(t, err)
require.Len(t, result.Responses, 1)
queryRes := result.Responses["A"]
require.NotNil(t, queryRes)
dataframes := queryRes.Frames
require.Len(t, dataframes, 1)
frame := dataframes[0]
require.Equal(t, 16, len(frame.Fields))
// Fields have the correct length
require.Equal(t, 2, frame.Fields[0].Len())
// First field is timeField
require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type())
// Second is log line
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
require.Equal(t, "line", frame.Fields[1].Name)
// Correctly renames lvl field to level
require.Equal(t, "level", frame.Fields[10].Name)
// Correctly uses string types
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
// Correctly detects float64 types
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[7].Type())
// Correctly detects json types
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[8].Type())
// Correctly flattens fields
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
// Correctly detects type even if first value is null
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
})
t.Run("Raw data query", func(t *testing.T) {
targets := map[string]string{
"A": `{
@ -1252,7 +1373,11 @@ func TestResponseParser(t *testing.T) {
func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) {
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
timeField := "@timestamp"
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
timeRange := backend.TimeRange{
From: from,
To: to,
@ -1280,7 +1405,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac
return nil, err
}
return parseResponse(response.Responses, queries, timeField)
return parseResponse(response.Responses, queries, configuredFields)
}
func TestLabelOrderInFieldName(t *testing.T) {

1
pkg/tsdb/elasticsearch/snapshot_test.go
Unescape View File

@ -132,6 +132,7 @@ func TestResponseSnapshots(t *testing.T) {
{name: "complex metric test", path: "metric_complex"},
{name: "multi metric test", path: "metric_multi"},
{name: "raw data test", path: "raw_data"},
{name: "logs test", path: "logs"},
}
snapshotCount := findResponseSnapshotCounts(t, "testdata_response")

534
pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc
Unescape View File

@ -0,0 +1,534 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "preferredVisualisationType": "logs"
// }
// Name:
// Dimensions: 17 Fields by 5 Rows
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:01.475Z"} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
// | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:00.513Z"} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
// | 2023-02-09 14:39:59.556 +0000 UTC | log text [894867430] | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","lvl":"info","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:59.556Z"} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
// | 2023-02-09 14:39:58.608 +0000 UTC | log text [478598889] | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","lvl":"info","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:58.608Z"} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
// | 2023-02-09 14:39:57.665 +0000 UTC | log text [526995818] | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","lvl":"info","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:57.665Z"} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"preferredVisualisationType": "logs"
},
"fields": [
{
"name": "testtime",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "line",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_id",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_index",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_source",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_type",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "abc",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "counter",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "float",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "highlight",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "is_true",
"type": "boolean",
"typeInfo": {
"frame": "bool",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "label",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "level",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "nested_field.internal.nested",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "shapes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "sort",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
}
]
},
"data": {
"values": [
[
1675953601475,
1675953600513,
1675953599556,
1675953598608,
1675953597665
],
[
"log text [106619125]",
"log text with [781660944]",
"log text [894867430]",
"log text [478598889]",
"log text [526995818]"
],
[
"g2aeNoYB7vaC3bq-ezfK",
"gmaeNoYB7vaC3bq-eDcN",
"gWaeNoYB7vaC3bq-dDdL",
"gGaeNoYB7vaC3bq-cDeY",
"f2aeNoYB7vaC3bq-bDf7"
],
[
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09"
],
[
{
"abc": null,
"counter": 81,
"float": 10.911972180833306,
"is_true": true,
"label": "val3",
"line": "log text [106619125]",
"location": "-42.73465234425797, -14.097854057104112",
"lvl": "info",
"nested_field.internal.nested": "value1",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:01.475Z"
},
{
"abc": null,
"counter": 80,
"float": 62.94120607636795,
"is_true": false,
"label": "val3",
"line": "log text with [781660944]",
"location": "42.07571917624318, 15.95725088484611",
"lvl": "error",
"nested_field.internal.nested": "value2",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:00.513Z"
},
{
"abc": "def",
"counter": 79,
"float": 53.323706427230455,
"is_true": true,
"label": "val1",
"line": "log text [894867430]",
"location": "-38.27341566189766, -23.66739642570781",
"lvl": "info",
"nested_field.internal.nested": "value3",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:59.556Z"
},
{
"abc": "def",
"counter": 78,
"float": 82.72012623471589,
"is_true": false,
"label": "val1",
"line": "log text [478598889]",
"location": "12.373240290451287, 43.265493464362024",
"lvl": "info",
"nested_field.internal.nested": "value4",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:58.608Z"
},
{
"abc": "def",
"counter": 77,
"float": 35.05784443331803,
"is_true": false,
"label": "val3",
"line": "log text [526995818]",
"location": "-31.524344042228194, -32.11254790120572",
"lvl": "info",
"nested_field.internal.nested": "value5",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:57.665Z"
}
],
[
null,
null,
null,
null,
null
],
[
null,
null,
"def",
"def",
"def"
],
[
81,
80,
79,
78,
77
],
[
10.911972180833306,
62.94120607636795,
53.323706427230455,
82.72012623471589,
35.05784443331803
],
[
null,
null,
null,
null,
null
],
[
true,
false,
true,
false,
false
],
[
"val3",
"val3",
"val1",
"val1",
"val3"
],
[
"info",
"error",
"info",
"info",
"info"
],
[
"-42.73465234425797, -14.097854057104112",
"42.07571917624318, 15.95725088484611",
"-38.27341566189766, -23.66739642570781",
"12.373240290451287, 43.265493464362024",
"-31.524344042228194, -32.11254790120572"
],
[
"value1",
"value2",
"value3",
"value4",
"value5"
],
[
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
]
],
[
[
1675953601475,
4
],
[
1675953600513,
7
],
[
1675953599556,
10
],
[
1675953598608,
15
],
[
1675953597665,
20
]
]
]
}
}
]
}

23
pkg/tsdb/elasticsearch/testdata_response/logs.queries.json
Unescape View File

@ -0,0 +1,23 @@
[
{
"alias": "",
"datasource": {
"type": "elasticsearch",
"uid": "haha"
},
"datasourceId": 42,
"expression": "",
"hide": true,
"intervalMs": 200,
"maxDataPoints": 1248,
"metrics": [
{
"id": "1",
"type": "logs"
}
],
"query": "",
"refId": "a",
"window": ""
}
]

208
pkg/tsdb/elasticsearch/testdata_response/logs.response.json
Unescape View File

@ -0,0 +1,208 @@
{
"took": 6,
"responses": [
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 81,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "logs-2023.02.09",
"_id": "g2aeNoYB7vaC3bq-ezfK",
"_score": null,
"_source": {
"abc": null,
"is_true": true,
"testtime": "2023-02-09T14:40:01.475Z",
"line": "log text [106619125]",
"counter": 81,
"float": 10.911972180833306,
"label": "val3",
"lvl": "info",
"location": "-42.73465234425797, -14.097854057104112",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value1"
}
}
},
"sort": [
1675953601475,
4
]
},
{
"_index": "logs-2023.02.09",
"_id": "gmaeNoYB7vaC3bq-eDcN",
"_score": null,
"_source": {
"abc": null,
"is_true": false,
"testtime": "2023-02-09T14:40:00.513Z",
"line": "log text with [781660944]",
"counter": 80,
"float": 62.94120607636795,
"label": "val3",
"lvl": "error",
"location": "42.07571917624318, 15.95725088484611",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value2"
}
}
},
"sort": [
1675953600513,
7
]
},
{
"_index": "logs-2023.02.09",
"_id": "gWaeNoYB7vaC3bq-dDdL",
"_score": null,
"_source": {
"abc": "def",
"is_true": true,
"testtime": "2023-02-09T14:39:59.556Z",
"line": "log text [894867430]",
"counter": 79,
"float": 53.323706427230455,
"label": "val1",
"lvl": "info",
"location": "-38.27341566189766, -23.66739642570781",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value3"
}
}
},
"sort": [
1675953599556,
10
]
},
{
"_index": "logs-2023.02.09",
"_id": "gGaeNoYB7vaC3bq-cDeY",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:58.608Z",
"line": "log text [478598889]",
"counter": 78,
"float": 82.72012623471589,
"label": "val1",
"lvl": "info",
"location": "12.373240290451287, 43.265493464362024",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value4"
}
}
},
"sort": [
1675953598608,
15
]
},
{
"_index": "logs-2023.02.09",
"_id": "f2aeNoYB7vaC3bq-bDf7",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:57.665Z",
"line": "log text [526995818]",
"counter": 77,
"float": 35.05784443331803,
"label": "val3",
"lvl": "info",
"location": "-31.524344042228194, -32.11254790120572",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value5"
}
}
},
"sort": [
1675953597665,
20
]
}
]
},
"status": 200
}
]
}

4
pkg/tsdb/elasticsearch/time_series_query.go
Unescape View File

@ -54,7 +54,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) {
return &backend.QueryDataResponse{}, err
}
return parseResponse(res.Responses, queries, e.client.GetTimeField())
return parseResponse(res.Responses, queries, e.client.GetConfiguredFields())
}
func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64) error {
@ -63,7 +63,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
return err
}
defaultTimeField := e.client.GetTimeField()
defaultTimeField := e.client.GetConfiguredFields().TimeField
b := ms.Search(q.Interval)
b.Size(0)
filters := b.Query().Bool().Filter()

22
pkg/tsdb/elasticsearch/time_series_query_test.go
Unescape View File

@ -28,7 +28,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.NoError(t, err)
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -418,7 +418,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -439,7 +439,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -1305,7 +1305,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.Equal(t, sr.Size, defaultSize)
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField)
require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -1687,7 +1687,7 @@ func TestSettingsCasting(t *testing.T) {
}
type fakeClient struct {
timeField string
configuredFields es.ConfiguredFields
multiSearchResponse *es.MultiSearchResponse
multiSearchError error
builder *es.MultiSearchRequestBuilder
@ -1695,15 +1695,21 @@ type fakeClient struct {
}
func newFakeClient() *fakeClient {
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &fakeClient{
timeField: "@timestamp",
configuredFields: configuredFields,
multisearchRequests: make([]*es.MultiSearchRequest, 0),
multiSearchResponse: &es.MultiSearchResponse{},
}
}
func (c *fakeClient) GetTimeField() string {
return c.timeField
func (c *fakeClient) GetConfiguredFields() es.ConfiguredFields {
return c.configuredFields
}
func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) {

Write Preview
Loading…
Cancel
Save