apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Elasticsearch: Implement processing of logs query results in backend (#63647)

Browse Source 
* Elasticsearch: Add processing of logs query to backend

* Add and fix tests

* Add snapshot tests

* Fix test in ES client

* Small updates, remove redundant logic

* Refactor setPreferredVisType to improve readability
pull/63803/head^2
Ivana Huckova 2 years ago committed by GitHub
parent 8ac92aab19
commit d258c8ef8a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
12 changed files with 1060 additions and 53 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 20
      pkg/tsdb/elasticsearch/client/client.go
  2. 8
      pkg/tsdb/elasticsearch/client/client_test.go
  3. 18
      pkg/tsdb/elasticsearch/elasticsearch.go
  4. 13
      pkg/tsdb/elasticsearch/querydata_test.go
  5. 113
      pkg/tsdb/elasticsearch/response_parser.go
  6. 129
      pkg/tsdb/elasticsearch/response_parser_test.go
  7. 1
      pkg/tsdb/elasticsearch/snapshot_test.go
  8. 534
      pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc
  9. 23
      pkg/tsdb/elasticsearch/testdata_response/logs.queries.json
  10. 208
      pkg/tsdb/elasticsearch/testdata_response/logs.response.json
  11. 4
      pkg/tsdb/elasticsearch/time_series_query.go
  12. 22
      pkg/tsdb/elasticsearch/time_series_query_test.go

20
pkg/tsdb/elasticsearch/client/client.go
Unescape View File

@ -25,7 +25,7 @@ type DatasourceInfo struct {
URL string URL string
Database string Database string
ESVersion *semver.Version ESVersion *semver.Version
TimeField string ConfiguredFields ConfiguredFields
Interval string Interval string
TimeInterval string TimeInterval string
MaxConcurrentShardRequests int64 MaxConcurrentShardRequests int64
@ -33,11 +33,17 @@ type DatasourceInfo struct {
XPack bool XPack bool
} }
type ConfiguredFields struct {
TimeField string
LogMessageField string
LogLevelField string
}
const loggerName = "tsdb.elasticsearch.client" const loggerName = "tsdb.elasticsearch.client"
// Client represents a client which can interact with elasticsearch api // Client represents a client which can interact with elasticsearch api
type Client interface { type Client interface {
GetTimeField() string GetConfiguredFields() ConfiguredFields
GetMinInterval(queryInterval string) (time.Duration, error) GetMinInterval(queryInterval string) (time.Duration, error)
ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearchResponse, error)
MultiSearch() *MultiSearchRequestBuilder MultiSearch() *MultiSearchRequestBuilder
@ -56,13 +62,13 @@ var NewClient = func(ctx context.Context, ds *DatasourceInfo, timeRange backend.
} }
logger := log.New(loggerName).FromContext(ctx) logger := log.New(loggerName).FromContext(ctx)
logger.Debug("Creating new client", "version", ds.ESVersion, "timeField", ds.TimeField, "indices", strings.Join(indices, ", ")) logger.Debug("Creating new client", "version", ds.ESVersion, "configuredFields", fmt.Sprintf("%#v", ds.ConfiguredFields), "indices", strings.Join(indices, ", "))
return &baseClientImpl{ return &baseClientImpl{
logger: logger, logger: logger,
ctx: ctx, ctx: ctx,
ds: ds, ds: ds,
timeField: ds.TimeField, configuredFields: ds.ConfiguredFields,
indices: indices, indices: indices,
timeRange: timeRange, timeRange: timeRange,
}, nil }, nil
@ -71,14 +77,14 @@ var NewClient = func(ctx context.Context, ds *DatasourceInfo, timeRange backend.
type baseClientImpl struct { type baseClientImpl struct {
ctx context.Context ctx context.Context
ds *DatasourceInfo ds *DatasourceInfo
timeField string configuredFields ConfiguredFields
indices []string indices []string
timeRange backend.TimeRange timeRange backend.TimeRange
logger log.Logger logger log.Logger
} }
func (c *baseClientImpl) GetTimeField() string { func (c *baseClientImpl) GetConfiguredFields() ConfiguredFields {
return c.timeField return c.configuredFields
} }
func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) { func (c *baseClientImpl) GetMinInterval(queryInterval string) (time.Duration, error) {

8
pkg/tsdb/elasticsearch/client/client_test.go
Unescape View File

@ -46,12 +46,18 @@ func TestClient_ExecuteMultisearch(t *testing.T) {
version, err := semver.NewVersion("8.0.0") version, err := semver.NewVersion("8.0.0")
require.NoError(t, err) require.NoError(t, err)
configuredFields := ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
ds := DatasourceInfo{ ds := DatasourceInfo{
URL: ts.URL, URL: ts.URL,
HTTPClient: ts.Client(), HTTPClient: ts.Client(),
Database: "[metrics-]YYYY.MM.DD", Database: "[metrics-]YYYY.MM.DD",
ESVersion: version, ESVersion: version,
TimeField: "@timestamp", ConfiguredFields: configuredFields,
Interval: "Daily", Interval: "Daily",
MaxConcurrentShardRequests: 6, MaxConcurrentShardRequests: 6,
IncludeFrozen: true, IncludeFrozen: true,

18
pkg/tsdb/elasticsearch/elasticsearch.go
Unescape View File

@ -98,6 +98,16 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
return nil, errors.New("elasticsearch time field name is required") return nil, errors.New("elasticsearch time field name is required")
} }
logLevelField, ok := jsonData["logLevelField"].(string)
if !ok {
logLevelField = ""
}
logMessageField, ok := jsonData["logMessageField"].(string)
if !ok {
logMessageField = ""
}
interval, ok := jsonData["interval"].(string) interval, ok := jsonData["interval"].(string)
if !ok { if !ok {
interval = "" interval = ""
@ -132,6 +142,12 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
xpack = false xpack = false
} }
configuredFields := es.ConfiguredFields{
TimeField: timeField,
LogLevelField: logLevelField,
LogMessageField: logMessageField,
}
model := es.DatasourceInfo{ model := es.DatasourceInfo{
ID: settings.ID, ID: settings.ID,
URL: settings.URL, URL: settings.URL,
@ -139,7 +155,7 @@ func newInstanceSettings(httpClientProvider httpclient.Provider) datasource.Inst
Database: settings.Database, Database: settings.Database,
MaxConcurrentShardRequests: int64(maxConcurrentShardRequests), MaxConcurrentShardRequests: int64(maxConcurrentShardRequests),
ESVersion: version, ESVersion: version,
TimeField: timeField, ConfiguredFields: configuredFields,
Interval: interval, Interval: interval,
TimeInterval: timeInterval, TimeInterval: timeInterval,
IncludeFrozen: includeFrozen, IncludeFrozen: includeFrozen,

13
pkg/tsdb/elasticsearch/querydata_test.go
Unescape View File

@ -37,15 +37,22 @@ func (rt *queryDataTestRoundTripper) RoundTrip(req *http.Request) (*http.Respons
} }
// we setup a fake datasource-info // we setup a fake datasource-info
func newFlowTestDsInfo(body []byte, statusCode int, reuestCallback func(req *http.Request) error) *es.DatasourceInfo { func newFlowTestDsInfo(body []byte, statusCode int, requestCallback func(req *http.Request) error) *es.DatasourceInfo {
client := http.Client{ client := http.Client{
Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: reuestCallback}, Transport: &queryDataTestRoundTripper{body: body, statusCode: statusCode, requestCallback: requestCallback},
} }
configuredFields := es.ConfiguredFields{
TimeField: "testtime",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &es.DatasourceInfo{ return &es.DatasourceInfo{
ESVersion: semver.MustParse("8.5.0"), ESVersion: semver.MustParse("8.5.0"),
Interval: "Daily", Interval: "Daily",
Database: "[testdb-]YYYY.MM.DD", Database: "[testdb-]YYYY.MM.DD",
TimeField: "testtime", ConfiguredFields: configuredFields,
TimeInterval: "1s", TimeInterval: "1s",
URL: "http://localhost:9200", URL: "http://localhost:9200",
HTTPClient: &client, HTTPClient: &client,

113
pkg/tsdb/elasticsearch/response_parser.go
Unescape View File

@ -36,7 +36,7 @@ const (
logsType = "logs" logsType = "logs"
) )
func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField string) (*backend.QueryDataResponse, error) { func parseResponse(responses []*es.SearchResponse, targets []*Query, configuredFields es.ConfiguredFields) (*backend.QueryDataResponse, error) {
result := backend.QueryDataResponse{ result := backend.QueryDataResponse{
Responses: backend.Responses{}, Responses: backend.Responses{},
} }
@ -58,12 +58,19 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
queryRes := backend.DataResponse{} queryRes := backend.DataResponse{}
if isDocumentQuery(target) { if isDocumentQuery(target) {
err := processDocumentResponse(res, target, timeField, &queryRes) err := processDocumentResponse(res, target, configuredFields, &queryRes)
if err != nil {
return &backend.QueryDataResponse{}, err
}
result.Responses[target.RefID] = queryRes
} else if isLogsQuery(target) {
err := processLogsResponse(res, target, configuredFields, &queryRes)
if err != nil { if err != nil {
return &backend.QueryDataResponse{}, err return &backend.QueryDataResponse{}, err
} }
result.Responses[target.RefID] = queryRes result.Responses[target.RefID] = queryRes
} else { } else {
// Process as metric query result
props := make(map[string]string) props := make(map[string]string)
err := processBuckets(res.Aggregations, target, &queryRes, props, 0) err := processBuckets(res.Aggregations, target, &queryRes, props, 0)
if err != nil { if err != nil {
@ -78,9 +85,56 @@ func parseResponse(responses []*es.SearchResponse, targets []*Query, timeField s
return &result, nil return &result, nil
} }
func processDocumentResponse(res *es.SearchResponse, target *Query, timeField string, queryRes *backend.DataResponse) error { func processLogsResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits)) docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{}
if hit["_source"] != nil {
flattened = flatten(hit["_source"].(map[string]interface{}))
}
doc := map[string]interface{}{
"_id": hit["_id"],
"_type": hit["_type"],
"_index": hit["_index"],
"sort": hit["sort"],
"highlight": hit["highlight"],
"_source": flattened,
}
for k, v := range flattened {
if configuredFields.LogLevelField != "" && k == configuredFields.LogLevelField {
doc["level"] = v
} else {
doc[k] = v
}
}
for key := range doc {
propNames[key] = true
}
// TODO: Implement highlighting
docs[hitIdx] = doc
}
sortedPropNames := sortPropNames(propNames, configuredFields, true)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
setPreferredVisType(frame, "logs")
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocumentResponse(res *es.SearchResponse, target *Query, configuredFields es.ConfiguredFields, queryRes *backend.DataResponse) error {
propNames := make(map[string]bool) propNames := make(map[string]bool)
docs := make([]map[string]interface{}, len(res.Hits.Hits))
for hitIdx, hit := range res.Hits.Hits { for hitIdx, hit := range res.Hits.Hits {
var flattened map[string]interface{} var flattened map[string]interface{}
@ -108,18 +162,28 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
docs[hitIdx] = doc docs[hitIdx] = doc
} }
sortedPropNames := sortPropNames(propNames, configuredFields, false)
fields := processDocsToDataFrameFields(docs, sortedPropNames, configuredFields)
frames := data.Frames{}
frame := data.NewFrame("", fields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
}
func processDocsToDataFrameFields(docs []map[string]interface{}, propNames []string, configuredFields es.ConfiguredFields) []*data.Field {
size := len(docs) size := len(docs)
isFilterable := true isFilterable := true
allFields := make([]*data.Field, len(propNames)) allFields := make([]*data.Field, len(propNames))
sortedPropNames := sortPropNames(propNames, timeField) for propNameIdx, propName := range propNames {
for propNameIdx, propName := range sortedPropNames {
// Special handling for time field // Special handling for time field
if propName == timeField { if propName == configuredFields.TimeField {
timeVector := make([]*time.Time, size) timeVector := make([]*time.Time, size)
for i, doc := range docs { for i, doc := range docs {
timeString, ok := doc[timeField].(string) timeString, ok := doc[configuredFields.TimeField].(string)
if !ok { if !ok {
continue continue
} }
@ -131,7 +195,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
timeVector[i] = &timeValue timeVector[i] = &timeValue
} }
} }
field := data.NewField(timeField, nil, timeVector) field := data.NewField(configuredFields.TimeField, nil, timeVector)
field.Config = &data.FieldConfig{Filterable: &isFilterable} field.Config = &data.FieldConfig{Filterable: &isFilterable}
allFields[propNameIdx] = field allFields[propNameIdx] = field
continue continue
@ -166,12 +230,7 @@ func processDocumentResponse(res *es.SearchResponse, target *Query, timeField st
} }
} }
frames := data.Frames{} return allFields
frame := data.NewFrame("", allFields...)
frames = append(frames, frame)
queryRes.Frames = frames
return nil
} }
func processBuckets(aggs map[string]interface{}, target *Query, func processBuckets(aggs map[string]interface{}, target *Query,
@ -894,14 +953,18 @@ func flatten(target map[string]interface{}) map[string]interface{} {
return output return output
} }
// sortPropNames orders propNames so that timeField is first (if it exists) and rest of propNames are ordered alphabetically // sortPropNames orders propNames so that timeField is first (if it exists), log message field is second
func sortPropNames(propNames map[string]bool, timeField string) []string { // if shouldSortLogMessageField is true, and rest of propNames are ordered alphabetically
func sortPropNames(propNames map[string]bool, configuredFields es.ConfiguredFields, shouldSortLogMessageField bool) []string {
hasTimeField := false hasTimeField := false
hasLogMessageField := false
var sortedPropNames []string var sortedPropNames []string
for k := range propNames { for k := range propNames {
if k == timeField { if configuredFields.TimeField != "" && k == configuredFields.TimeField {
hasTimeField = true hasTimeField = true
} else if shouldSortLogMessageField && configuredFields.LogMessageField != "" && k == configuredFields.LogMessageField {
hasLogMessageField = true
} else { } else {
sortedPropNames = append(sortedPropNames, k) sortedPropNames = append(sortedPropNames, k)
} }
@ -909,8 +972,12 @@ func sortPropNames(propNames map[string]bool, timeField string) []string {
sort.Strings(sortedPropNames) sort.Strings(sortedPropNames)
if hasLogMessageField {
sortedPropNames = append([]string{configuredFields.LogMessageField}, sortedPropNames...)
}
if hasTimeField { if hasTimeField {
sortedPropNames = append([]string{timeField}, sortedPropNames...) sortedPropNames = append([]string{configuredFields.TimeField}, sortedPropNames...)
} }
return sortedPropNames return sortedPropNames
@ -939,3 +1006,11 @@ func createFieldOfType[T int | float64 | bool | string](docs []map[string]interf
field.Config = &data.FieldConfig{Filterable: &isFilterable} field.Config = &data.FieldConfig{Filterable: &isFilterable}
return field return field
} }
func setPreferredVisType(frame *data.Frame, visType data.VisType) {
if frame.Meta == nil {
frame.Meta = &data.FrameMeta{}
}
frame.Meta.PreferredVisualization = visType
}

129
pkg/tsdb/elasticsearch/response_parser_test.go
Unescape View File

@ -1043,6 +1043,127 @@ func TestResponseParser(t *testing.T) {
require.Nil(t, frame.Fields[1].Config) require.Nil(t, frame.Fields[1].Config)
}) })
t.Run("Log query", func(t *testing.T) {
targets := map[string]string{
"A": `{
"metrics": [{ "type": "logs" }]
}`,
}
response := `{
"responses":[
{
"hits":{
"total":{
"value":109,
"relation":"eq"
},
"max_score":null,
"hits":[
{
"_index":"logs-2023.02.08",
"_id":"GB2UMYYBfCQ-FCMjayJa",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:55.830Z",
"line":"log text [479231733]",
"counter":"109",
"float":58.253758485091,
"label":"val1",
"lvl":"info",
"location":"17.089705232090438, 41.62861966340297",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": null
},
"sort":[
1675869055830,
4
]
},
{
"_index":"logs-2023.02.08",
"_id":"Fx2UMYYBfCQ-FCMjZyJ_",
"_score":null,
"_source":{
"@timestamp":"2023-02-08T15:10:54.835Z",
"line":"log text with ANSI \u001b[31mpart of the text\u001b[0m [493139080]",
"counter":"108",
"float":54.5977098233944,
"label":"val1",
"lvl":"info",
"location":"19.766305918490463, 40.42639175509792",
"nested": {
"field": {
"double_nested": "value"
}
},
"shapes":[
{
"type":"triangle"
},
{
"type":"square"
}
],
"xyz": "def"
},
"sort":[
1675869054835,
7
]
}
]
},
"status":200
}
]
}`
result, err := parseTestResponse(targets, response)
require.NoError(t, err)
require.Len(t, result.Responses, 1)
queryRes := result.Responses["A"]
require.NotNil(t, queryRes)
dataframes := queryRes.Frames
require.Len(t, dataframes, 1)
frame := dataframes[0]
require.Equal(t, 16, len(frame.Fields))
// Fields have the correct length
require.Equal(t, 2, frame.Fields[0].Len())
// First field is timeField
require.Equal(t, data.FieldTypeNullableTime, frame.Fields[0].Type())
// Second is log line
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
require.Equal(t, "line", frame.Fields[1].Name)
// Correctly renames lvl field to level
require.Equal(t, "level", frame.Fields[10].Name)
// Correctly uses string types
require.Equal(t, data.FieldTypeNullableString, frame.Fields[1].Type())
// Correctly detects float64 types
require.Equal(t, data.FieldTypeNullableFloat64, frame.Fields[7].Type())
// Correctly detects json types
require.Equal(t, data.FieldTypeNullableJSON, frame.Fields[8].Type())
// Correctly flattens fields
require.Equal(t, "nested.field.double_nested", frame.Fields[12].Name)
require.Equal(t, data.FieldTypeNullableString, frame.Fields[12].Type())
// Correctly detects type even if first value is null
require.Equal(t, data.FieldTypeNullableString, frame.Fields[15].Type())
})
t.Run("Raw data query", func(t *testing.T) { t.Run("Raw data query", func(t *testing.T) {
targets := map[string]string{ targets := map[string]string{
"A": `{ "A": `{
@ -1252,7 +1373,11 @@ func TestResponseParser(t *testing.T) {
func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) { func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*backend.QueryDataResponse, error) {
from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC) from := time.Date(2018, 5, 15, 17, 50, 0, 0, time.UTC)
to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC) to := time.Date(2018, 5, 15, 17, 55, 0, 0, time.UTC)
timeField := "@timestamp" configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
timeRange := backend.TimeRange{ timeRange := backend.TimeRange{
From: from, From: from,
To: to, To: to,
@ -1280,7 +1405,7 @@ func parseTestResponse(tsdbQueries map[string]string, responseBody string) (*bac
return nil, err return nil, err
} }
return parseResponse(response.Responses, queries, timeField) return parseResponse(response.Responses, queries, configuredFields)
} }
func TestLabelOrderInFieldName(t *testing.T) { func TestLabelOrderInFieldName(t *testing.T) {

1
pkg/tsdb/elasticsearch/snapshot_test.go
Unescape View File

@ -132,6 +132,7 @@ func TestResponseSnapshots(t *testing.T) {
{name: "complex metric test", path: "metric_complex"}, {name: "complex metric test", path: "metric_complex"},
{name: "multi metric test", path: "metric_multi"}, {name: "multi metric test", path: "metric_multi"},
{name: "raw data test", path: "raw_data"}, {name: "raw data test", path: "raw_data"},
{name: "logs test", path: "logs"},
} }
snapshotCount := findResponseSnapshotCounts(t, "testdata_response") snapshotCount := findResponseSnapshotCounts(t, "testdata_response")

534
pkg/tsdb/elasticsearch/testdata_response/logs.a.golden.jsonc
Unescape View File

@ -0,0 +1,534 @@
// 🌟 This was machine generated. Do not edit. 🌟
//
// Frame[0] {
// "typeVersion": [
// 0,
// 0
// ],
// "preferredVisualisationType": "logs"
// }
// Name:
// Dimensions: 17 Fields by 5 Rows
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | Name: testtime | Name: line | Name: _id | Name: _index | Name: _source | Name: _type | Name: abc | Name: counter | Name: float | Name: highlight | Name: is_true | Name: label | Name: level | Name: location | Name: nested_field.internal.nested | Name: shapes | Name: sort |
// | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: | Labels: |
// | Type: []*time.Time | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage | Type: []*string | Type: []*float64 | Type: []*float64 | Type: []*json.RawMessage | Type: []*bool | Type: []*string | Type: []*string | Type: []*string | Type: []*string | Type: []*json.RawMessage | Type: []*json.RawMessage |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
// | 2023-02-09 14:40:01.475 +0000 UTC | log text [106619125] | g2aeNoYB7vaC3bq-ezfK | logs-2023.02.09 | {"abc":null,"counter":81,"float":10.911972180833306,"is_true":true,"label":"val3","line":"log text [106619125]","location":"-42.73465234425797, -14.097854057104112","lvl":"info","nested_field.internal.nested":"value1","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:01.475Z"} | null | null | 81 | 10.911972180833306 | null | true | val3 | info | -42.73465234425797, -14.097854057104112 | value1 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953601475,4] |
// | 2023-02-09 14:40:00.513 +0000 UTC | log text with [781660944] | gmaeNoYB7vaC3bq-eDcN | logs-2023.02.09 | {"abc":null,"counter":80,"float":62.94120607636795,"is_true":false,"label":"val3","line":"log text with [781660944]","location":"42.07571917624318, 15.95725088484611","lvl":"error","nested_field.internal.nested":"value2","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:40:00.513Z"} | null | null | 80 | 62.94120607636795 | null | false | val3 | error | 42.07571917624318, 15.95725088484611 | value2 | [{"type":"triangle"},{"type":"square"}] | [1675953600513,7] |
// | 2023-02-09 14:39:59.556 +0000 UTC | log text [894867430] | gWaeNoYB7vaC3bq-dDdL | logs-2023.02.09 | {"abc":"def","counter":79,"float":53.323706427230455,"is_true":true,"label":"val1","line":"log text [894867430]","location":"-38.27341566189766, -23.66739642570781","lvl":"info","nested_field.internal.nested":"value3","shapes":[{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:59.556Z"} | null | def | 79 | 53.323706427230455 | null | true | val1 | info | -38.27341566189766, -23.66739642570781 | value3 | [{"type":"triangle"},{"type":"square"}] | [1675953599556,10] |
// | 2023-02-09 14:39:58.608 +0000 UTC | log text [478598889] | gGaeNoYB7vaC3bq-cDeY | logs-2023.02.09 | {"abc":"def","counter":78,"float":82.72012623471589,"is_true":false,"label":"val1","line":"log text [478598889]","location":"12.373240290451287, 43.265493464362024","lvl":"info","nested_field.internal.nested":"value4","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:58.608Z"} | null | def | 78 | 82.72012623471589 | null | false | val1 | info | 12.373240290451287, 43.265493464362024 | value4 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953598608,15] |
// | 2023-02-09 14:39:57.665 +0000 UTC | log text [526995818] | f2aeNoYB7vaC3bq-bDf7 | logs-2023.02.09 | {"abc":"def","counter":77,"float":35.05784443331803,"is_true":false,"label":"val3","line":"log text [526995818]","location":"-31.524344042228194, -32.11254790120572","lvl":"info","nested_field.internal.nested":"value5","shapes":[{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}],"testtime":"2023-02-09T14:39:57.665Z"} | null | def | 77 | 35.05784443331803 | null | false | val3 | info | -31.524344042228194, -32.11254790120572 | value5 | [{"type":"triangle"},{"type":"triangle"},{"type":"triangle"},{"type":"square"}] | [1675953597665,20] |
// +-----------------------------------+---------------------------+----------------------+-----------------+-------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------------+--------------------------+-----------------+------------------+--------------------+--------------------------+---------------+-----------------+-----------------+-----------------------------------------+------------------------------------+---------------------------------------------------------------------------------+--------------------------+
//
//
// 🌟 This was machine generated. Do not edit. 🌟
{
"status": 200,
"frames": [
{
"schema": {
"meta": {
"typeVersion": [
0,
0
],
"preferredVisualisationType": "logs"
},
"fields": [
{
"name": "testtime",
"type": "time",
"typeInfo": {
"frame": "time.Time",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "line",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_id",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_index",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_source",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "_type",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "abc",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "counter",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "float",
"type": "number",
"typeInfo": {
"frame": "float64",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "highlight",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "is_true",
"type": "boolean",
"typeInfo": {
"frame": "bool",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "label",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "level",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "location",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "nested_field.internal.nested",
"type": "string",
"typeInfo": {
"frame": "string",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "shapes",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
},
{
"name": "sort",
"type": "other",
"typeInfo": {
"frame": "json.RawMessage",
"nullable": true
},
"config": {
"filterable": true
}
}
]
},
"data": {
"values": [
[
1675953601475,
1675953600513,
1675953599556,
1675953598608,
1675953597665
],
[
"log text [106619125]",
"log text with [781660944]",
"log text [894867430]",
"log text [478598889]",
"log text [526995818]"
],
[
"g2aeNoYB7vaC3bq-ezfK",
"gmaeNoYB7vaC3bq-eDcN",
"gWaeNoYB7vaC3bq-dDdL",
"gGaeNoYB7vaC3bq-cDeY",
"f2aeNoYB7vaC3bq-bDf7"
],
[
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09",
"logs-2023.02.09"
],
[
{
"abc": null,
"counter": 81,
"float": 10.911972180833306,
"is_true": true,
"label": "val3",
"line": "log text [106619125]",
"location": "-42.73465234425797, -14.097854057104112",
"lvl": "info",
"nested_field.internal.nested": "value1",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:01.475Z"
},
{
"abc": null,
"counter": 80,
"float": 62.94120607636795,
"is_true": false,
"label": "val3",
"line": "log text with [781660944]",
"location": "42.07571917624318, 15.95725088484611",
"lvl": "error",
"nested_field.internal.nested": "value2",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:40:00.513Z"
},
{
"abc": "def",
"counter": 79,
"float": 53.323706427230455,
"is_true": true,
"label": "val1",
"line": "log text [894867430]",
"location": "-38.27341566189766, -23.66739642570781",
"lvl": "info",
"nested_field.internal.nested": "value3",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:59.556Z"
},
{
"abc": "def",
"counter": 78,
"float": 82.72012623471589,
"is_true": false,
"label": "val1",
"line": "log text [478598889]",
"location": "12.373240290451287, 43.265493464362024",
"lvl": "info",
"nested_field.internal.nested": "value4",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:58.608Z"
},
{
"abc": "def",
"counter": 77,
"float": 35.05784443331803,
"is_true": false,
"label": "val3",
"line": "log text [526995818]",
"location": "-31.524344042228194, -32.11254790120572",
"lvl": "info",
"nested_field.internal.nested": "value5",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"testtime": "2023-02-09T14:39:57.665Z"
}
],
[
null,
null,
null,
null,
null
],
[
null,
null,
"def",
"def",
"def"
],
[
81,
80,
79,
78,
77
],
[
10.911972180833306,
62.94120607636795,
53.323706427230455,
82.72012623471589,
35.05784443331803
],
[
null,
null,
null,
null,
null
],
[
true,
false,
true,
false,
false
],
[
"val3",
"val3",
"val1",
"val1",
"val3"
],
[
"info",
"error",
"info",
"info",
"info"
],
[
"-42.73465234425797, -14.097854057104112",
"42.07571917624318, 15.95725088484611",
"-38.27341566189766, -23.66739642570781",
"12.373240290451287, 43.265493464362024",
"-31.524344042228194, -32.11254790120572"
],
[
"value1",
"value2",
"value3",
"value4",
"value5"
],
[
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
[
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
]
],
[
[
1675953601475,
4
],
[
1675953600513,
7
],
[
1675953599556,
10
],
[
1675953598608,
15
],
[
1675953597665,
20
]
]
]
}
}
]
}

23
pkg/tsdb/elasticsearch/testdata_response/logs.queries.json
Unescape View File

@ -0,0 +1,23 @@
[
{
"alias": "",
"datasource": {
"type": "elasticsearch",
"uid": "haha"
},
"datasourceId": 42,
"expression": "",
"hide": true,
"intervalMs": 200,
"maxDataPoints": 1248,
"metrics": [
{
"id": "1",
"type": "logs"
}
],
"query": "",
"refId": "a",
"window": ""
}
]

208
pkg/tsdb/elasticsearch/testdata_response/logs.response.json
Unescape View File

@ -0,0 +1,208 @@
{
"took": 6,
"responses": [
{
"took": 6,
"timed_out": false,
"_shards": {
"total": 1,
"successful": 1,
"skipped": 0,
"failed": 0
},
"hits": {
"total": {
"value": 81,
"relation": "eq"
},
"max_score": null,
"hits": [
{
"_index": "logs-2023.02.09",
"_id": "g2aeNoYB7vaC3bq-ezfK",
"_score": null,
"_source": {
"abc": null,
"is_true": true,
"testtime": "2023-02-09T14:40:01.475Z",
"line": "log text [106619125]",
"counter": 81,
"float": 10.911972180833306,
"label": "val3",
"lvl": "info",
"location": "-42.73465234425797, -14.097854057104112",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value1"
}
}
},
"sort": [
1675953601475,
4
]
},
{
"_index": "logs-2023.02.09",
"_id": "gmaeNoYB7vaC3bq-eDcN",
"_score": null,
"_source": {
"abc": null,
"is_true": false,
"testtime": "2023-02-09T14:40:00.513Z",
"line": "log text with [781660944]",
"counter": 80,
"float": 62.94120607636795,
"label": "val3",
"lvl": "error",
"location": "42.07571917624318, 15.95725088484611",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value2"
}
}
},
"sort": [
1675953600513,
7
]
},
{
"_index": "logs-2023.02.09",
"_id": "gWaeNoYB7vaC3bq-dDdL",
"_score": null,
"_source": {
"abc": "def",
"is_true": true,
"testtime": "2023-02-09T14:39:59.556Z",
"line": "log text [894867430]",
"counter": 79,
"float": 53.323706427230455,
"label": "val1",
"lvl": "info",
"location": "-38.27341566189766, -23.66739642570781",
"shapes": [
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value3"
}
}
},
"sort": [
1675953599556,
10
]
},
{
"_index": "logs-2023.02.09",
"_id": "gGaeNoYB7vaC3bq-cDeY",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:58.608Z",
"line": "log text [478598889]",
"counter": 78,
"float": 82.72012623471589,
"label": "val1",
"lvl": "info",
"location": "12.373240290451287, 43.265493464362024",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value4"
}
}
},
"sort": [
1675953598608,
15
]
},
{
"_index": "logs-2023.02.09",
"_id": "f2aeNoYB7vaC3bq-bDf7",
"_score": null,
"_source": {
"abc": "def",
"is_true": false,
"testtime": "2023-02-09T14:39:57.665Z",
"line": "log text [526995818]",
"counter": 77,
"float": 35.05784443331803,
"label": "val3",
"lvl": "info",
"location": "-31.524344042228194, -32.11254790120572",
"shapes": [
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "triangle"
},
{
"type": "square"
}
],
"nested_field": {
"internal": {
"nested": "value5"
}
}
},
"sort": [
1675953597665,
20
]
}
]
},
"status": 200
}
]
}

4
pkg/tsdb/elasticsearch/time_series_query.go
Unescape View File

@ -54,7 +54,7 @@ func (e *timeSeriesQuery) execute() (*backend.QueryDataResponse, error) {
return &backend.QueryDataResponse{}, err return &backend.QueryDataResponse{}, err
} }
return parseResponse(res.Responses, queries, e.client.GetTimeField()) return parseResponse(res.Responses, queries, e.client.GetConfiguredFields())
} }
func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64) error { func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilder, from, to int64) error {
@ -63,7 +63,7 @@ func (e *timeSeriesQuery) processQuery(q *Query, ms *es.MultiSearchRequestBuilde
return err return err
} }
defaultTimeField := e.client.GetTimeField() defaultTimeField := e.client.GetConfiguredFields().TimeField
b := ms.Search(q.Interval) b := ms.Search(q.Interval)
b.Size(0) b.Size(0)
filters := b.Query().Bool().Filter() filters := b.Query().Bool().Filter()

22
pkg/tsdb/elasticsearch/time_series_query_test.go
Unescape View File

@ -28,7 +28,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.NoError(t, err) require.NoError(t, err)
sr := c.multisearchRequests[0].Requests[0] sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField) require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs) require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -418,7 +418,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0] sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField) require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs) require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -439,7 +439,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
sr := c.multisearchRequests[0].Requests[0] sr := c.multisearchRequests[0].Requests[0]
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField) require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs) require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -1305,7 +1305,7 @@ func TestExecuteTimeSeriesQuery(t *testing.T) {
require.Equal(t, sr.Size, defaultSize) require.Equal(t, sr.Size, defaultSize)
rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter) rangeFilter := sr.Query.Bool.Filters[0].(*es.RangeFilter)
require.Equal(t, rangeFilter.Key, c.timeField) require.Equal(t, rangeFilter.Key, c.configuredFields.TimeField)
require.Equal(t, rangeFilter.Lte, toMs) require.Equal(t, rangeFilter.Lte, toMs)
require.Equal(t, rangeFilter.Gte, fromMs) require.Equal(t, rangeFilter.Gte, fromMs)
require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS) require.Equal(t, rangeFilter.Format, es.DateFormatEpochMS)
@ -1687,7 +1687,7 @@ func TestSettingsCasting(t *testing.T) {
} }
type fakeClient struct { type fakeClient struct {
timeField string configuredFields es.ConfiguredFields
multiSearchResponse *es.MultiSearchResponse multiSearchResponse *es.MultiSearchResponse
multiSearchError error multiSearchError error
builder *es.MultiSearchRequestBuilder builder *es.MultiSearchRequestBuilder
@ -1695,15 +1695,21 @@ type fakeClient struct {
} }
func newFakeClient() *fakeClient { func newFakeClient() *fakeClient {
configuredFields := es.ConfiguredFields{
TimeField: "@timestamp",
LogMessageField: "line",
LogLevelField: "lvl",
}
return &fakeClient{ return &fakeClient{
timeField: "@timestamp", configuredFields: configuredFields,
multisearchRequests: make([]*es.MultiSearchRequest, 0), multisearchRequests: make([]*es.MultiSearchRequest, 0),
multiSearchResponse: &es.MultiSearchResponse{}, multiSearchResponse: &es.MultiSearchResponse{},
} }
} }
func (c *fakeClient) GetTimeField() string { func (c *fakeClient) GetConfiguredFields() es.ConfiguredFields {
return c.timeField return c.configuredFields
} }
func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) { func (c *fakeClient) GetMinInterval(queryInterval string) (time.Duration, error) {

Write Preview
Loading…
Cancel
Save