apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Elasticsearch: Run getTerms trough resource call if enableElasticsearchBackendQuerying enabled (#67848)

Browse Source 
* Elasticsearch: Run getTerms torugh backend if toggle enabled

* Add template variables to devenv dashboard for easier testing

* Add TODO

* Run feature toggle gen to fix build
pull/64649/head^2
Ivana Huckova 2 years ago committed by GitHub
parent b2e1b3ad91
commit 3145660f5a
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
4 changed files with 591 additions and 8 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 508
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json
  2. 5
      pkg/tsdb/elasticsearch/elasticsearch.go
  3. 63
      public/app/plugins/datasource/elasticsearch/datasource.test.ts
  4. 23
      public/app/plugins/datasource/elasticsearch/datasource.ts

508
devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_complex.json vendored
Unescape View File

@ -5274,18 +5274,520 @@
],
"title": "Logs",
"type": "row"
},
{
"collapsed": true,
"gridPos": {
"h": 1,
"w": 24,
"x": 0,
"y": 13
},
"id": 97,
"panels": [
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 1
},
"id": 98,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "hostname: $hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname count",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 1
},
"id": 99,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"field": "float",
"id": "1",
"type": "avg"
}
],
"query": "hostname: $hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname average float",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 0,
"y": 9
},
"id": 100,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"id": "1",
"type": "count"
}
],
"query": "hostname: $hostname AND level: $level_by_hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname and $level_by_hostname count",
"type": "timeseries"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"fieldConfig": {
"defaults": {
"color": {
"mode": "palette-classic"
},
"custom": {
"axisCenteredZero": false,
"axisColorMode": "text",
"axisLabel": "",
"axisPlacement": "auto",
"barAlignment": 0,
"drawStyle": "line",
"fillOpacity": 0,
"gradientMode": "none",
"hideFrom": {
"legend": false,
"tooltip": false,
"viz": false
},
"lineInterpolation": "linear",
"lineWidth": 1,
"pointSize": 5,
"scaleDistribution": {
"type": "linear"
},
"showPoints": "auto",
"spanNulls": false,
"stacking": {
"group": "A",
"mode": "none"
},
"thresholdsStyle": {
"mode": "off"
}
},
"mappings": [],
"thresholds": {
"mode": "absolute",
"steps": [
{
"color": "green",
"value": null
},
{
"color": "red",
"value": 80
}
]
}
},
"overrides": []
},
"gridPos": {
"h": 8,
"w": 12,
"x": 12,
"y": 9
},
"id": 101,
"options": {
"legend": {
"calcs": [],
"displayMode": "list",
"placement": "bottom",
"showLegend": true
},
"tooltip": {
"mode": "single",
"sort": "none"
}
},
"targets": [
{
"alias": "",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto"
},
"type": "date_histogram"
}
],
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"metrics": [
{
"field": "float",
"id": "1",
"type": "avg"
}
],
"query": "hostname: $hostname AND level: $level_by_hostname",
"refId": "A",
"timeField": "@timestamp"
}
],
"title": "$hostname and $level_by_hostname average float",
"type": "timeseries"
}
],
"title": "Using template variables",
"type": "row"
}
],
"refresh": "",
"schemaVersion": 38,
"style": "dark",
"tags": [
"tags": [
"gdev",
"elasticsearch",
"datasource-test"
],
"templating": {
"list": []
"list": [
{
"current": {
"selected": true,
"text": "hostname2",
"value": "hostname2"
},
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"definition": "{\"find\": \"terms\", \"field\": \"hostname\"}",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"hostname\"}",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"current": {
"selected": false,
"text": "error",
"value": "error"
},
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"definition": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
"hide": 0,
"includeAll": false,
"multi": false,
"name": "level_by_hostname",
"options": [],
"query": "{\"find\": \"terms\", \"field\": \"level\", \"query\": \"hostname:$hostname\"}",
"refresh": 1,
"regex": "",
"skipUrlSync": false,
"sort": 0,
"type": "query"
},
{
"datasource": {
"type": "elasticsearch",
"uid": "gdev-elasticsearch"
},
"filters": [],
"hide": 0,
"name": "adhoc",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-6h",
@ -5293,7 +5795,7 @@
},
"timepicker": {},
"timezone": "",
"title": "Datasource tests - Elasticsearch complex",
"title": "Datasource tests - Elasticsearch complex with template variables",
"uid": "es_complex",
"version": 1,
"weekStart": ""

5
pkg/tsdb/elasticsearch/elasticsearch.go
Unescape View File

@ -183,8 +183,9 @@ func (s *Service) CallResource(ctx context.Context, req *backend.CallResourceReq
logger := eslog.FromContext(ctx)
// allowed paths for resource calls:
// - empty string for fetching db version
// - /?/_mapping for fetching index mapping
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") {
// - ?/_mapping for fetching index mapping
// - _msearch for executing getTerms queries
if req.Path != "" && !strings.HasSuffix(req.Path, "/_mapping") && req.Path != "_msearch" {
return fmt.Errorf("invalid resource URL: %s", req.Path)
}

63
public/app/plugins/datasource/elasticsearch/datasource.test.ts
Unescape View File

@ -1327,6 +1327,69 @@ describe('ElasticDatasource using backend', () => {
});
});
describe('metricFindQuery', () => {
async function runScenario() {
const data = {
responses: [
{
aggregations: {
'1': {
buckets: [
{ doc_count: 1, key: 'test' },
{
doc_count: 2,
key: 'test2',
key_as_string: 'test2_as_string',
},
],
},
},
},
],
};
const { ds } = getTestContext();
const postResourceMock = jest.spyOn(ds, 'postResource');
postResourceMock.mockResolvedValue(data);
const results = await ds.metricFindQuery('{"find": "terms", "field": "test"}');
expect(ds.postResource).toHaveBeenCalledTimes(1);
const requestOptions = postResourceMock.mock.calls[0][1];
const parts = requestOptions.split('\n');
const header = JSON.parse(parts[0]);
const body = JSON.parse(parts[1]);
return { results, body, header };
}
it('should get results', async () => {
const { results } = await runScenario();
expect(results.length).toEqual(2);
});
it('should use key or key_as_string', async () => {
const { results } = await runScenario();
expect(results[0].text).toEqual('test');
expect(results[1].text).toEqual('test2_as_string');
});
it('should not set search type to count', async () => {
const { header } = await runScenario();
expect(header.search_type).not.toEqual('count');
});
it('should set size to 0', async () => {
const { body } = await runScenario();
expect(body.size).toBe(0);
});
it('should not set terms aggregation size to 0', async () => {
const { body } = await runScenario();
expect(body['aggs']['1']['terms'].size).not.toBe(0);
});
});
describe('getFields', () => {
const getFieldsMockData = {
'[test-]YYYY.MM.DD': {

23
public/app/plugins/datasource/elasticsearch/datasource.ts
Unescape View File

@ -30,7 +30,7 @@ import {
LogRowContextOptions,
SupplementaryQueryOptions,
} from '@grafana/data';
import { DataSourceWithBackend, getDataSourceSrv, config } from '@grafana/runtime';
import { DataSourceWithBackend, getDataSourceSrv, config, BackendSrvRequest } from '@grafana/runtime';
import { queryLogsVolume } from 'app/core/logsModel';
import { getTimeSrv, TimeSrv } from 'app/features/dashboard/services/TimeSrv';
import { getTemplateSrv, TemplateSrv } from 'app/features/templating/template_srv';
@ -148,6 +148,18 @@ export class ElasticDatasource
this.legacyQueryRunner = new LegacyQueryRunner(this, this.templateSrv);
}
getResourceRequest(path: string, params?: BackendSrvRequest['params'], options?: Partial<BackendSrvRequest>) {
return this.getResource(path, params, options);
}
postResourceRequest(path: string, data?: BackendSrvRequest['data'], options?: Partial<BackendSrvRequest>) {
const resourceOptions = options ?? {};
resourceOptions.headers = resourceOptions.headers ?? {};
resourceOptions.headers['content-type'] = 'application/x-ndjson';
return this.postResource(path, data, resourceOptions);
}
async importFromAbstractQueries(abstractQueries: AbstractQuery[]): Promise<ElasticsearchQuery[]> {
return abstractQueries.map((abstractQuery) => this.languageProvider.importFromAbstractQuery(abstractQuery));
}
@ -532,7 +544,12 @@ export class ElasticDatasource
const url = this.getMultiSearchUrl();
return this.legacyQueryRunner.request('POST', url, esQuery).pipe(
const termsObservable = config.featureToggles.enableElasticsearchBackendQuerying
? // TODO: This is run trough resource call, but maybe should run trough query
from(this.postResourceRequest(url, esQuery))
: this.legacyQueryRunner.request('POST', url, esQuery);
return termsObservable.pipe(
map((res) => {
if (!res.responses[0].aggregations) {
return [];
@ -743,7 +760,7 @@ export class ElasticDatasource
private getDatabaseVersionUncached(): Promise<SemVer | null> {
// we want this function to never fail
const getDbVersionObservable = config.featureToggles.enableElasticsearchBackendQuerying
? from(this.getResource(''))
? from(this.getResourceRequest(''))
: this.legacyQueryRunner.request('GET', '/');
return lastValueFrom(getDbVersionObservable).then(

Write Preview
Loading…
Cancel
Save