elasticsearch: add 7.x version support (#16646)

Adds a new version option 7.0+ (70 internally).
Version 7.0+ doesn't include document types in index mappings 
so support for handling this have been added.
Version 7.0+ returns number of hits in a different way so 
support for handling this have been added.
Version 7.0+ doesn't support sending 
max_concurrent_shard_requests in multisearch header so 
support for sending this in query string have been added.
Update elastic6 docker block and dashboards (devenv) to use 
6.7.1 images, filebeat index name is now filebeat-YYYY.MM.DD 
and dashboard include correct tags and links.
Add elastic7 docker block and provisioning (devenv).
Updates documentation regarding new version.

Closes #15622
pull/16764/head
Alcides Viamontes E 7 years ago committed by Marcus Efraimsson
parent 42b745a098
commit cff2be0d66
  1. 33
      devenv/datasources.yaml
  2. 14
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_compare.json
  3. 4
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v2.json
  4. 6
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v5.json
  5. 4
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6.json
  6. 10
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6_filebeat.json
  7. 683
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7.json
  8. 264
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7_filebeat.json
  9. 4
      devenv/docker/blocks/elastic6/docker-compose.yaml
  10. 1921
      devenv/docker/blocks/elastic6/filebeat.yml
  11. 23
      devenv/docker/blocks/elastic7/docker-compose.yaml
  12. 2
      devenv/docker/blocks/elastic7/elasticsearch.yml
  13. 1978
      devenv/docker/blocks/elastic7/filebeat.yml
  14. 2
      docs/sources/administration/provisioning.md
  15. 6
      docs/sources/features/datasources/elasticsearch.md
  16. 23
      pkg/tsdb/elasticsearch/client/client.go
  17. 67
      pkg/tsdb/elasticsearch/client/client_test.go
  18. 2
      pkg/tsdb/elasticsearch/client/models.go
  19. 5
      public/app/plugins/datasource/elasticsearch/config_ctrl.ts
  20. 29
      public/app/plugins/datasource/elasticsearch/datasource.ts
  21. 6
      public/app/plugins/datasource/elasticsearch/elastic_response.ts
  22. 222
      public/app/plugins/datasource/elasticsearch/specs/datasource.test.ts

@ -99,13 +99,44 @@ datasources:
- name: gdev-elasticsearch-v6-filebeat - name: gdev-elasticsearch-v6-filebeat
type: elasticsearch type: elasticsearch
access: proxy access: proxy
database: "[filebeat-6.2.4-]YYYY.MM.DD" database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:11200 url: http://localhost:11200
jsonData: jsonData:
interval: Daily interval: Daily
timeField: "@timestamp" timeField: "@timestamp"
esVersion: 60 esVersion: 60
- name: gdev-elasticsearch-v7-metrics
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
timeInterval: 10s
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-logs
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-filebeat
type: elasticsearch
access: proxy
database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-mysql - name: gdev-mysql
type: mysql type: mysql
url: localhost:3306 url: localhost:3306

@ -17,11 +17,13 @@
"editable": true, "editable": true,
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"iteration": 1545263815779, "iteration": 1555595032099,
"links": [ "links": [
{ {
"asDropdown": true,
"icon": "external link", "icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"], "tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards" "type": "dashboards"
} }
], ],
@ -5841,9 +5843,9 @@
} }
], ],
"refresh": false, "refresh": false,
"schemaVersion": 16, "schemaVersion": 18,
"style": "dark", "style": "dark",
"tags": ["gdev", "elasticsearch"], "tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": { "templating": {
"list": [ "list": [
{ {
@ -5852,7 +5854,9 @@
"value": "gdev-elasticsearch-v5-metrics" "value": "gdev-elasticsearch-v5-metrics"
}, },
"hide": 0, "hide": 0,
"includeAll": false,
"label": "Version One", "label": "Version One",
"multi": false,
"name": "version_one", "name": "version_one",
"options": [], "options": [],
"query": "elasticsearch", "query": "elasticsearch",
@ -5867,7 +5871,9 @@
"value": "gdev-elasticsearch-v6-metrics" "value": "gdev-elasticsearch-v6-metrics"
}, },
"hide": 0, "hide": 0,
"includeAll": false,
"label": "Version Two", "label": "Version Two",
"multi": false,
"name": "version_two", "name": "version_two",
"options": [], "options": [],
"query": "elasticsearch", "query": "elasticsearch",
@ -5889,5 +5895,5 @@
"timezone": "", "timezone": "",
"title": "Datasource tests - Elasticsearch comparison", "title": "Datasource tests - Elasticsearch comparison",
"uid": "fuFWehBmk", "uid": "fuFWehBmk",
"version": 4 "version": 2
} }

@ -31,8 +31,10 @@
"iteration": 1554310942895, "iteration": 1554310942895,
"links": [ "links": [
{ {
"asDropdown": true,
"icon": "external link", "icon": "external link",
"tags": ["gdev", "elasticsearch"], "tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards" "type": "dashboards"
} }
], ],

@ -31,10 +31,10 @@
"iteration": 1554310560048, "iteration": 1554310560048,
"links": [ "links": [
{ {
"asDropdown": false, "asDropdown": true,
"icon": "external link", "icon": "external link",
"tags": ["gdev", "elasticsearch"], "tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboard", "title": "Dashboards",
"type": "dashboards" "type": "dashboards"
} }
], ],

@ -31,8 +31,10 @@
"iteration": 1554310839317, "iteration": 1554310839317,
"links": [ "links": [
{ {
"asDropdown": true,
"icon": "external link", "icon": "external link",
"tags": ["gdev", "elasticsearch"], "tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards" "type": "dashboards"
} }
], ],

@ -16,7 +16,15 @@
"gnetId": null, "gnetId": null,
"graphTooltip": 0, "graphTooltip": 0,
"iteration": 1554902936982, "iteration": 1554902936982,
"links": [], "links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [ "panels": [
{ {
"aliasColors": { "aliasColors": {

@ -0,0 +1,683 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
},
{
"datasource": "gdev-elasticsearch-v7-logs",
"enable": false,
"iconColor": "rgba(255, 96, 96, 1)",
"limit": 100,
"name": "test",
"query": "",
"showIn": 0,
"textField": "description",
"type": "alert"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2342,
"iteration": 1555593977614,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "1",
"size": "5"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "max"
}
],
"query": "*",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Top 5 servers",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 7
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {
"percents": [25, 50, 75, 95, 99]
},
"type": "percentiles"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Percentiles & Metric filter",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {
"std_deviation_bounds_lower": true,
"std_deviation_bounds_upper": true
},
"settings": {},
"type": "extended_stats"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Standard dev",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@hostname",
"value": "@hostname"
},
{
"text": "Average",
"value": "Average"
},
{
"text": "Max",
"value": "Max"
},
{
"text": "Sum",
"value": "Sum"
}
],
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 13
},
"id": 6,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "_term",
"size": "0"
},
"type": "terms"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
},
{
"field": "@value",
"id": "3",
"meta": {},
"settings": {},
"type": "max"
},
{
"field": "@value",
"id": "4",
"meta": {},
"settings": {},
"type": "sum"
}
],
"refId": "B",
"timeField": "@timestamp"
}
],
"title": "ES Metrics",
"transform": "table",
"type": "table"
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "@message",
"value": "@message"
},
{
"text": "tags",
"value": "tags"
},
{
"text": "description",
"value": "description"
}
],
"datasource": "gdev-elasticsearch-v7-logs",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 20
},
"id": 5,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
}
],
"targets": [
{
"bucketAggs": [],
"dsType": "elasticsearch",
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"title": "ES Log query",
"transform": "json",
"type": "table"
},
{
"circleMaxSize": 30,
"circleMinSize": 2,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"datasource": "gdev-elasticsearch-v7-metrics",
"decimals": 0,
"esGeoPoint": "@location",
"esMetric": "Average",
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"hideEmpty": false,
"hideZero": false,
"id": 8,
"initialZoom": "1",
"links": [],
"locationData": "geohash",
"mapCenter": "(0°, 0°)",
"mapCenterLatitude": 0,
"mapCenterLongitude": 0,
"maxDataPoints": 1,
"mouseWheelZoom": false,
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "@location",
"id": "3",
"settings": {
"precision": 2
},
"type": "geohash_grid"
}
],
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": "0,10",
"title": "World map panel",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"valueName": "total"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-metrics",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"status": "Stable",
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"],
"type": "timepicker"
},
"timezone": "browser",
"title": "Datasource tests - Elasticsearch v7",
"uid": "Y-RvmuRWk",
"version": 1
}

@ -0,0 +1,264 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2341,
"iteration": 1555591591930,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {
"error": "red"
},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-filebeat",
"fill": 1,
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "fields.level",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_term",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 1,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Panel Title",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "fields.level",
"value": "fields.level"
},
{
"text": "message",
"value": "message"
}
],
"datasource": "gdev-elasticsearch-v7-filebeat",
"fontSize": "100%",
"gridPos": {
"h": 22,
"w": 24,
"x": 0,
"y": 5
},
"id": 2,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"alias": "Level",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "fields.level",
"thresholds": [""],
"type": "string",
"unit": "short",
"valueMaps": []
},
{
"alias": "Message",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"decimals": 2,
"pattern": "message",
"preserveFormat": false,
"sanitize": false,
"thresholds": [],
"type": "string",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [],
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel Title",
"transform": "json",
"type": "table"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-filebeat",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"]
},
"timezone": "",
"title": "Datasource tests - Elasticsearch v7 Filebeat",
"uid": "M94gguRWz",
"version": 1
}

@ -1,7 +1,7 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine # You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch6: elasticsearch6:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4 image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.7.1
command: elasticsearch command: elasticsearch
ports: ports:
- "11200:9200" - "11200:9200"
@ -15,7 +15,7 @@
FD_PORT: 11200 FD_PORT: 11200
filebeat6: filebeat6:
image: docker.elastic.co/beats/filebeat:6.2.4 image: docker.elastic.co/beats/filebeat-oss:6.7.1
command: filebeat -e -strict.perms=false command: filebeat -e -strict.perms=false
volumes: volumes:
- ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro - ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro

File diff suppressed because it is too large Load Diff

@ -0,0 +1,23 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch7:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0
command: elasticsearch -E "discovery.type=single-node"
ports:
- "12200:9200"
- "12300:9300"
fake-elastic7-data:
image: grafana/fake-data-gen
network_mode: bridge
environment:
FD_DATASOURCE: elasticsearch7
FD_PORT: 12200
filebeat7:
image: docker.elastic.co/beats/filebeat-oss:7.0.0
command: filebeat -e -strict.perms=false
volumes:
- ./docker/blocks/elastic7/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/log:/var/log:ro
- ../data/log:/var/log/grafana:ro

@ -0,0 +1,2 @@
script.inline: on
script.indexed: on

File diff suppressed because it is too large Load Diff

@ -146,7 +146,7 @@ Since not all datasources have the same configuration settings we only have the
| tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. | | tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. |
| graphiteVersion | string | Graphite | Graphite version | | graphiteVersion | string | Graphite | Graphite version |
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source | | timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source |
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60) | | esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60/70) |
| timeField | string | Elasticsearch | Which field that should be used as timestamp | | timeField | string | Elasticsearch | Which field that should be used as timestamp |
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' | | interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
| authType | string | Cloudwatch | Auth provider. keys/credentials/arn | | authType | string | Cloudwatch | Auth provider. keys/credentials/arn |

@ -58,10 +58,12 @@ a time pattern for the index name or a wildcard.
### Elasticsearch version ### Elasticsearch version
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed. Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences on how queries are composed.
Currently the versions available is 2.x, 5.x, 5.6+ or 6.0+. 5.6+ means a version of 5.6 or less than 6.0. 6.0+ means a version of 6.0 or higher, 6.3.2 for example. Currently the versions available are `2.x`, `5.x`, `5.6+`, `6.0+` or `7.0+`. The value `5.6+` means version 5.6 or higher, but lower than 6.0. The value `6.0+` means
version 6.0 or higher, but lower than 7.0. Finally, `7.0+` means version 7.0 or higher, but lower than 8.0.
### Min time interval ### Min time interval
A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute. A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute.
This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a
number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported: number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported:

@ -65,7 +65,7 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", ")) clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", "))
switch version { switch version {
case 2, 5, 56, 60: case 2, 5, 56, 60, 70:
return &baseClientImpl{ return &baseClientImpl{
ctx: ctx, ctx: ctx,
ds: ds, ds: ds,
@ -112,12 +112,12 @@ type multiRequest struct {
interval tsdb.Interval interval tsdb.Interval
} }
func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) { func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
bytes, err := c.encodeBatchRequests(requests) bytes, err := c.encodeBatchRequests(requests)
if err != nil { if err != nil {
return nil, err return nil, err
} }
return c.executeRequest(http.MethodPost, uriPath, bytes) return c.executeRequest(http.MethodPost, uriPath, uriQuery, bytes)
} }
func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) { func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) {
@ -150,9 +150,10 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
return payload.Bytes(), nil return payload.Bytes(), nil
} }
func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) { func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
u, _ := url.Parse(c.ds.Url) u, _ := url.Parse(c.ds.Url)
u.Path = path.Join(u.Path, uriPath) u.Path = path.Join(u.Path, uriPath)
u.RawQuery = uriQuery
var req *http.Request var req *http.Request
var err error var err error
@ -197,7 +198,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
clientLog.Debug("Executing multisearch", "search requests", len(r.Requests)) clientLog.Debug("Executing multisearch", "search requests", len(r.Requests))
multiRequests := c.createMultiSearchRequests(r.Requests) multiRequests := c.createMultiSearchRequests(r.Requests)
res, err := c.executeBatchRequest("_msearch", multiRequests) queryParams := c.getMultiSearchQueryParameters()
res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
if err != nil { if err != nil {
return nil, err return nil, err
} }
@ -241,7 +243,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
mr.header["search_type"] = "count" mr.header["search_type"] = "count"
} }
if c.version >= 56 { if c.version >= 56 && c.version < 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256) maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256)
mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests
} }
@ -252,6 +254,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
return multiRequests return multiRequests
} }
func (c *baseClientImpl) getMultiSearchQueryParameters() string {
if c.version >= 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
}
return ""
}
func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder { func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
return NewMultiSearchRequestBuilder(c.GetVersion()) return NewMultiSearchRequestBuilder(c.GetVersion())
} }

@ -103,6 +103,19 @@ func TestClient(t *testing.T) {
So(err, ShouldBeNil) So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 60) So(c.GetVersion(), ShouldEqual, 60)
}) })
Convey("When version 70 should return v7.0 client", func() {
ds := &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"timeField": "@timestamp",
}),
}
c, err := NewClient(context.Background(), ds, nil)
So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 70)
})
}) })
Convey("Given a fake http client", func() { Convey("Given a fake http client", func() {
@ -290,6 +303,60 @@ func TestClient(t *testing.T) {
}) })
}) })
Convey("and a v7.0 client", func() {
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"maxConcurrentShardRequests": 6,
"timeField": "@timestamp",
"interval": "Daily",
}),
}
c, err := NewClient(context.Background(), &ds, timeRange)
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c)
So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch")
So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
So(responseBuffer, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n')
So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes()
jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil)
jBody, err := simplejson.NewJson(bodyBytes)
So(err, ShouldBeNil)
So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
Convey("and replace $__interval variable", func() {
So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
})
Convey("and replace $__interval_ms variable", func() {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
})
})
})
})
Reset(func() { Reset(func() {
newDatasourceHttpClient = currentNewDatasourceHttpClient newDatasourceHttpClient = currentNewDatasourceHttpClient
}) })

@ -42,7 +42,7 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
// SearchResponseHits represents search response hits // SearchResponseHits represents search response hits
type SearchResponseHits struct { type SearchResponseHits struct {
Hits []map[string]interface{} Hits []map[string]interface{}
Total int64 Total map[string]interface{}
} }
// SearchResponse represents a search response // SearchResponse represents a search response

@ -8,7 +8,9 @@ export class ElasticConfigCtrl {
constructor($scope) { constructor($scope) {
this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp'; this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp';
this.current.jsonData.esVersion = this.current.jsonData.esVersion || 5; this.current.jsonData.esVersion = this.current.jsonData.esVersion || 5;
this.current.jsonData.maxConcurrentShardRequests = this.current.jsonData.maxConcurrentShardRequests || 256; const defaultMaxConcurrentShardRequests = this.current.jsonData.esVersion >= 70 ? 5 : 256;
this.current.jsonData.maxConcurrentShardRequests =
this.current.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
} }
indexPatternTypes = [ indexPatternTypes = [
@ -25,6 +27,7 @@ export class ElasticConfigCtrl {
{ name: '5.x', value: 5 }, { name: '5.x', value: 5 },
{ name: '5.6+', value: 56 }, { name: '5.6+', value: 56 },
{ name: '6.0+', value: 60 }, { name: '6.0+', value: 60 },
{ name: '7.0+', value: 70 },
]; ];
indexPatternTypeChanged() { indexPatternTypeChanged() {

@ -234,7 +234,7 @@ export class ElasticDatasource {
ignore_unavailable: true, ignore_unavailable: true,
index: this.indexPattern.getIndexList(timeFrom, timeTo), index: this.indexPattern.getIndexList(timeFrom, timeTo),
}; };
if (this.esVersion >= 56) { if (this.esVersion >= 56 && this.esVersion < 70) {
queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests; queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests;
} }
return angular.toJson(queryHeader); return angular.toJson(queryHeader);
@ -277,12 +277,15 @@ export class ElasticDatasource {
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf()); payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
payload = this.templateSrv.replace(payload, options.scopedVars); payload = this.templateSrv.replace(payload, options.scopedVars);
return this.post('_msearch', payload).then(res => { const url = this.getMultiSearchUrl();
return this.post(url, payload).then(res => {
return new ElasticResponse(sentTargets, res).getTimeSeries(); return new ElasticResponse(sentTargets, res).getTimeSeries();
}); });
} }
getFields(query) { getFields(query) {
const configuredEsVersion = this.esVersion;
return this.get('/_mapping').then(result => { return this.get('/_mapping').then(result => {
const typeMap = { const typeMap = {
float: 'number', float: 'number',
@ -347,8 +350,14 @@ export class ElasticDatasource {
const index = result[indexName]; const index = result[indexName];
if (index && index.mappings) { if (index && index.mappings) {
const mappings = index.mappings; const mappings = index.mappings;
for (const typeName in mappings) {
const properties = mappings[typeName].properties; if (configuredEsVersion < 70) {
for (const typeName in mappings) {
const properties = mappings[typeName].properties;
getFieldsRecursively(properties);
}
} else {
const properties = mappings.properties;
getFieldsRecursively(properties); getFieldsRecursively(properties);
} }
} }
@ -371,7 +380,9 @@ export class ElasticDatasource {
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf()); esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
esQuery = header + '\n' + esQuery + '\n'; esQuery = header + '\n' + esQuery + '\n';
return this.post('_msearch?search_type=' + searchType, esQuery).then(res => { const url = this.getMultiSearchUrl();
return this.post(url, esQuery).then(res => {
if (!res.responses[0].aggregations) { if (!res.responses[0].aggregations) {
return []; return [];
} }
@ -386,6 +397,14 @@ export class ElasticDatasource {
}); });
} }
getMultiSearchUrl() {
if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
}
return '_msearch';
}
metricFindQuery(query) { metricFindQuery(query) {
query = angular.fromJson(query); query = angular.fromJson(query);
if (!query) { if (!query) {

@ -184,7 +184,7 @@ export class ElasticResponse {
} }
// This is quite complex // This is quite complex
// need to recurise down the nested buckets to build series // need to recurse down the nested buckets to build series
processBuckets(aggs, target, seriesList, table, props, depth) { processBuckets(aggs, target, seriesList, table, props, depth) {
let bucket, aggDef, esAgg, aggId; let bucket, aggDef, esAgg, aggId;
const maxDepth = target.bucketAggs.length - 1; const maxDepth = target.bucketAggs.length - 1;
@ -310,11 +310,13 @@ export class ElasticResponse {
} }
processHits(hits, seriesList) { processHits(hits, seriesList) {
const hitsTotal = typeof hits.total === 'number' ? hits.total : hits.total.value; // <- Works with Elasticsearch 7.0+
const series = { const series = {
target: 'docs', target: 'docs',
type: 'docs', type: 'docs',
datapoints: [], datapoints: [],
total: hits.total, total: hitsTotal,
filterable: true, filterable: true,
}; };
let propName, hit, doc, i; let propName, hit, doc, i;

@ -187,7 +187,7 @@ describe('ElasticDatasource', function(this: any) {
describe('When getting fields', () => { describe('When getting fields', () => {
beforeEach(() => { beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'metricbeat' }); createDatasource({ url: 'http://es.com', index: 'metricbeat', jsonData: { esVersion: 50 } });
ctx.backendSrv.datasourceRequest = jest.fn(options => { ctx.backendSrv.datasourceRequest = jest.fn(options => {
return Promise.resolve({ return Promise.resolve({
@ -236,49 +236,191 @@ describe('ElasticDatasource', function(this: any) {
}); });
}); });
it('should return nested fields', () => { it('should return nested fields', async () => {
ctx.ds const fieldObjects = await ctx.ds.getFields({
.getFields({ find: 'fields',
find: 'fields', query: '*',
query: '*', });
}) const fields = _.map(fieldObjects, 'text');
.then(fieldObjects => { expect(fields).toEqual([
const fields = _.map(fieldObjects, 'text'); '@timestamp',
expect(fields).toEqual([ 'beat.name.raw',
'@timestamp', 'beat.name',
'beat.name.raw', 'beat.hostname',
'beat.name', 'system.cpu.system',
'beat.hostname', 'system.cpu.user',
'system.cpu.system', 'system.process.cpu.total',
'system.cpu.user', 'system.process.name',
'system.process.cpu.total', ]);
'system.process.name',
]);
});
}); });
it('should return fields related to query type', () => { it('should return number fields', async () => {
ctx.ds const fieldObjects = await ctx.ds.getFields({
.getFields({ find: 'fields',
find: 'fields', query: '*',
query: '*', type: 'number',
type: 'number', });
})
.then(fieldObjects => { const fields = _.map(fieldObjects, 'text');
const fields = _.map(fieldObjects, 'text'); expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']); });
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp']);
});
});
describe('When getting fields from ES 7.0', () => {
beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'genuine.es7._mapping.response', jsonData: { esVersion: 70 } });
ctx.ds ctx.backendSrv.datasourceRequest = jest.fn(options => {
.getFields({ return Promise.resolve({
find: 'fields', data: {
query: '*', 'genuine.es7._mapping.response': {
type: 'date', mappings: {
}) properties: {
.then(fieldObjects => { '@timestamp_millis': {
const fields = _.map(fieldObjects, 'text'); type: 'date',
expect(fields).toEqual(['@timestamp']); format: 'epoch_millis',
},
classification_terms: {
type: 'keyword',
},
domains: {
type: 'keyword',
},
ip_address: {
type: 'ip',
},
justification_blob: {
properties: {
criterion: {
type: 'text',
fields: {
keyword: {
type: 'keyword',
ignore_above: 256,
},
},
},
overall_vote_score: {
type: 'float',
},
shallow: {
properties: {
jsi: {
properties: {
sdb: {
properties: {
dsel2: {
properties: {
'bootlegged-gille': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
'uncombed-boris': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
},
},
},
},
},
},
},
},
},
},
overall_vote_score: {
type: 'float',
},
ua_terms_long: {
type: 'keyword',
},
ua_terms_short: {
type: 'keyword',
},
},
},
},
},
}); });
});
});
it('should return nested fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp_millis',
'classification_terms',
'domains',
'ip_address',
'justification_blob.criterion.keyword',
'justification_blob.criterion',
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
'ua_terms_long',
'ua_terms_short',
]);
});
it('should return number fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'number',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp_millis']);
}); });
}); });

Loading…
Cancel
Save