elasticsearch: add 7.x version support (#16646)

Adds a new version option 7.0+ (70 internally).
Version 7.0+ doesn't include document types in index mappings 
so support for handling this have been added.
Version 7.0+ returns number of hits in a different way so 
support for handling this have been added.
Version 7.0+ doesn't support sending 
max_concurrent_shard_requests in multisearch header so 
support for sending this in query string have been added.
Update elastic6 docker block and dashboards (devenv) to use 
6.7.1 images, filebeat index name is now filebeat-YYYY.MM.DD 
and dashboard include correct tags and links.
Add elastic7 docker block and provisioning (devenv).
Updates documentation regarding new version.

Closes #15622
pull/16764/head
Alcides Viamontes E 7 years ago committed by Marcus Efraimsson
parent 42b745a098
commit cff2be0d66
  1. 33
      devenv/datasources.yaml
  2. 14
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_compare.json
  3. 4
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v2.json
  4. 6
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v5.json
  5. 4
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6.json
  6. 10
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v6_filebeat.json
  7. 683
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7.json
  8. 264
      devenv/dev-dashboards/datasource-elasticsearch/elasticsearch_v7_filebeat.json
  9. 4
      devenv/docker/blocks/elastic6/docker-compose.yaml
  10. 1921
      devenv/docker/blocks/elastic6/filebeat.yml
  11. 23
      devenv/docker/blocks/elastic7/docker-compose.yaml
  12. 2
      devenv/docker/blocks/elastic7/elasticsearch.yml
  13. 1978
      devenv/docker/blocks/elastic7/filebeat.yml
  14. 2
      docs/sources/administration/provisioning.md
  15. 6
      docs/sources/features/datasources/elasticsearch.md
  16. 23
      pkg/tsdb/elasticsearch/client/client.go
  17. 67
      pkg/tsdb/elasticsearch/client/client_test.go
  18. 2
      pkg/tsdb/elasticsearch/client/models.go
  19. 5
      public/app/plugins/datasource/elasticsearch/config_ctrl.ts
  20. 29
      public/app/plugins/datasource/elasticsearch/datasource.ts
  21. 6
      public/app/plugins/datasource/elasticsearch/elastic_response.ts
  22. 222
      public/app/plugins/datasource/elasticsearch/specs/datasource.test.ts

@ -99,13 +99,44 @@ datasources:
- name: gdev-elasticsearch-v6-filebeat
type: elasticsearch
access: proxy
database: "[filebeat-6.2.4-]YYYY.MM.DD"
database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:11200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 60
- name: gdev-elasticsearch-v7-metrics
type: elasticsearch
access: proxy
database: "[metrics-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
timeInterval: 10s
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-logs
type: elasticsearch
access: proxy
database: "[logs-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-elasticsearch-v7-filebeat
type: elasticsearch
access: proxy
database: "[filebeat-]YYYY.MM.DD"
url: http://localhost:12200
jsonData:
interval: Daily
timeField: "@timestamp"
esVersion: 70
- name: gdev-mysql
type: mysql
url: localhost:3306

@ -17,11 +17,13 @@
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"iteration": 1545263815779,
"iteration": 1555595032099,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
@ -5841,9 +5843,9 @@
}
],
"refresh": false,
"schemaVersion": 16,
"schemaVersion": 18,
"style": "dark",
"tags": ["gdev", "elasticsearch"],
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
@ -5852,7 +5854,9 @@
"value": "gdev-elasticsearch-v5-metrics"
},
"hide": 0,
"includeAll": false,
"label": "Version One",
"multi": false,
"name": "version_one",
"options": [],
"query": "elasticsearch",
@ -5867,7 +5871,9 @@
"value": "gdev-elasticsearch-v6-metrics"
},
"hide": 0,
"includeAll": false,
"label": "Version Two",
"multi": false,
"name": "version_two",
"options": [],
"query": "elasticsearch",
@ -5889,5 +5895,5 @@
"timezone": "",
"title": "Datasource tests - Elasticsearch comparison",
"uid": "fuFWehBmk",
"version": 4
"version": 2
}

@ -31,8 +31,10 @@
"iteration": 1554310942895,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],

@ -31,10 +31,10 @@
"iteration": 1554310560048,
"links": [
{
"asDropdown": false,
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"title": "Dashboard",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],

@ -31,8 +31,10 @@
"iteration": 1554310839317,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch"],
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],

@ -16,7 +16,15 @@
"gnetId": null,
"graphTooltip": 0,
"iteration": 1554902936982,
"links": [],
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {

@ -0,0 +1,683 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": false,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"limit": 100,
"name": "Annotations & Alerts",
"showIn": 0,
"type": "dashboard"
},
{
"datasource": "gdev-elasticsearch-v7-logs",
"enable": false,
"iconColor": "rgba(255, 96, 96, 1)",
"limit": 100,
"name": "test",
"query": "",
"showIn": 0,
"textField": "description",
"type": "alert"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2342,
"iteration": 1555593977614,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 0
},
"id": 1,
"legend": {
"alignAsTable": true,
"avg": false,
"current": false,
"max": true,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "1",
"size": "5"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "max"
}
],
"query": "*",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Top 5 servers",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 0,
"y": 7
},
"id": 2,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {
"percents": [25, 50, 75, 95, 99]
},
"type": "percentiles"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Percentiles & Metric filter",
"tooltip": {
"msResolution": false,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"aliasColors": {
"Count": "#6ED0E0"
},
"bars": false,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fill": 1,
"grid": {},
"gridPos": {
"h": 6,
"w": 12,
"x": 12,
"y": 7
},
"id": 3,
"legend": {
"alignAsTable": true,
"avg": true,
"current": false,
"max": false,
"min": false,
"rightSide": true,
"show": true,
"total": false,
"values": true
},
"lines": true,
"linewidth": 2,
"links": [],
"nullPointMode": "connected",
"percentage": false,
"pointradius": 5,
"points": false,
"renderer": "flot",
"seriesOverrides": [
{
"alias": "Count",
"lines": false,
"yaxis": 2,
"zindex": -1
}
],
"spaceLength": 10,
"stack": false,
"steppedLine": false,
"targets": [
{
"alias": "{{metric}}",
"bucketAggs": [
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "auto",
"min_doc_count": 0,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {
"std_deviation_bounds_lower": true,
"std_deviation_bounds_upper": true
},
"settings": {},
"type": "extended_stats"
}
],
"query": "@metric:cpu",
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Standard dev",
"tooltip": {
"msResolution": true,
"shared": true,
"sort": 0,
"value_type": "cumulative"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@hostname",
"value": "@hostname"
},
{
"text": "Average",
"value": "Average"
},
{
"text": "Max",
"value": "Max"
},
{
"text": "Sum",
"value": "Sum"
}
],
"datasource": "gdev-elasticsearch-v7-metrics",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 13
},
"id": 6,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"pattern": "/.*/",
"thresholds": [],
"type": "number",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [
{
"field": "@hostname",
"id": "2",
"settings": {
"min_doc_count": 1,
"order": "asc",
"orderBy": "_term",
"size": "0"
},
"type": "terms"
}
],
"dsType": "elasticsearch",
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
},
{
"field": "@value",
"id": "3",
"meta": {},
"settings": {},
"type": "max"
},
{
"field": "@value",
"id": "4",
"meta": {},
"settings": {},
"type": "sum"
}
],
"refId": "B",
"timeField": "@timestamp"
}
],
"title": "ES Metrics",
"transform": "table",
"type": "table"
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "@message",
"value": "@message"
},
{
"text": "tags",
"value": "tags"
},
{
"text": "description",
"value": "description"
}
],
"datasource": "gdev-elasticsearch-v7-logs",
"editable": true,
"error": false,
"fontSize": "100%",
"gridPos": {
"h": 7,
"w": 24,
"x": 0,
"y": 20
},
"id": 5,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
}
],
"targets": [
{
"bucketAggs": [],
"dsType": "elasticsearch",
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"title": "ES Log query",
"transform": "json",
"type": "table"
},
{
"circleMaxSize": 30,
"circleMinSize": 2,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"datasource": "gdev-elasticsearch-v7-metrics",
"decimals": 0,
"esGeoPoint": "@location",
"esMetric": "Average",
"gridPos": {
"h": 12,
"w": 24,
"x": 0,
"y": 27
},
"hideEmpty": false,
"hideZero": false,
"id": 8,
"initialZoom": "1",
"links": [],
"locationData": "geohash",
"mapCenter": "(0°, 0°)",
"mapCenterLatitude": 0,
"mapCenterLongitude": 0,
"maxDataPoints": 1,
"mouseWheelZoom": false,
"showLegend": true,
"stickyLabels": false,
"tableQueryOptions": {
"geohashField": "geohash",
"latitudeField": "latitude",
"longitudeField": "longitude",
"metricField": "metric",
"queryType": "geohash"
},
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "@location",
"id": "3",
"settings": {
"precision": 2
},
"type": "geohash_grid"
}
],
"metrics": [
{
"field": "@value",
"id": "1",
"meta": {},
"settings": {},
"type": "avg"
}
],
"refId": "A",
"target": "",
"timeField": "@timestamp"
}
],
"thresholds": "0,10",
"title": "World map panel",
"type": "grafana-worldmap-panel",
"unitPlural": "",
"unitSingle": "",
"valueName": "total"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-metrics",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-30m",
"to": "now"
},
"timepicker": {
"collapse": false,
"enable": true,
"notice": false,
"now": true,
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"status": "Stable",
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"],
"type": "timepicker"
},
"timezone": "browser",
"title": "Datasource tests - Elasticsearch v7",
"uid": "Y-RvmuRWk",
"version": 1
}

@ -0,0 +1,264 @@
{
"annotations": {
"list": [
{
"builtIn": 1,
"datasource": "-- Grafana --",
"enable": true,
"hide": true,
"iconColor": "rgba(0, 211, 255, 1)",
"name": "Annotations & Alerts",
"type": "dashboard"
}
]
},
"editable": true,
"gnetId": null,
"graphTooltip": 0,
"id": 2341,
"iteration": 1555591591930,
"links": [
{
"asDropdown": true,
"icon": "external link",
"tags": ["gdev", "elasticsearch", "datasource-test"],
"title": "Dashboards",
"type": "dashboards"
}
],
"panels": [
{
"aliasColors": {
"error": "red"
},
"bars": true,
"dashLength": 10,
"dashes": false,
"datasource": "gdev-elasticsearch-v7-filebeat",
"fill": 1,
"gridPos": {
"h": 5,
"w": 24,
"x": 0,
"y": 0
},
"id": 4,
"legend": {
"avg": false,
"current": false,
"max": false,
"min": false,
"show": true,
"total": false,
"values": false
},
"lines": false,
"linewidth": 1,
"nullPointMode": "null",
"percentage": false,
"pointradius": 2,
"points": false,
"renderer": "flot",
"seriesOverrides": [],
"spaceLength": 10,
"stack": true,
"steppedLine": false,
"targets": [
{
"bucketAggs": [
{
"fake": true,
"field": "fields.level",
"id": "3",
"settings": {
"min_doc_count": 1,
"order": "desc",
"orderBy": "_term",
"size": "10"
},
"type": "terms"
},
{
"field": "@timestamp",
"id": "2",
"settings": {
"interval": "5m",
"min_doc_count": 1,
"trimEdges": 0
},
"type": "date_histogram"
}
],
"metrics": [
{
"field": "select field",
"id": "1",
"type": "count"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"thresholds": [],
"timeFrom": null,
"timeRegions": [],
"timeShift": null,
"title": "Panel Title",
"tooltip": {
"shared": true,
"sort": 0,
"value_type": "individual"
},
"type": "graph",
"xaxis": {
"buckets": null,
"mode": "time",
"name": null,
"show": true,
"values": []
},
"yaxes": [
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
},
{
"format": "short",
"label": null,
"logBase": 1,
"max": null,
"min": null,
"show": true
}
],
"yaxis": {
"align": false,
"alignLevel": null
}
},
{
"columns": [
{
"text": "@timestamp",
"value": "@timestamp"
},
{
"text": "fields.level",
"value": "fields.level"
},
{
"text": "message",
"value": "message"
}
],
"datasource": "gdev-elasticsearch-v7-filebeat",
"fontSize": "100%",
"gridPos": {
"h": 22,
"w": 24,
"x": 0,
"y": 5
},
"id": 2,
"links": [],
"pageSize": null,
"scroll": true,
"showHeader": true,
"sort": {
"col": 0,
"desc": true
},
"styles": [
{
"alias": "Time",
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"pattern": "@timestamp",
"type": "date"
},
{
"alias": "Level",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"dateFormat": "YYYY-MM-DD HH:mm:ss",
"decimals": 2,
"mappingType": 1,
"pattern": "fields.level",
"thresholds": [""],
"type": "string",
"unit": "short",
"valueMaps": []
},
{
"alias": "Message",
"colorMode": null,
"colors": ["rgba(245, 54, 54, 0.9)", "rgba(237, 129, 40, 0.89)", "rgba(50, 172, 45, 0.97)"],
"decimals": 2,
"pattern": "message",
"preserveFormat": false,
"sanitize": false,
"thresholds": [],
"type": "string",
"unit": "short"
}
],
"targets": [
{
"bucketAggs": [],
"metrics": [
{
"field": "select field",
"id": "1",
"meta": {},
"settings": {
"size": 500
},
"type": "raw_document"
}
],
"query": "fields.app:grafana",
"refId": "A",
"timeField": "@timestamp"
}
],
"timeFrom": null,
"timeShift": null,
"title": "Panel Title",
"transform": "json",
"type": "table"
}
],
"schemaVersion": 18,
"style": "dark",
"tags": ["elasticsearch", "gdev", "datasource-test"],
"templating": {
"list": [
{
"datasource": "gdev-elasticsearch-v7-filebeat",
"filters": [],
"hide": 0,
"label": "",
"name": "Filters",
"skipUrlSync": false,
"type": "adhoc"
}
]
},
"time": {
"from": "now-1h",
"to": "now"
},
"timepicker": {
"refresh_intervals": ["5s", "10s", "30s", "1m", "5m", "15m", "30m", "1h", "2h", "1d"],
"time_options": ["5m", "15m", "1h", "6h", "12h", "24h", "2d", "7d", "30d"]
},
"timezone": "",
"title": "Datasource tests - Elasticsearch v7 Filebeat",
"uid": "M94gguRWz",
"version": 1
}

@ -1,7 +1,7 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch6:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.2.4
image: docker.elastic.co/elasticsearch/elasticsearch-oss:6.7.1
command: elasticsearch
ports:
- "11200:9200"
@ -15,7 +15,7 @@
FD_PORT: 11200
filebeat6:
image: docker.elastic.co/beats/filebeat:6.2.4
image: docker.elastic.co/beats/filebeat-oss:6.7.1
command: filebeat -e -strict.perms=false
volumes:
- ./docker/blocks/elastic6/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro

File diff suppressed because it is too large Load Diff

@ -0,0 +1,23 @@
# You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
elasticsearch7:
image: docker.elastic.co/elasticsearch/elasticsearch-oss:7.0.0
command: elasticsearch -E "discovery.type=single-node"
ports:
- "12200:9200"
- "12300:9300"
fake-elastic7-data:
image: grafana/fake-data-gen
network_mode: bridge
environment:
FD_DATASOURCE: elasticsearch7
FD_PORT: 12200
filebeat7:
image: docker.elastic.co/beats/filebeat-oss:7.0.0
command: filebeat -e -strict.perms=false
volumes:
- ./docker/blocks/elastic7/filebeat.yml:/usr/share/filebeat/filebeat.yml:ro
- /var/log:/var/log:ro
- ../data/log:/var/log/grafana:ro

@ -0,0 +1,2 @@
script.inline: on
script.indexed: on

File diff suppressed because it is too large Load Diff

@ -146,7 +146,7 @@ Since not all datasources have the same configuration settings we only have the
| tlsSkipVerify | boolean | *All* | Controls whether a client verifies the server's certificate chain and host name. |
| graphiteVersion | string | Graphite | Graphite version |
| timeInterval | string | Prometheus, Elasticsearch, InfluxDB, MySQL, PostgreSQL & MSSQL | Lowest interval/step value that should be used for this data source |
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60) |
| esVersion | number | Elasticsearch | Elasticsearch version as a number (2/5/56/60/70) |
| timeField | string | Elasticsearch | Which field that should be used as timestamp |
| interval | string | Elasticsearch | Index date time format. nil(No Pattern), 'Hourly', 'Daily', 'Weekly', 'Monthly' or 'Yearly' |
| authType | string | Cloudwatch | Auth provider. keys/credentials/arn |

@ -58,10 +58,12 @@ a time pattern for the index name or a wildcard.
### Elasticsearch version
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences how queries are composed.
Currently the versions available is 2.x, 5.x, 5.6+ or 6.0+. 5.6+ means a version of 5.6 or less than 6.0. 6.0+ means a version of 6.0 or higher, 6.3.2 for example.
Be sure to specify your Elasticsearch version in the version selection dropdown. This is very important as there are differences on how queries are composed.
Currently the versions available are `2.x`, `5.x`, `5.6+`, `6.0+` or `7.0+`. The value `5.6+` means version 5.6 or higher, but lower than 6.0. The value `6.0+` means
version 6.0 or higher, but lower than 7.0. Finally, `7.0+` means version 7.0 or higher, but lower than 8.0.
### Min time interval
A lower limit for the auto group by time interval. Recommended to be set to write frequency, for example `1m` if your data is written every minute.
This option can also be overridden/configured in a dashboard panel under data source options. It's important to note that this value **needs** to be formatted as a
number followed by a valid time identifier, e.g. `1m` (1 minute) or `30s` (30 seconds). The following time identifiers are supported:

@ -65,7 +65,7 @@ var NewClient = func(ctx context.Context, ds *models.DataSource, timeRange *tsdb
clientLog.Debug("Creating new client", "version", version, "timeField", timeField, "indices", strings.Join(indices, ", "))
switch version {
case 2, 5, 56, 60:
case 2, 5, 56, 60, 70:
return &baseClientImpl{
ctx: ctx,
ds: ds,
@ -112,12 +112,12 @@ type multiRequest struct {
interval tsdb.Interval
}
func (c *baseClientImpl) executeBatchRequest(uriPath string, requests []*multiRequest) (*http.Response, error) {
func (c *baseClientImpl) executeBatchRequest(uriPath, uriQuery string, requests []*multiRequest) (*http.Response, error) {
bytes, err := c.encodeBatchRequests(requests)
if err != nil {
return nil, err
}
return c.executeRequest(http.MethodPost, uriPath, bytes)
return c.executeRequest(http.MethodPost, uriPath, uriQuery, bytes)
}
func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte, error) {
@ -150,9 +150,10 @@ func (c *baseClientImpl) encodeBatchRequests(requests []*multiRequest) ([]byte,
return payload.Bytes(), nil
}
func (c *baseClientImpl) executeRequest(method, uriPath string, body []byte) (*http.Response, error) {
func (c *baseClientImpl) executeRequest(method, uriPath, uriQuery string, body []byte) (*http.Response, error) {
u, _ := url.Parse(c.ds.Url)
u.Path = path.Join(u.Path, uriPath)
u.RawQuery = uriQuery
var req *http.Request
var err error
@ -197,7 +198,8 @@ func (c *baseClientImpl) ExecuteMultisearch(r *MultiSearchRequest) (*MultiSearch
clientLog.Debug("Executing multisearch", "search requests", len(r.Requests))
multiRequests := c.createMultiSearchRequests(r.Requests)
res, err := c.executeBatchRequest("_msearch", multiRequests)
queryParams := c.getMultiSearchQueryParameters()
res, err := c.executeBatchRequest("_msearch", queryParams, multiRequests)
if err != nil {
return nil, err
}
@ -241,7 +243,7 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
mr.header["search_type"] = "count"
}
if c.version >= 56 {
if c.version >= 56 && c.version < 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(256)
mr.header["max_concurrent_shard_requests"] = maxConcurrentShardRequests
}
@ -252,6 +254,15 @@ func (c *baseClientImpl) createMultiSearchRequests(searchRequests []*SearchReque
return multiRequests
}
func (c *baseClientImpl) getMultiSearchQueryParameters() string {
if c.version >= 70 {
maxConcurrentShardRequests := c.getSettings().Get("maxConcurrentShardRequests").MustInt(5)
return fmt.Sprintf("max_concurrent_shard_requests=%d", maxConcurrentShardRequests)
}
return ""
}
func (c *baseClientImpl) MultiSearch() *MultiSearchRequestBuilder {
return NewMultiSearchRequestBuilder(c.GetVersion())
}

@ -103,6 +103,19 @@ func TestClient(t *testing.T) {
So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 60)
})
Convey("When version 70 should return v7.0 client", func() {
ds := &models.DataSource{
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"timeField": "@timestamp",
}),
}
c, err := NewClient(context.Background(), ds, nil)
So(err, ShouldBeNil)
So(c.GetVersion(), ShouldEqual, 70)
})
})
Convey("Given a fake http client", func() {
@ -290,6 +303,60 @@ func TestClient(t *testing.T) {
})
})
Convey("and a v7.0 client", func() {
ds := models.DataSource{
Database: "[metrics-]YYYY.MM.DD",
Url: ts.URL,
JsonData: simplejson.NewFromAny(map[string]interface{}{
"esVersion": 70,
"maxConcurrentShardRequests": 6,
"timeField": "@timestamp",
"interval": "Daily",
}),
}
c, err := NewClient(context.Background(), &ds, timeRange)
So(err, ShouldBeNil)
So(c, ShouldNotBeNil)
Convey("When executing multi search", func() {
ms, err := createMultisearchForTest(c)
So(err, ShouldBeNil)
c.ExecuteMultisearch(ms)
Convey("Should send correct request and payload", func() {
So(req, ShouldNotBeNil)
So(req.Method, ShouldEqual, http.MethodPost)
So(req.URL.Path, ShouldEqual, "/_msearch")
So(req.URL.RawQuery, ShouldEqual, "max_concurrent_shard_requests=6")
So(responseBuffer, ShouldNotBeNil)
headerBytes, err := responseBuffer.ReadBytes('\n')
So(err, ShouldBeNil)
bodyBytes := responseBuffer.Bytes()
jHeader, err := simplejson.NewJson(headerBytes)
So(err, ShouldBeNil)
jBody, err := simplejson.NewJson(bodyBytes)
So(err, ShouldBeNil)
So(jHeader.Get("index").MustString(), ShouldEqual, "metrics-2018.05.15")
So(jHeader.Get("ignore_unavailable").MustBool(false), ShouldEqual, true)
So(jHeader.Get("search_type").MustString(), ShouldEqual, "query_then_fetch")
Convey("and replace $__interval variable", func() {
So(jBody.GetPath("aggs", "2", "aggs", "1", "avg", "script").MustString(), ShouldEqual, "15000*@hostname")
})
Convey("and replace $__interval_ms variable", func() {
So(jBody.GetPath("aggs", "2", "date_histogram", "interval").MustString(), ShouldEqual, "15s")
})
})
})
})
Reset(func() {
newDatasourceHttpClient = currentNewDatasourceHttpClient
})

@ -42,7 +42,7 @@ func (r *SearchRequest) MarshalJSON() ([]byte, error) {
// SearchResponseHits represents search response hits
type SearchResponseHits struct {
Hits []map[string]interface{}
Total int64
Total map[string]interface{}
}
// SearchResponse represents a search response

@ -8,7 +8,9 @@ export class ElasticConfigCtrl {
constructor($scope) {
this.current.jsonData.timeField = this.current.jsonData.timeField || '@timestamp';
this.current.jsonData.esVersion = this.current.jsonData.esVersion || 5;
this.current.jsonData.maxConcurrentShardRequests = this.current.jsonData.maxConcurrentShardRequests || 256;
const defaultMaxConcurrentShardRequests = this.current.jsonData.esVersion >= 70 ? 5 : 256;
this.current.jsonData.maxConcurrentShardRequests =
this.current.jsonData.maxConcurrentShardRequests || defaultMaxConcurrentShardRequests;
}
indexPatternTypes = [
@ -25,6 +27,7 @@ export class ElasticConfigCtrl {
{ name: '5.x', value: 5 },
{ name: '5.6+', value: 56 },
{ name: '6.0+', value: 60 },
{ name: '7.0+', value: 70 },
];
indexPatternTypeChanged() {

@ -234,7 +234,7 @@ export class ElasticDatasource {
ignore_unavailable: true,
index: this.indexPattern.getIndexList(timeFrom, timeTo),
};
if (this.esVersion >= 56) {
if (this.esVersion >= 56 && this.esVersion < 70) {
queryHeader['max_concurrent_shard_requests'] = this.maxConcurrentShardRequests;
}
return angular.toJson(queryHeader);
@ -277,12 +277,15 @@ export class ElasticDatasource {
payload = payload.replace(/\$timeTo/g, options.range.to.valueOf());
payload = this.templateSrv.replace(payload, options.scopedVars);
return this.post('_msearch', payload).then(res => {
const url = this.getMultiSearchUrl();
return this.post(url, payload).then(res => {
return new ElasticResponse(sentTargets, res).getTimeSeries();
});
}
getFields(query) {
const configuredEsVersion = this.esVersion;
return this.get('/_mapping').then(result => {
const typeMap = {
float: 'number',
@ -347,8 +350,14 @@ export class ElasticDatasource {
const index = result[indexName];
if (index && index.mappings) {
const mappings = index.mappings;
for (const typeName in mappings) {
const properties = mappings[typeName].properties;
if (configuredEsVersion < 70) {
for (const typeName in mappings) {
const properties = mappings[typeName].properties;
getFieldsRecursively(properties);
}
} else {
const properties = mappings.properties;
getFieldsRecursively(properties);
}
}
@ -371,7 +380,9 @@ export class ElasticDatasource {
esQuery = esQuery.replace(/\$timeTo/g, range.to.valueOf());
esQuery = header + '\n' + esQuery + '\n';
return this.post('_msearch?search_type=' + searchType, esQuery).then(res => {
const url = this.getMultiSearchUrl();
return this.post(url, esQuery).then(res => {
if (!res.responses[0].aggregations) {
return [];
}
@ -386,6 +397,14 @@ export class ElasticDatasource {
});
}
getMultiSearchUrl() {
if (this.esVersion >= 70 && this.maxConcurrentShardRequests) {
return `_msearch?max_concurrent_shard_requests=${this.maxConcurrentShardRequests}`;
}
return '_msearch';
}
metricFindQuery(query) {
query = angular.fromJson(query);
if (!query) {

@ -184,7 +184,7 @@ export class ElasticResponse {
}
// This is quite complex
// need to recurise down the nested buckets to build series
// need to recurse down the nested buckets to build series
processBuckets(aggs, target, seriesList, table, props, depth) {
let bucket, aggDef, esAgg, aggId;
const maxDepth = target.bucketAggs.length - 1;
@ -310,11 +310,13 @@ export class ElasticResponse {
}
processHits(hits, seriesList) {
const hitsTotal = typeof hits.total === 'number' ? hits.total : hits.total.value; // <- Works with Elasticsearch 7.0+
const series = {
target: 'docs',
type: 'docs',
datapoints: [],
total: hits.total,
total: hitsTotal,
filterable: true,
};
let propName, hit, doc, i;

@ -187,7 +187,7 @@ describe('ElasticDatasource', function(this: any) {
describe('When getting fields', () => {
beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'metricbeat' });
createDatasource({ url: 'http://es.com', index: 'metricbeat', jsonData: { esVersion: 50 } });
ctx.backendSrv.datasourceRequest = jest.fn(options => {
return Promise.resolve({
@ -236,49 +236,191 @@ describe('ElasticDatasource', function(this: any) {
});
});
it('should return nested fields', () => {
ctx.ds
.getFields({
find: 'fields',
query: '*',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp',
'beat.name.raw',
'beat.name',
'beat.hostname',
'system.cpu.system',
'system.cpu.user',
'system.process.cpu.total',
'system.process.name',
]);
});
it('should return nested fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp',
'beat.name.raw',
'beat.name',
'beat.hostname',
'system.cpu.system',
'system.cpu.user',
'system.process.cpu.total',
'system.process.name',
]);
});
it('should return fields related to query type', () => {
ctx.ds
.getFields({
find: 'fields',
query: '*',
type: 'number',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
});
it('should return number fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'number',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['system.cpu.system', 'system.cpu.user', 'system.process.cpu.total']);
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp']);
});
});
describe('When getting fields from ES 7.0', () => {
beforeEach(() => {
createDatasource({ url: 'http://es.com', index: 'genuine.es7._mapping.response', jsonData: { esVersion: 70 } });
ctx.ds
.getFields({
find: 'fields',
query: '*',
type: 'date',
})
.then(fieldObjects => {
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp']);
ctx.backendSrv.datasourceRequest = jest.fn(options => {
return Promise.resolve({
data: {
'genuine.es7._mapping.response': {
mappings: {
properties: {
'@timestamp_millis': {
type: 'date',
format: 'epoch_millis',
},
classification_terms: {
type: 'keyword',
},
domains: {
type: 'keyword',
},
ip_address: {
type: 'ip',
},
justification_blob: {
properties: {
criterion: {
type: 'text',
fields: {
keyword: {
type: 'keyword',
ignore_above: 256,
},
},
},
overall_vote_score: {
type: 'float',
},
shallow: {
properties: {
jsi: {
properties: {
sdb: {
properties: {
dsel2: {
properties: {
'bootlegged-gille': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
'uncombed-boris': {
properties: {
botness: {
type: 'float',
},
general_algorithm_score: {
type: 'float',
},
},
},
},
},
},
},
},
},
},
},
},
},
overall_vote_score: {
type: 'float',
},
ua_terms_long: {
type: 'keyword',
},
ua_terms_short: {
type: 'keyword',
},
},
},
},
},
});
});
});
it('should return nested fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'@timestamp_millis',
'classification_terms',
'domains',
'ip_address',
'justification_blob.criterion.keyword',
'justification_blob.criterion',
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
'ua_terms_long',
'ua_terms_short',
]);
});
it('should return number fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'number',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual([
'justification_blob.overall_vote_score',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.botness',
'justification_blob.shallow.jsi.sdb.dsel2.bootlegged-gille.general_algorithm_score',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.botness',
'justification_blob.shallow.jsi.sdb.dsel2.uncombed-boris.general_algorithm_score',
'overall_vote_score',
]);
});
it('should return date fields', async () => {
const fieldObjects = await ctx.ds.getFields({
find: 'fields',
query: '*',
type: 'date',
});
const fields = _.map(fieldObjects, 'text');
expect(fields).toEqual(['@timestamp_millis']);
});
});

Loading…
Cancel
Save