devenv: influxdb data sources and blocks cleanup (#32894)

* devenv: influxdb cleanup

* devenv: influxdb: auto-delete now-unused gdev-data-sources

devenv/datasources.yaml

@@ -1,5 +1,11 @@
 apiVersion: 1
 
+deleteDatasources:
+  - name: gdev-influxdb
+  - name: gdev-influxdb-telegraf
+  - name: gdev-influxdb2
+  - name: gdev-influxdb2-influxql
+
 datasources:
   - name: gdev-graphite
     type: graphite
@@ -36,19 +42,19 @@ datasources:
     isDefault: true
     type: testdata
 
-  - name: gdev-influxdb
+  - name: gdev-influxdb1-influxql
     type: influxdb
     access: proxy
     database: site
     user: grafana
-    url: http://localhost:8086
+    url: http://localhost:8087
     secureJsonData:
       password: grafana
 
-  - name: gdev-influxdb2
+  - name: gdev-influxdb-flux
     type: influxdb
     access: proxy
-    url: http://localhost:8087
+    url: http://localhost:8086
     user: grafana
     secureJsonData:
       token: mytoken
@@ -58,25 +64,16 @@ datasources:
       organization: myorg
       defaultBucket: mybucket
 
-  - name: gdev-influxdb2-influxql
+  - name: gdev-influxdb-influxql
     type: influxdb
     access: proxy
-    database: site
+    database: mybucket
-    url: http://localhost:8087
+    url: http://localhost:8086
     jsonData:
       httpHeaderName1: "Authorization"
     secureJsonData:
       httpHeaderValue1: "Token mytoken"
 
-  - name: gdev-influxdb-telegraf
-    type: influxdb
-    access: proxy
-    database: telegraf
-    user: grafana
-    url: http://localhost:8086
-    secureJsonData:
-      password: grafana
-
   - name: gdev-opentsdb
     type: opentsdb
     access: proxy

devenv/datasources_docker.yaml

@@ -23,19 +23,19 @@ datasources:
     isDefault: true
     type: testdata
 
-  - name: gdev-influxdb
+  - name: gdev-influxdb1-influxql
     type: influxdb
     access: proxy
     database: site
     user: grafana
-    url: http://influxdb:8086
+    url: http://influxdb1:8086
     secureJsonData:
       password: grafana
 
-  - name: gdev-influxdb2
+  - name: gdev-influxdb-flux
     type: influxdb
     access: proxy
-    url: http://influxdb2:8086
+    url: http://influxdb:8086
     secureJsonData:
       token: mytoken
     jsonData:
@@ -43,25 +43,16 @@ datasources:
       organization: myorg
       defaultBucket: mybucket
       
-  - name: gdev-influxdb2-influxql
+  - name: gdev-influxdb-influxql
     type: influxdb
     access: proxy
-    database: site
+    database: mybucket
-    url: http://influxdb2:8086
+    url: http://influxdb:8086
     jsonData:
       httpHeaderName1: "Authorization"
     secureJsonData:
       httpHeaderValue1: "Token mytoken"
 
-  - name: gdev-influxdb-telegraf
-    type: influxdb
-    access: proxy
-    database: telegraf
-    user: grafana
-    url: http://influxdb:8086
-    secureJsonData:
-      password: grafana
-
   - name: gdev-opentsdb
     type: opentsdb
     access: proxy

devenv/dev-dashboards/datasource-influxdb/influxdb-logs.json

@@ -23,7 +23,7 @@
       "bars": true,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "gdev-influxdb-telegraf",
+      "datasource": "gdev-influxdb1-influxql",
       "fill": 1,
       "fillGradient": 0,
       "gridPos": {
@@ -70,7 +70,7 @@
               "type": "fill"
             }
           ],
-          "measurement": "logs",
+          "measurement": "t_logs",
           "orderByTime": "ASC",
           "policy": "default",
           "refId": "A",
@@ -132,7 +132,7 @@
       }
     },
     {
-      "datasource": "gdev-influxdb-telegraf",
+      "datasource": "gdev-influxdb1-influxql",
       "gridPos": {
         "h": 18,
         "w": 24,
@@ -149,7 +149,7 @@
       "targets": [
         {
           "groupBy": [],
-          "measurement": "logs",
+          "measurement": "t_logs",
           "orderByTime": "ASC",
           "policy": "default",
           "refId": "A",

devenv/dev-dashboards/datasource-influxdb/influxdb-templated.json

@@ -27,7 +27,7 @@
       "bars": false,
       "dashLength": 10,
       "dashes": false,
-      "datasource": "gdev-influxdb",
+      "datasource": "gdev-influxdb1-influxql",
       "editable": true,
       "error": false,
       "fill": 2,
@@ -167,7 +167,7 @@
           "text": "America",
           "value": "America"
         },
-        "datasource": "gdev-influxdb",
+        "datasource": "gdev-influxdb1-influxql",
         "definition": "",
         "hide": 0,
         "includeAll": false,
@@ -193,7 +193,7 @@
           "text": "All",
           "value": "$__all"
         },
-        "datasource": "gdev-influxdb",
+        "datasource": "gdev-influxdb1-influxql",
         "definition": "",
         "hide": 0,
         "includeAll": true,

devenv/docker/blocks/influxdb/.env

@@ -1 +0,0 @@
-influxdb_version=1.8.1-alpine

devenv/docker/blocks/influxdb/docker-compose.yaml

@@ -1,31 +1,35 @@
   influxdb:
-    image: influxdb:${influxdb_version}
+    image: influxdb:latest
     container_name: influxdb
     ports:
-      - '2004:2004'
-      - '8083:8083'
       - '8086:8086'
     environment:
-      INFLUXDB_ADMIN_USER: grafana
+      INFLUXD_REPORTING_DISABLED: 'true'
-      INFLUXDB_ADMIN_PASSWORD: grafana
-      INFLUXDB_DB: site
     volumes:
-      - ./docker/blocks/influxdb/influxdb.conf:/etc/influxdb/influxdb.conf
+      - ./docker/blocks/influxdb/config.yaml:/etc/influxdb2/config.yaml
 
-  fake-influxdb-data:
+  # Use the influx cli to set up an influxdb instance.
-    image: grafana/fake-data-gen
+  influxdb_cli:
     links:
       - influxdb
-    environment:
+    image: influxdb:2.0.4
-      FD_DATASOURCE: influxdb
+    # The following long command does 2 things:
-      FD_PORT: 8086
+    #   1. It initializes the bucket
+    #   2. Maps bucket to database to enable backward-compatible access (influxql queries)
+    # Use these same configurations parameters in your telegraf configuration, mytelegraf.conf.
+    entrypoint: bash -c "influx setup --bucket mybucket -t mytoken -o myorg --username=grafana --password=grafana12345 --host=http://influxdb:8086 -f && influx -t mytoken -o myorg --host=http://influxdb:8086 bucket list -n mybucket --hide-headers | cut -f 1 | xargs influx -t mytoken -o myorg --host=http://influxdb:8086 v1 dbrp create --db mybucket --rp default --default --bucket-id"
+      # Wait for the influxd service in the influxdb container has fully bootstrapped before trying to setup an influxdb instance with the influxdb_cli service.
+    restart: on-failure:10
+    depends_on:
+      - influxdb
 
   telegraf:
     image: telegraf:1.10.4
     links:
       - influxdb
+    depends_on:
+      - influxdb_cli
     volumes:
       - ./docker/blocks/influxdb/telegraf.conf:/etc/telegraf/telegraf.conf:ro
       - /var/log:/var/log
       - ../data/log:/var/log/grafana
-

devenv/docker/blocks/influxdb1/docker-compose.yaml

@@ -0,0 +1,32 @@
+  influxdb1:
+    image: influxdb:1.8.4-alpine
+    container_name: influxdb1
+    ports:
+      - '2004:2004'
+      - '8083:8083'
+      - '8087:8086'
+    environment:
+      INFLUXDB_ADMIN_USER: grafana
+      INFLUXDB_ADMIN_PASSWORD: grafana
+      INFLUXDB_DB: site
+    volumes:
+      - ./docker/blocks/influxdb1/influxdb.conf:/etc/influxdb/influxdb.conf
+
+  fake-influxdb1-data:
+    image: grafana/fake-data-gen
+    links:
+      - influxdb1
+    environment:
+      FD_SERVER: influxdb1
+      FD_DATASOURCE: influxdb
+      FD_PORT: 8086
+
+  telegraf-influxdb1:
+    image: telegraf:1.10.4
+    links:
+      - influxdb1
+    volumes:
+      - ./docker/blocks/influxdb1/telegraf.conf:/etc/telegraf/telegraf.conf:ro
+      - /var/log:/var/log
+      - ../data/log:/var/log/grafana
+

devenv/docker/blocks/influxdb1/telegraf.conf

@@ -0,0 +1,60 @@
+[global_tags]
+
+[agent]
+  interval = "10s"
+  round_interval = true
+  metric_batch_size = 1000
+  metric_buffer_limit = 10000
+  collection_jitter = "0s"
+  flush_interval = "10s"
+  flush_jitter = "0s"
+  precision = ""
+  debug = false
+  quiet = false
+  logfile = ""
+  hostname = ""
+  omit_hostname = false
+
+[[outputs.influxdb]]
+  urls = ["http://influxdb1:8086"]
+  username = "grafana"
+  password = "grafana"
+  database = "site"
+
+[[inputs.cpu]]
+  percpu = true
+  totalcpu = true
+  collect_cpu_time = false
+  report_active = false
+  name_prefix = "t_"
+
+[[inputs.disk]]
+  ignore_fs = ["tmpfs", "devtmpfs", "devfs", "overlay", "aufs", "squashfs"]
+  name_prefix = "t_"
+
+[[inputs.diskio]]
+  name_prefix = "t_"
+
+[[inputs.kernel]]
+  name_prefix = "t_"
+
+[[inputs.mem]]
+  name_prefix = "t_"
+
+[[inputs.processes]]
+  name_prefix = "t_"
+
+[[inputs.swap]]
+  name_prefix = "t_"
+
+[[inputs.system]]
+  name_prefix = "t_"
+
+[[inputs.logparser]]
+  files = [
+    "/var/log/*.log",
+    "/var/log/grafana/*.log"
+  ]
+  [inputs.logparser.grok]
+    measurement = "t_logs"
+    patterns = ['^%{GREEDYDATA:message:string}']

devenv/docker/blocks/influxdb2/docker-compose.yaml

@@ -1,44 +0,0 @@
-  influxdb2:
-    image: influxdb:2.0.4
-    container_name: influxdb2
-    ports:
-      - '8087:8086'
-    environment:
-      INFLUXD_REPORTING_DISABLED: 'true'
-    volumes:
-      - ./docker/blocks/influxdb2/config.yaml:/etc/influxdb2/config.yaml
-
-  # Use the influx cli to set up an influxdb instance.
-  influxdb2_cli:
-    links:
-      - influxdb2
-    image: influxdb:2.0.4
-    # The following long command does 2 things:
-    #   1. It initializes the bucket
-    #   2. Maps bucket to database to enable backward-compatible access (influxql queries)
-    # Use these same configurations parameters in your telegraf configuration, mytelegraf.conf.
-    entrypoint: bash -c "influx setup --bucket mybucket -t mytoken -o myorg --username=grafana --password=grafana12345 --host=http://influxdb2:8086 -f && influx -t mytoken -o myorg --host=http://influxdb2:8086 bucket list -n mybucket --hide-headers | cut -f 1 | xargs influx -t mytoken -o myorg --host=http://influxdb2:8086 v1 dbrp create --db site --rp default --default --bucket-id"
-      # Wait for the influxd service in the influxdb container has fully bootstrapped before trying to setup an influxdb instance with the influxdb_cli service.
-    restart: on-failure:10
-    depends_on:
-      - influxdb2
-
-  fake-influxdb2-data:
-    image: grafana/fake-data-gen
-    links:
-      - influxdb2
-    environment:
-      FD_SERVER: influxdb2
-      FD_DATASOURCE: influxdb
-      FD_PORT: 8086
-
-  telegraf2:
-    image: telegraf
-    links:
-      - influxdb2
-    depends_on:
-      - influxdb2_cli
-    volumes:
-      - ./docker/blocks/influxdb2/telegraf.conf:/etc/telegraf/telegraf.conf:ro
-      - /var/log:/var/log
-      - ../data/log:/var/log/grafana