devenv: update ha test and load test

better ha setup for many mysql connections
prometheus now scrapes mysql metrics in ha setup
ha setup provisions mysql dashboard
adds configurable virtual users for load test run script

devenv/docker/ha_test/docker-compose.yaml

@@ -15,6 +15,7 @@ services:
       MYSQL_DATABASE: grafana
       MYSQL_USER: grafana
       MYSQL_PASSWORD: password
+    command: [mysqld, --character-set-server=utf8mb4, --collation-server=utf8mb4_unicode_ci, --innodb_monitor_enable=all, --max-connections=1001]
     ports:
       - 3306
     healthcheck:
@@ -22,6 +23,16 @@ services:
       timeout: 10s
       retries: 10
 
+  mysqld-exporter:
+    image: prom/mysqld-exporter
+    environment:
+      - DATA_SOURCE_NAME=root:rootpass@(db:3306)/
+    ports:
+      - 9104
+    depends_on:
+      db:
+        condition: service_healthy
+
   # db:
   #   image: postgres:9.3
   #   environment:
@@ -47,6 +58,7 @@ services:
       - GF_DATABASE_PASSWORD=password
       - GF_DATABASE_TYPE=mysql
       - GF_DATABASE_HOST=db:3306
+      - GF_DATABASE_MAX_OPEN_CONN=300
       - GF_SESSION_PROVIDER=mysql
       - GF_SESSION_PROVIDER_CONFIG=grafana:password@tcp(db:3306)/grafana?allowNativePasswords=true
       # - GF_DATABASE_TYPE=postgres
@@ -55,7 +67,7 @@ services:
       # - GF_SESSION_PROVIDER=postgres
       # - GF_SESSION_PROVIDER_CONFIG=user=grafana password=password host=db port=5432 dbname=grafana sslmode=disable
       - GF_LOG_FILTERS=alerting.notifier:debug,alerting.notifier.slack:debug,auth:debug
-      - GF_LOGIN_ROTATE_TOKEN_MINUTES=2
+      - GF_AUTH_TOKEN_ROTATION_INTERVAL_MINUTES=2
     ports:
       - 3000
     depends_on:
@@ -70,10 +82,3 @@ services:
       - VIRTUAL_HOST=prometheus.loc
     ports:
       - 9090
-
-  # mysqld-exporter:
-  #   image: prom/mysqld-exporter
-  #   environment:
-  #     - DATA_SOURCE_NAME=grafana:password@(mysql:3306)/
-  #   ports:
-  #     - 9104

devenv/docker/ha_test/grafana/provisioning/dashboards/dashboards.yaml

@@ -6,3 +6,9 @@ providers:
    type: file
    options:
      path: /etc/grafana/provisioning/dashboards/alerts
+
+ - name: 'MySQL'
+   folder: 'MySQL'
+   type: file
+   options:
+     path: /etc/grafana/provisioning/dashboards/mysql

devenv/docker/ha_test/prometheus/prometheus.yml

@@ -30,10 +30,10 @@ scrape_configs:
         port: 3000
         refresh_interval: 10s
 
-  # - job_name: 'mysql'
-  #   dns_sd_configs:
-  #     - names:
-  #       - 'mysqld-exporter'
-  #       type: 'A'
-  #       port: 9104
-  #       refresh_interval: 10s
+  - job_name: 'mysql'
+    dns_sd_configs:
+      - names:
+        - 'mysqld-exporter'
+        type: 'A'
+        port: 9104
+        refresh_interval: 10s

devenv/docker/loadtest/README.md

@@ -8,7 +8,7 @@ Docker
 
 ## Run
 
-Run load test for 15 minutes:
+Run load test for 15 minutes using 2 virtual users and targeting http://localhost:3000.
 
 ```bash
 $ ./run.sh
@@ -20,6 +20,18 @@ Run load test for custom duration:
 $ ./run.sh -d 10s
 ```
 
+Run load test for custom target url:
+
+```bash
+$ ./run.sh -u http://grafana.loc
+```
+
+Run load test for 10 virtual users:
+
+```bash
+$ ./run.sh -v 10
+```
+
 Example output:
 
 ```bash

devenv/docker/loadtest/auth_token_test.js

@@ -65,7 +65,7 @@ export default (data) => {
     }
   });
 
-  sleep(1)
+  sleep(5)
 }
 
 export const teardown = (data) => {}

devenv/docker/loadtest/run.sh

@@ -5,8 +5,9 @@ PWD=$(pwd)
 run() {
   duration='15m'
   url='http://localhost:3000'
+  vus='2'
 
-  while getopts ":d:u:" o; do
+  while getopts ":d:u:v:" o; do
     case "${o}" in
 				d)
             duration=${OPTARG}
@@ -14,11 +15,14 @@ run() {
         u)
             url=${OPTARG}
             ;;
+        v)
+            vus=${OPTARG}
+            ;;
     esac
 	done
 	shift $((OPTIND-1))
 
-  docker run -t --network=host -v $PWD:/src -e URL=$url --rm -i loadimpact/k6:master run --vus 2 --duration $duration src/auth_token_test.js
+  docker run -t --network=host -v $PWD:/src -e URL=$url --rm -i loadimpact/k6:master run --vus $vus --duration $duration src/auth_token_test.js
 }
 
 run "$@"