Elastic: devenv: improved devenv-image (#49002)

* devenv: elastic: improved config

* better ascii-codes

devenv/datasources.yaml

@@ -114,6 +114,18 @@ datasources:
       tsdbResolution: 1
       tsdbVersion: 3
 
+  - name: gdev-elasticsearch
+    type: elasticsearch
+    access: proxy
+    database: "[logs-]YYYY.MM.DD"
+    url: http://localhost:9200
+    jsonData:
+      interval: Daily
+      timeField: "@timestamp"
+      logLevelField: level
+      logMessageField: line
+      esVersion: 8.0.0
+
   - name: gdev-elasticsearch-v7-metrics
     type: elasticsearch
     access: proxy

devenv/docker/blocks/elastic/.env

@@ -0,0 +1 @@
+elastic_version=8.2.0

devenv/docker/blocks/elastic/data/Dockerfile

@@ -0,0 +1,3 @@
+FROM node:16-alpine
+
+COPY data.js /home/node/data.js

devenv/docker/blocks/elastic/data/data.js

@@ -0,0 +1,137 @@
+const http = require('http');
+
+if (process.argv.length !== 3) {
+  throw new Error('invalid command line: use node sendLogs.js ELASTIC_BASE_URL');
+}
+
+const ELASTIC_BASE_URL = process.argv[2];
+
+// helper function, do a http request
+async function jsonRequest(data, method, url, expectedStatusCode) {
+  return new Promise((resolve, reject) => {
+    const req = http.request(
+      {
+        protocol: url.protocol,
+        host: url.hostname,
+        port: url.port,
+        path: `${url.pathname}${url.search}`,
+        method,
+        headers: { 'content-type': 'application/json' },
+      },
+      (res) => {
+        if (res.statusCode !== expectedStatusCode) {
+          reject(new Error(`Invalid response: ${res.statusCode}`));
+        } else {
+          resolve();
+        }
+      }
+    );
+
+    req.on('error', (err) => reject(err));
+
+    req.write(JSON.stringify(data));
+    req.end();
+  });
+}
+
+// helper function, choose a random element from an array
+function chooseRandomElement(items) {
+  const index = Math.trunc(Math.random() * items.length);
+  return items[index];
+}
+
+// helper function, sleep for a duration
+async function sleep(duration) {
+  return new Promise((resolve) => {
+    setTimeout(resolve, duration);
+  });
+}
+
+async function elasticSendLogItem(timestamp, item) {
+  // we need the YYYY.MM.DD format
+  const timestampText = timestamp.toISOString().slice(0, 10).replace(/-/g, '.');
+  const url = new URL(ELASTIC_BASE_URL);
+  url.pathname = `/logs-${timestampText}/_doc`;
+  await jsonRequest(item, 'POST', url, 201);
+  console.log(`posted to ${url.toString()}`);
+}
+
+async function elasticSetupIndexTemplate() {
+  const data = {
+    index_patterns: ['logs-*'],
+    template: {
+      mappings: {
+        properties: {
+          '@timestamp': {
+            type: 'date',
+          },
+          counter: {
+            type: 'integer',
+          },
+          float: {
+            type: 'float',
+          },
+          level: {
+            type: 'keyword',
+          },
+          label: {
+            type: 'keyword',
+          },
+          location: {
+            type: 'geo_point',
+          },
+        },
+      },
+    },
+  };
+  const url = new URL(ELASTIC_BASE_URL);
+  url.pathname = '/_index_template/gdev';
+  await jsonRequest(data, 'PUT', url, 200);
+}
+
+function makeRandomPoint() {
+  const angle = Math.random() * 2 * Math.PI;
+  const x = 45 * Math.sin(angle);
+  const y = 45 * Math.cos(angle);
+  return y + ', ' + x;
+}
+
+function getRandomLogItem(counter, timestamp) {
+  const randomText = `${Math.trunc(Math.random() * 1000 * 1000 * 1000)}`;
+  const maybeAnsiText = Math.random() < 0.5 ? 'with ANSI \u001b[31mpart of the text\u001b[0m' : '';
+  return {
+    '@timestamp': timestamp.toISOString(),
+    line: `log text ${maybeAnsiText} [${randomText}]`,
+    counter: counter.toString(),
+    float: 100 * Math.random().toString(),
+    label: chooseRandomElement(['val1', 'val2', 'val3']),
+    level: chooseRandomElement(['info', 'info', 'error']),
+    // location: chooseRandomElement(LOCATIONS),
+    location: makeRandomPoint(),
+  };
+}
+
+async function main() {
+  await elasticSetupIndexTemplate();
+  const SLEEP_ANGLE_STEP = Math.PI / 200;
+  let sleepAngle = 0;
+  function getNextSineWaveSleepDuration() {
+    sleepAngle += SLEEP_ANGLE_STEP;
+    return Math.trunc(1000 * Math.abs(Math.sin(sleepAngle)));
+  }
+
+  for (let step = 0; step < 300; step++) {
+    await sleep(getNextSineWaveSleepDuration());
+    const timestamp = new Date();
+    const item = getRandomLogItem(step + 1, timestamp);
+    elasticSendLogItem(timestamp, item);
+  }
+}
+
+// when running in docker, we catch the needed stop-signal, to shutdown fast
+process.on('SIGTERM', () => {
+  console.log('shutdown requested');
+  process.exit(0);
+});
+
+main();

devenv/docker/blocks/elastic/docker-compose.yaml

@@ -1,22 +1,24 @@
 # You need to run 'sysctl -w vm.max_map_count=262144' on the host machine
 
-  elasticsearch-latest:
-    image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1
+  elasticsearch:
+    image: docker.elastic.co/elasticsearch/elasticsearch:${elastic_version}
     command: elasticsearch
     environment:
       - "discovery.type=single-node"
       - "xpack.license.self_generated.type=basic"
       - "xpack.security.enabled=false"
     ports:
-      - "14200:9200"
-      - "14300:9300"
+      - "9200:9200"
 
-  # TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged
-  # fake-elastic-latest-data:
-  #   image: grafana/fake-data-gen
-  #   links:
-  #     - elasticsearch-latest
-  #   environment:
-  #     FD_SERVER: elasticsearch-latest
-  #     FD_DATASOURCE: elasticsearch8
-  #     FD_PORT: 9200
+  data:
+    build: docker/blocks/elastic/data
+    command: node /home/node/data.js http://elasticsearch:9200
+    depends_on:
+      - elasticsearch
+    # elastic starts slowly, the first couple start of data.js
+    # might fail, so we auto-restart it on failure.
+    deploy:
+      restart_policy:
+        condition: on-failure
+        delay: 5s
+        max_attempts: 20 # should be enough