From a3c583459461609754f9532b7fd28569a7b9e894 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?G=C3=A1bor=20Farkas?= Date: Thu, 19 May 2022 08:29:23 +0200 Subject: [PATCH] Elastic: devenv: improved devenv-image (#49002) * devenv: elastic: improved config * better ascii-codes --- devenv/datasources.yaml | 12 ++ devenv/docker/blocks/elastic/.env | 1 + devenv/docker/blocks/elastic/data/Dockerfile | 3 + devenv/docker/blocks/elastic/data/data.js | 137 ++++++++++++++++++ .../docker/blocks/elastic/docker-compose.yaml | 28 ++-- 5 files changed, 168 insertions(+), 13 deletions(-) create mode 100644 devenv/docker/blocks/elastic/.env create mode 100644 devenv/docker/blocks/elastic/data/Dockerfile create mode 100644 devenv/docker/blocks/elastic/data/data.js diff --git a/devenv/datasources.yaml b/devenv/datasources.yaml index cbcf4f74f9d..a8c68ee1bb1 100644 --- a/devenv/datasources.yaml +++ b/devenv/datasources.yaml @@ -114,6 +114,18 @@ datasources: tsdbResolution: 1 tsdbVersion: 3 + - name: gdev-elasticsearch + type: elasticsearch + access: proxy + database: "[logs-]YYYY.MM.DD" + url: http://localhost:9200 + jsonData: + interval: Daily + timeField: "@timestamp" + logLevelField: level + logMessageField: line + esVersion: 8.0.0 + - name: gdev-elasticsearch-v7-metrics type: elasticsearch access: proxy diff --git a/devenv/docker/blocks/elastic/.env b/devenv/docker/blocks/elastic/.env new file mode 100644 index 00000000000..b12a09c5237 --- /dev/null +++ b/devenv/docker/blocks/elastic/.env @@ -0,0 +1 @@ +elastic_version=8.2.0 \ No newline at end of file diff --git a/devenv/docker/blocks/elastic/data/Dockerfile b/devenv/docker/blocks/elastic/data/Dockerfile new file mode 100644 index 00000000000..ea13482e4d3 --- /dev/null +++ b/devenv/docker/blocks/elastic/data/Dockerfile @@ -0,0 +1,3 @@ +FROM node:16-alpine + +COPY data.js /home/node/data.js \ No newline at end of file diff --git a/devenv/docker/blocks/elastic/data/data.js b/devenv/docker/blocks/elastic/data/data.js new file mode 100644 index 00000000000..93e5085df31 --- /dev/null +++ b/devenv/docker/blocks/elastic/data/data.js @@ -0,0 +1,137 @@ +const http = require('http'); + +if (process.argv.length !== 3) { + throw new Error('invalid command line: use node sendLogs.js ELASTIC_BASE_URL'); +} + +const ELASTIC_BASE_URL = process.argv[2]; + +// helper function, do a http request +async function jsonRequest(data, method, url, expectedStatusCode) { + return new Promise((resolve, reject) => { + const req = http.request( + { + protocol: url.protocol, + host: url.hostname, + port: url.port, + path: `${url.pathname}${url.search}`, + method, + headers: { 'content-type': 'application/json' }, + }, + (res) => { + if (res.statusCode !== expectedStatusCode) { + reject(new Error(`Invalid response: ${res.statusCode}`)); + } else { + resolve(); + } + } + ); + + req.on('error', (err) => reject(err)); + + req.write(JSON.stringify(data)); + req.end(); + }); +} + +// helper function, choose a random element from an array +function chooseRandomElement(items) { + const index = Math.trunc(Math.random() * items.length); + return items[index]; +} + +// helper function, sleep for a duration +async function sleep(duration) { + return new Promise((resolve) => { + setTimeout(resolve, duration); + }); +} + +async function elasticSendLogItem(timestamp, item) { + // we need the YYYY.MM.DD format + const timestampText = timestamp.toISOString().slice(0, 10).replace(/-/g, '.'); + const url = new URL(ELASTIC_BASE_URL); + url.pathname = `/logs-${timestampText}/_doc`; + await jsonRequest(item, 'POST', url, 201); + console.log(`posted to ${url.toString()}`); +} + +async function elasticSetupIndexTemplate() { + const data = { + index_patterns: ['logs-*'], + template: { + mappings: { + properties: { + '@timestamp': { + type: 'date', + }, + counter: { + type: 'integer', + }, + float: { + type: 'float', + }, + level: { + type: 'keyword', + }, + label: { + type: 'keyword', + }, + location: { + type: 'geo_point', + }, + }, + }, + }, + }; + const url = new URL(ELASTIC_BASE_URL); + url.pathname = '/_index_template/gdev'; + await jsonRequest(data, 'PUT', url, 200); +} + +function makeRandomPoint() { + const angle = Math.random() * 2 * Math.PI; + const x = 45 * Math.sin(angle); + const y = 45 * Math.cos(angle); + return y + ', ' + x; +} + +function getRandomLogItem(counter, timestamp) { + const randomText = `${Math.trunc(Math.random() * 1000 * 1000 * 1000)}`; + const maybeAnsiText = Math.random() < 0.5 ? 'with ANSI \u001b[31mpart of the text\u001b[0m' : ''; + return { + '@timestamp': timestamp.toISOString(), + line: `log text ${maybeAnsiText} [${randomText}]`, + counter: counter.toString(), + float: 100 * Math.random().toString(), + label: chooseRandomElement(['val1', 'val2', 'val3']), + level: chooseRandomElement(['info', 'info', 'error']), + // location: chooseRandomElement(LOCATIONS), + location: makeRandomPoint(), + }; +} + +async function main() { + await elasticSetupIndexTemplate(); + const SLEEP_ANGLE_STEP = Math.PI / 200; + let sleepAngle = 0; + function getNextSineWaveSleepDuration() { + sleepAngle += SLEEP_ANGLE_STEP; + return Math.trunc(1000 * Math.abs(Math.sin(sleepAngle))); + } + + for (let step = 0; step < 300; step++) { + await sleep(getNextSineWaveSleepDuration()); + const timestamp = new Date(); + const item = getRandomLogItem(step + 1, timestamp); + elasticSendLogItem(timestamp, item); + } +} + +// when running in docker, we catch the needed stop-signal, to shutdown fast +process.on('SIGTERM', () => { + console.log('shutdown requested'); + process.exit(0); +}); + +main(); diff --git a/devenv/docker/blocks/elastic/docker-compose.yaml b/devenv/docker/blocks/elastic/docker-compose.yaml index 78c67982814..aee8b0eae5c 100644 --- a/devenv/docker/blocks/elastic/docker-compose.yaml +++ b/devenv/docker/blocks/elastic/docker-compose.yaml @@ -1,22 +1,24 @@ # You need to run 'sysctl -w vm.max_map_count=262144' on the host machine - elasticsearch-latest: - image: docker.elastic.co/elasticsearch/elasticsearch:8.0.0-beta1 + elasticsearch: + image: docker.elastic.co/elasticsearch/elasticsearch:${elastic_version} command: elasticsearch environment: - "discovery.type=single-node" - "xpack.license.self_generated.type=basic" - "xpack.security.enabled=false" ports: - - "14200:9200" - - "14300:9300" + - "9200:9200" - # TODO: uncomment when https://github.com/grafana/fake-data-gen/pull/20 is merged - # fake-elastic-latest-data: - # image: grafana/fake-data-gen - # links: - # - elasticsearch-latest - # environment: - # FD_SERVER: elasticsearch-latest - # FD_DATASOURCE: elasticsearch8 - # FD_PORT: 9200 + data: + build: docker/blocks/elastic/data + command: node /home/node/data.js http://elasticsearch:9200 + depends_on: + - elasticsearch + # elastic starts slowly, the first couple start of data.js + # might fail, so we auto-restart it on failure. + deploy: + restart_policy: + condition: on-failure + delay: 5s + max_attempts: 20 # should be enough \ No newline at end of file