mirror of https://github.com/grafana/grafana
Plugins: Migrate Elasticsearch to backend plugin SDK (#36132)
* Migrate Elasticsearch to backend plugin SDK * Fix linting * Move away from Convey! * Rebase commit * Small logger fix * Fixes according to reviewer's comments * Fixes according to reviewer's comments * Fixes according to reviewer's comments * More cleanup * Move things around - small refactoring * Fix typo * Update calculator - add tests * Fixes according to reviewer's commentspull/36809/head
Dimitris Sotirakis
5 years ago
committed by
GitHub
parent
75947da527
commit
0df1b33d71
@ -0,0 +1,222 @@ |
||||
package tsdb |
||||
|
||||
import ( |
||||
"fmt" |
||||
"regexp" |
||||
"strings" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/grafana/grafana/pkg/tsdb/interval" |
||||
) |
||||
|
||||
var ( |
||||
defaultRes int64 = 1500 |
||||
defaultMinInterval = time.Millisecond * 1 |
||||
year = time.Hour * 24 * 365 |
||||
day = time.Hour * 24 |
||||
) |
||||
|
||||
type Interval struct { |
||||
Text string |
||||
Value time.Duration |
||||
} |
||||
|
||||
type intervalCalculator struct { |
||||
minInterval time.Duration |
||||
} |
||||
|
||||
type Calculator interface { |
||||
Calculate(timerange backend.TimeRange, minInterval time.Duration) Interval |
||||
} |
||||
|
||||
type CalculatorOptions struct { |
||||
MinInterval time.Duration |
||||
} |
||||
|
||||
func NewCalculator(opts ...CalculatorOptions) *intervalCalculator { |
||||
calc := &intervalCalculator{} |
||||
|
||||
for _, o := range opts { |
||||
if o.MinInterval == 0 { |
||||
calc.minInterval = defaultMinInterval |
||||
} else { |
||||
calc.minInterval = o.MinInterval |
||||
} |
||||
} |
||||
|
||||
return calc |
||||
} |
||||
|
||||
func (i *Interval) Milliseconds() int64 { |
||||
return i.Value.Nanoseconds() / int64(time.Millisecond) |
||||
} |
||||
|
||||
func (ic *intervalCalculator) Calculate(timerange backend.TimeRange, minInterval time.Duration) Interval { |
||||
to := timerange.To.UnixNano() |
||||
from := timerange.From.UnixNano() |
||||
intrvl := time.Duration((to - from) / defaultRes) |
||||
|
||||
if intrvl < minInterval { |
||||
return Interval{Text: interval.FormatDuration(minInterval), Value: minInterval} |
||||
} |
||||
|
||||
rounded := roundInterval(intrvl) |
||||
return Interval{Text: interval.FormatDuration(rounded), Value: rounded} |
||||
} |
||||
|
||||
// GetIntervalFrom returns the minimum interval.
|
||||
// dsInterval is the string representation of data source min interval, if configured.
|
||||
// queryInterval is the string representation of query interval (min interval), e.g. "10ms" or "10s".
|
||||
// queryIntervalMS is a pre-calculated numeric representation of the query interval in milliseconds.
|
||||
func GetIntervalFrom(dsInterval, queryInterval string, queryIntervalMS int64, defaultInterval time.Duration) (time.Duration, error) { |
||||
if queryInterval == "" { |
||||
if queryIntervalMS != 0 { |
||||
return time.Duration(queryIntervalMS) * time.Millisecond, nil |
||||
} |
||||
} |
||||
interval := queryInterval |
||||
if queryInterval == "" && dsInterval != "" { |
||||
interval = dsInterval |
||||
} |
||||
if interval == "" { |
||||
return defaultInterval, nil |
||||
} |
||||
interval = strings.Replace(strings.Replace(interval, "<", "", 1), ">", "", 1) |
||||
isPureNum, err := regexp.MatchString(`^\d+$`, interval) |
||||
if err != nil { |
||||
return time.Duration(0), err |
||||
} |
||||
if isPureNum { |
||||
interval += "s" |
||||
} |
||||
parsedInterval, err := time.ParseDuration(interval) |
||||
if err != nil { |
||||
return time.Duration(0), err |
||||
} |
||||
return parsedInterval, nil |
||||
} |
||||
|
||||
// FormatDuration converts a duration into the kbn format e.g. 1m 2h or 3d
|
||||
func FormatDuration(inter time.Duration) string { |
||||
if inter >= year { |
||||
return fmt.Sprintf("%dy", inter/year) |
||||
} |
||||
|
||||
if inter >= day { |
||||
return fmt.Sprintf("%dd", inter/day) |
||||
} |
||||
|
||||
if inter >= time.Hour { |
||||
return fmt.Sprintf("%dh", inter/time.Hour) |
||||
} |
||||
|
||||
if inter >= time.Minute { |
||||
return fmt.Sprintf("%dm", inter/time.Minute) |
||||
} |
||||
|
||||
if inter >= time.Second { |
||||
return fmt.Sprintf("%ds", inter/time.Second) |
||||
} |
||||
|
||||
if inter >= time.Millisecond { |
||||
return fmt.Sprintf("%dms", inter/time.Millisecond) |
||||
} |
||||
|
||||
return "1ms" |
||||
} |
||||
|
||||
//nolint: gocyclo
|
||||
func roundInterval(interval time.Duration) time.Duration { |
||||
switch { |
||||
// 0.015s
|
||||
case interval <= 15*time.Millisecond: |
||||
return time.Millisecond * 10 // 0.01s
|
||||
// 0.035s
|
||||
case interval <= 35*time.Millisecond: |
||||
return time.Millisecond * 20 // 0.02s
|
||||
// 0.075s
|
||||
case interval <= 75*time.Millisecond: |
||||
return time.Millisecond * 50 // 0.05s
|
||||
// 0.15s
|
||||
case interval <= 150*time.Millisecond: |
||||
return time.Millisecond * 100 // 0.1s
|
||||
// 0.35s
|
||||
case interval <= 350*time.Millisecond: |
||||
return time.Millisecond * 200 // 0.2s
|
||||
// 0.75s
|
||||
case interval <= 750*time.Millisecond: |
||||
return time.Millisecond * 500 // 0.5s
|
||||
// 1.5s
|
||||
case interval <= 1500*time.Millisecond: |
||||
return time.Millisecond * 1000 // 1s
|
||||
// 3.5s
|
||||
case interval <= 3500*time.Millisecond: |
||||
return time.Millisecond * 2000 // 2s
|
||||
// 7.5s
|
||||
case interval <= 7500*time.Millisecond: |
||||
return time.Millisecond * 5000 // 5s
|
||||
// 12.5s
|
||||
case interval <= 12500*time.Millisecond: |
||||
return time.Millisecond * 10000 // 10s
|
||||
// 17.5s
|
||||
case interval <= 17500*time.Millisecond: |
||||
return time.Millisecond * 15000 // 15s
|
||||
// 25s
|
||||
case interval <= 25000*time.Millisecond: |
||||
return time.Millisecond * 20000 // 20s
|
||||
// 45s
|
||||
case interval <= 45000*time.Millisecond: |
||||
return time.Millisecond * 30000 // 30s
|
||||
// 1.5m
|
||||
case interval <= 90000*time.Millisecond: |
||||
return time.Millisecond * 60000 // 1m
|
||||
// 3.5m
|
||||
case interval <= 210000*time.Millisecond: |
||||
return time.Millisecond * 120000 // 2m
|
||||
// 7.5m
|
||||
case interval <= 450000*time.Millisecond: |
||||
return time.Millisecond * 300000 // 5m
|
||||
// 12.5m
|
||||
case interval <= 750000*time.Millisecond: |
||||
return time.Millisecond * 600000 // 10m
|
||||
// 12.5m
|
||||
case interval <= 1050000*time.Millisecond: |
||||
return time.Millisecond * 900000 // 15m
|
||||
// 25m
|
||||
case interval <= 1500000*time.Millisecond: |
||||
return time.Millisecond * 1200000 // 20m
|
||||
// 45m
|
||||
case interval <= 2700000*time.Millisecond: |
||||
return time.Millisecond * 1800000 // 30m
|
||||
// 1.5h
|
||||
case interval <= 5400000*time.Millisecond: |
||||
return time.Millisecond * 3600000 // 1h
|
||||
// 2.5h
|
||||
case interval <= 9000000*time.Millisecond: |
||||
return time.Millisecond * 7200000 // 2h
|
||||
// 4.5h
|
||||
case interval <= 16200000*time.Millisecond: |
||||
return time.Millisecond * 10800000 // 3h
|
||||
// 9h
|
||||
case interval <= 32400000*time.Millisecond: |
||||
return time.Millisecond * 21600000 // 6h
|
||||
// 24h
|
||||
case interval <= 86400000*time.Millisecond: |
||||
return time.Millisecond * 43200000 // 12h
|
||||
// 48h
|
||||
case interval <= 172800000*time.Millisecond: |
||||
return time.Millisecond * 86400000 // 24h
|
||||
// 1w
|
||||
case interval <= 604800000*time.Millisecond: |
||||
return time.Millisecond * 86400000 // 24h
|
||||
// 3w
|
||||
case interval <= 1814400000*time.Millisecond: |
||||
return time.Millisecond * 604800000 // 1w
|
||||
// 2y
|
||||
case interval < 3628800000*time.Millisecond: |
||||
return time.Millisecond * 2592000000 // 30d
|
||||
default: |
||||
return time.Millisecond * 31536000000 // 1y
|
||||
} |
||||
} |
||||
@ -0,0 +1,98 @@ |
||||
package tsdb |
||||
|
||||
import ( |
||||
"testing" |
||||
"time" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/grafana/grafana/pkg/models" |
||||
"github.com/stretchr/testify/assert" |
||||
) |
||||
|
||||
func TestIntervalCalculator_Calculate(t *testing.T) { |
||||
calculator := NewCalculator(CalculatorOptions{}) |
||||
|
||||
timeNow := time.Now() |
||||
|
||||
testCases := []struct { |
||||
name string |
||||
timeRange backend.TimeRange |
||||
expected string |
||||
}{ |
||||
{"from 5m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(5 * time.Minute)}, "200ms"}, |
||||
{"from 15m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(15 * time.Minute)}, "500ms"}, |
||||
{"from 30m to now", backend.TimeRange{From: timeNow, To: timeNow.Add(30 * time.Minute)}, "1s"}, |
||||
{"from 1h to now", backend.TimeRange{From: timeNow, To: timeNow.Add(60 * time.Minute)}, "2s"}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
interval := calculator.Calculate(tc.timeRange, time.Millisecond*1) |
||||
assert.Equal(t, tc.expected, interval.Text) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestRoundInterval(t *testing.T) { |
||||
testCases := []struct { |
||||
name string |
||||
interval time.Duration |
||||
expected time.Duration |
||||
}{ |
||||
{"30ms", time.Millisecond * 30, time.Millisecond * 20}, |
||||
{"45ms", time.Millisecond * 45, time.Millisecond * 50}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
assert.Equal(t, tc.expected, roundInterval(tc.interval)) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestFormatDuration(t *testing.T) { |
||||
testCases := []struct { |
||||
name string |
||||
duration time.Duration |
||||
expected string |
||||
}{ |
||||
{"61s", time.Second * 61, "1m"}, |
||||
{"30ms", time.Millisecond * 30, "30ms"}, |
||||
{"23h", time.Hour * 23, "23h"}, |
||||
{"24h", time.Hour * 24, "1d"}, |
||||
{"367d", time.Hour * 24 * 367, "1y"}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
assert.Equal(t, tc.expected, FormatDuration(tc.duration)) |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func TestGetIntervalFrom(t *testing.T) { |
||||
testCases := []struct { |
||||
name string |
||||
dsInfo *models.DataSource |
||||
queryInterval string |
||||
queryIntervalMs int64 |
||||
defaultInterval time.Duration |
||||
expected time.Duration |
||||
}{ |
||||
{"45s", nil, "45s", 0, time.Second * 15, time.Second * 45}, |
||||
{"45", nil, "45", 0, time.Second * 15, time.Second * 45}, |
||||
{"2m", nil, "2m", 0, time.Second * 15, time.Minute * 2}, |
||||
{"intervalMs", nil, "", 45000, time.Second * 15, time.Second * 45}, |
||||
{"intervalMs sub-seconds", nil, "", 45200, time.Second * 15, time.Millisecond * 45200}, |
||||
{"defaultInterval when interval empty", nil, "", 0, time.Second * 15, time.Second * 15}, |
||||
{"defaultInterval when intervalMs 0", nil, "", 0, time.Second * 15, time.Second * 15}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
t.Run(tc.name, func(t *testing.T) { |
||||
actual, err := GetIntervalFrom(tc.queryInterval, "", tc.queryIntervalMs, tc.defaultInterval) |
||||
assert.Nil(t, err) |
||||
assert.Equal(t, tc.expected, actual) |
||||
}) |
||||
} |
||||
} |
||||
@ -0,0 +1,191 @@ |
||||
package elasticsearch |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"testing" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
type datasourceInfo struct { |
||||
ESVersion interface{} `json:"esVersion"` |
||||
TimeField interface{} `json:"timeField"` |
||||
MaxConcurrentShardRequests int64 `json:"maxConcurrentShardRequests"` |
||||
Interval string `json:"interval"` |
||||
TimeInterval string `json:"timeInterval"` |
||||
} |
||||
|
||||
func TestCoerceVersion(t *testing.T) { |
||||
t.Run("version is string", func(t *testing.T) { |
||||
ver := "7.0.0" |
||||
smvr, err := coerceVersion(ver) |
||||
require.NoError(t, err) |
||||
require.NotNil(t, smvr) |
||||
require.Equal(t, "7.0.0", smvr.String()) |
||||
}) |
||||
|
||||
t.Run("version is int", func(t *testing.T) { |
||||
testCases := []struct { |
||||
intVersion float64 |
||||
stringVersion string |
||||
}{ |
||||
{intVersion: 2, stringVersion: "2.0.0"}, |
||||
{intVersion: 5, stringVersion: "5.0.0"}, |
||||
{intVersion: 56, stringVersion: "5.6.0"}, |
||||
{intVersion: 60, stringVersion: "6.0.0"}, |
||||
{intVersion: 70, stringVersion: "7.0.0"}, |
||||
} |
||||
|
||||
for _, tc := range testCases { |
||||
smvr, err := coerceVersion(tc.intVersion) |
||||
require.NoError(t, err) |
||||
require.Equal(t, tc.stringVersion, smvr.String()) |
||||
} |
||||
|
||||
smvr, err := coerceVersion(12345) |
||||
require.Error(t, err) |
||||
require.Nil(t, smvr) |
||||
}) |
||||
} |
||||
|
||||
func TestNewInstanceSettings(t *testing.T) { |
||||
t.Run("fields exist", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: "7.0.0", |
||||
TimeField: "@timestamp", |
||||
MaxConcurrentShardRequests: 5, |
||||
} |
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.NoError(t, err) |
||||
}) |
||||
|
||||
t.Run("esVersion", func(t *testing.T) { |
||||
t.Run("correct version", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: 5, |
||||
TimeField: "@timestamp", |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.NoError(t, err) |
||||
}) |
||||
|
||||
t.Run("faulty version int", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: 1234, |
||||
TimeField: "@timestamp", |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.EqualError(t, err, "elasticsearch version is required, err=elasticsearch version=1234 is not supported") |
||||
}) |
||||
|
||||
t.Run("faulty version string", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: "NOT_VALID", |
||||
TimeField: "@timestamp", |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.EqualError(t, err, "elasticsearch version is required, err=Invalid Semantic Version") |
||||
}) |
||||
|
||||
t.Run("no version", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
TimeField: "@timestamp", |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.EqualError(t, err, "elasticsearch version is required, err=elasticsearch version <nil>, cannot be cast to int") |
||||
}) |
||||
}) |
||||
|
||||
t.Run("timeField", func(t *testing.T) { |
||||
t.Run("is nil", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: 2, |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.EqualError(t, err, "timeField cannot be cast to string") |
||||
}) |
||||
|
||||
t.Run("is empty", func(t *testing.T) { |
||||
dsInfo := datasourceInfo{ |
||||
ESVersion: 2, |
||||
MaxConcurrentShardRequests: 5, |
||||
Interval: "Daily", |
||||
TimeField: "", |
||||
TimeInterval: "TimeInterval", |
||||
} |
||||
|
||||
settingsJSON, err := json.Marshal(dsInfo) |
||||
require.NoError(t, err) |
||||
|
||||
dsSettings := backend.DataSourceInstanceSettings{ |
||||
JSONData: json.RawMessage(settingsJSON), |
||||
} |
||||
|
||||
_, err = newInstanceSettings()(dsSettings) |
||||
require.EqualError(t, err, "elasticsearch time field name is required") |
||||
}) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue