From b69ebee066ae3eea79a79213a2360e55be837726 Mon Sep 17 00:00:00 2001 From: Marcus Efraimsson Date: Thu, 22 Mar 2018 15:23:12 +0100 Subject: [PATCH] mssql: fix precision for the time column in table/annotation query mode Use the ConvertSqlTimeColumnToEpochMs function to convert any native datetime data type or epoch time (millisecond precision). Additional tests and update of existing due to timezone issues running MSSQL on UTC and dev environment on non-utc. Update stored procedures test to handle more parameters. Update test dashboard. --- docker/blocks/mssql_tests/dashboard.json | 380 +++++++++++++++--- pkg/tsdb/mssql/mssql.go | 13 +- pkg/tsdb/mssql/mssql_test.go | 283 ++++++++++--- .../mssql/partials/annotations.editor.html | 2 +- .../datasource/mssql/response_parser.ts | 2 +- 5 files changed, 547 insertions(+), 133 deletions(-) diff --git a/docker/blocks/mssql_tests/dashboard.json b/docker/blocks/mssql_tests/dashboard.json index 323a61bb49a..20e3907b48b 100644 --- a/docker/blocks/mssql_tests/dashboard.json +++ b/docker/blocks/mssql_tests/dashboard.json @@ -53,7 +53,7 @@ "iconColor": "#6ed0e0", "limit": 100, "name": "Deploys", - "rawQuery": "SELECT\n time_sec as time,\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='deploy'\n ORDER BY 1 ASC\n ", + "rawQuery": "SELECT\n $__time(time_sec),\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='deploy'\n ORDER BY 1 ASC\n ", "showIn": 0, "tags": [], "type": "tags" @@ -65,7 +65,7 @@ "iconColor": "rgba(255, 96, 96, 1)", "limit": 100, "name": "Tickets", - "rawQuery": "SELECT\n time_sec as time,\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='ticket'\n ORDER BY 1 ASC\n ", + "rawQuery": "SELECT\n $__time(time_sec),\n description as [text],\n tags\n FROM [event]\n WHERE $__unixEpochFilter(time_sec) AND tags='ticket'\n ORDER BY 1 ASC\n ", "showIn": 0, "tags": [], "type": "tags" @@ -76,8 +76,20 @@ "hide": false, "iconColor": "#7eb26d", "limit": 100, - "name": "Metric Values", - "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nORDER BY 1", + "name": "Metric Values timeEpoch macro", + "rawQuery": "SELECT \n $__timeEpoch(time), \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", + "showIn": 0, + "tags": [], + "type": "tags" + }, + { + "datasource": "${DS_MSSQL_TEST}", + "enable": false, + "hide": false, + "iconColor": "#1f78c1", + "limit": 100, + "name": "Metric Values native time", + "rawQuery": "SELECT \n time, \n measurement as text, \n '' as tags\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nORDER BY 1", "showIn": 0, "tags": [], "type": "tags" @@ -88,7 +100,7 @@ "gnetId": null, "graphTooltip": 0, "id": null, - "iteration": 1521481503341, + "iteration": 1521715844826, "links": [], "panels": [ { @@ -138,6 +150,222 @@ "transform": "table", "type": "table" }, + { + "columns": [], + "datasource": "${DS_MSSQL_TEST}", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 0, + "y": 4 + }, + "id": 32, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as bigint) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as bigint) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "${DS_MSSQL_TEST}", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 6, + "y": 4 + }, + "id": 33, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT cast(null as datetime) as time", + "refId": "A", + "target": "" + } + ], + "title": "cast(null as datetime) as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "${DS_MSSQL_TEST}", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 12, + "y": 4 + }, + "id": 34, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT GETDATE() as time", + "refId": "A", + "target": "" + } + ], + "title": "GETDATE() as time", + "transform": "table", + "type": "table" + }, + { + "columns": [], + "datasource": "${DS_MSSQL_TEST}", + "fontSize": "100%", + "gridPos": { + "h": 3, + "w": 6, + "x": 18, + "y": 4 + }, + "id": 35, + "links": [], + "pageSize": null, + "scroll": true, + "showHeader": true, + "sort": { + "col": 0, + "desc": true + }, + "styles": [ + { + "alias": "Time", + "dateFormat": "YYYY-MM-DD HH:mm:ss", + "pattern": "time", + "type": "date" + }, + { + "alias": "", + "colorMode": null, + "colors": [ + "rgba(245, 54, 54, 0.9)", + "rgba(237, 129, 40, 0.89)", + "rgba(50, 172, 45, 0.97)" + ], + "decimals": 2, + "pattern": "/.*/", + "thresholds": [], + "type": "number", + "unit": "short" + } + ], + "targets": [ + { + "alias": "", + "format": "table", + "rawSql": "SELECT GETUTCDATE() as time", + "refId": "A", + "target": "" + } + ], + "title": "GETUTCDATE() as time", + "transform": "table", + "type": "table" + }, { "aliasColors": {}, "bars": false, @@ -149,7 +377,7 @@ "h": 9, "w": 8, "x": 0, - "y": 4 + "y": 7 }, "id": 7, "legend": { @@ -228,7 +456,7 @@ "h": 9, "w": 8, "x": 8, - "y": 4 + "y": 7 }, "id": 9, "legend": { @@ -307,7 +535,7 @@ "h": 9, "w": 8, "x": 16, - "y": 4 + "y": 7 }, "id": 10, "legend": { @@ -386,7 +614,7 @@ "h": 9, "w": 8, "x": 0, - "y": 13 + "y": 16 }, "id": 16, "legend": { @@ -465,7 +693,7 @@ "h": 9, "w": 8, "x": 8, - "y": 13 + "y": 16 }, "id": 12, "legend": { @@ -544,7 +772,7 @@ "h": 9, "w": 8, "x": 16, - "y": 13 + "y": 16 }, "id": 13, "legend": { @@ -623,7 +851,7 @@ "h": 8, "w": 12, "x": 0, - "y": 22 + "y": 25 }, "id": 27, "legend": { @@ -655,13 +883,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value one' as metric, \n avg(valueOne) as valueOne\nFROM\n metric_values \nWHERE\n $__timeFilter(time)\nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", + "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value one' as metric, \n avg(valueOne) as valueOne\nFROM\n metric_values \nWHERE\n $__timeFilter(time) AND\n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value two' as metric, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", + "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n measurement + ' - value two' as metric, \n avg(valueTwo) as valueTwo \nFROM\n metric_values\nWHERE\n $__timeFilter(time) AND\n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__timeGroup(time, '$summarize'), \n measurement \nORDER BY 1", "refId": "B" } ], @@ -712,7 +940,7 @@ "h": 8, "w": 12, "x": 12, - "y": 22 + "y": 25 }, "id": 5, "legend": { @@ -734,7 +962,19 @@ "pointradius": 3, "points": false, "renderer": "flot", - "seriesOverrides": [], + "seriesOverrides": [ + { + "alias": "MovingAverageValueOne", + "dashes": true, + "lines": false + }, + { + "alias": "MovingAverageValueTwo", + "dashes": true, + "lines": false, + "yaxis": 1 + } + ], "spaceLength": 10, "stack": false, "steppedLine": false, @@ -742,8 +982,14 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n avg(valueOne) as valueOne, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nGROUP BY \n $__timeGroup(time, '$summarize')\nORDER BY 1", + "rawSql": "SELECT \n $__timeGroup(time, '$summarize') as time, \n avg(valueOne) as valueOne, \n avg(valueTwo) as valueTwo \nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n ($metric = 'ALL' OR measurement = $metric)\nGROUP BY \n $__timeGroup(time, '$summarize')\nORDER BY 1", "refId": "A" + }, + { + "alias": "", + "format": "time_series", + "rawSql": "SELECT \n time,\n avg(valueOne) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueOne,\n avg(valueTwo) OVER (ORDER BY time ROWS BETWEEN 6 PRECEDING AND 6 FOLLOWING) as MovingAverageValueTwo\nFROM\n metric_values \nWHERE \n $__timeFilter(time) AND \n ($metric = 'ALL' OR measurement = $metric)\nORDER BY 1", + "refId": "B" } ], "thresholds": [], @@ -793,7 +1039,7 @@ "h": 8, "w": 12, "x": 0, - "y": 30 + "y": 33 }, "id": 4, "legend": { @@ -825,13 +1071,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -882,7 +1128,7 @@ "h": 8, "w": 12, "x": 12, - "y": 30 + "y": 33 }, "id": 28, "legend": { @@ -963,7 +1209,7 @@ "h": 8, "w": 12, "x": 0, - "y": 38 + "y": 41 }, "id": 19, "legend": { @@ -995,13 +1241,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -1052,7 +1298,7 @@ "h": 8, "w": 12, "x": 12, - "y": 38 + "y": 41 }, "id": 18, "legend": { @@ -1082,7 +1328,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -1133,7 +1379,7 @@ "h": 8, "w": 12, "x": 0, - "y": 46 + "y": 49 }, "id": 17, "legend": { @@ -1165,13 +1411,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values WHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -1222,7 +1468,7 @@ "h": 8, "w": 12, "x": 12, - "y": 46 + "y": 49 }, "id": 20, "legend": { @@ -1252,7 +1498,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -1303,7 +1549,7 @@ "h": 8, "w": 12, "x": 0, - "y": 54 + "y": 57 }, "id": 29, "legend": { @@ -1335,7 +1581,7 @@ { "alias": "", "format": "time_series", - "rawSql": "DECLARE \n @from int = $__unixEpochFrom(),\n @to int = $__unixEpochTo()\n \nEXEC dbo.sp_test_epoch @from, @to", + "rawSql": "DECLARE\n @from int = $__unixEpochFrom(), \n @to int = $__unixEpochTo(), \n @interval nvarchar(50) = '$summarize', \n @metric nvarchar(200) = $metric\n \nEXEC dbo.sp_test_epoch @from, @to, @interval, @metric", "refId": "A" } ], @@ -1386,7 +1632,7 @@ "h": 8, "w": 12, "x": 12, - "y": 54 + "y": 57 }, "id": 30, "legend": { @@ -1418,7 +1664,7 @@ { "alias": "", "format": "time_series", - "rawSql": "DECLARE \n @from datetime = $__timeFrom(),\n @to datetime = $__timeTo()\n \nEXEC dbo.sp_test_datetime @from, @to", + "rawSql": "DECLARE\n @from datetime = $__timeFrom(), \n @to datetime = $__timeTo(), \n @interval nvarchar(50) = '$summarize', \n @metric nvarchar(200) = $metric\n \nEXEC dbo.sp_test_datetime @from, @to, @interval, @metric", "refId": "A" } ], @@ -1469,7 +1715,7 @@ "h": 8, "w": 12, "x": 0, - "y": 62 + "y": 65 }, "id": 14, "legend": { @@ -1499,13 +1745,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -1559,7 +1805,7 @@ "h": 8, "w": 12, "x": 12, - "y": 62 + "y": 65 }, "id": 15, "legend": { @@ -1589,7 +1835,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -1642,7 +1888,7 @@ "h": 8, "w": 12, "x": 0, - "y": 70 + "y": 73 }, "id": 25, "legend": { @@ -1672,13 +1918,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -1732,7 +1978,7 @@ "h": 8, "w": 12, "x": 12, - "y": 70 + "y": 73 }, "id": 22, "legend": { @@ -1762,7 +2008,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -1815,7 +2061,7 @@ "h": 8, "w": 12, "x": 0, - "y": 78 + "y": 81 }, "id": 21, "legend": { @@ -1845,13 +2091,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -1905,7 +2151,7 @@ "h": 8, "w": 12, "x": 12, - "y": 78 + "y": 81 }, "id": 26, "legend": { @@ -1935,7 +2181,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -1988,7 +2234,7 @@ "h": 8, "w": 12, "x": 0, - "y": 86 + "y": 89 }, "id": 23, "legend": { @@ -2018,13 +2264,13 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value one' as metric, valueOne FROM metric_values\nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" }, { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), measurement + ' - value two' as metric, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "B" } ], @@ -2078,7 +2324,7 @@ "h": 8, "w": 12, "x": 12, - "y": 86 + "y": 89 }, "id": 24, "legend": { @@ -2108,7 +2354,7 @@ { "alias": "", "format": "time_series", - "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values ORDER BY 1", + "rawSql": "SELECT $__timeEpoch(time), valueOne, valueTwo FROM metric_values \nWHERE $__timeFilter(time) AND ($metric = 'ALL' OR measurement = $metric) ORDER BY 1", "refId": "A" } ], @@ -2157,6 +2403,26 @@ "tags": [], "templating": { "list": [ + { + "allValue": "'ALL'", + "current": {}, + "datasource": "${DS_MSSQL_TEST}", + "hide": 0, + "includeAll": true, + "label": "Metric", + "multi": false, + "name": "metric", + "options": [], + "query": "SELECT DISTINCT measurement FROM metric_values", + "refresh": 1, + "regex": "", + "sort": 0, + "tagValuesQuery": "", + "tags": [], + "tagsQuery": "", + "type": "query", + "useTags": false + }, { "auto": false, "auto_count": 30, @@ -2208,7 +2474,7 @@ }, "time": { "from": "2018-03-15T12:30:00.000Z", - "to": "2018-03-15T13:55:00.000Z" + "to": "2018-03-15T13:55:01.000Z" }, "timepicker": { "refresh_intervals": [ @@ -2238,5 +2504,5 @@ "timezone": "", "title": "Microsoft SQL Server Data Source Test", "uid": "GlAqcPgmz", - "version": 37 + "version": 57 } \ No newline at end of file diff --git a/pkg/tsdb/mssql/mssql.go b/pkg/tsdb/mssql/mssql.go index 6da61d63e42..2638fd8bb40 100644 --- a/pkg/tsdb/mssql/mssql.go +++ b/pkg/tsdb/mssql/mssql.go @@ -119,15 +119,10 @@ func (e MssqlQueryEndpoint) transformToTable(query *tsdb.Query, rows *core.Rows, return err } - // convert column named time to unix timestamp to make - // native datetime mssql types work in annotation queries - if timeIndex != -1 { - switch value := values[timeIndex].(type) { - case time.Time: - values[timeIndex] = (float64(value.Unix()) * 1000) + float64(value.Nanosecond()/1e6) // in case someone is trying to map times beyond 2262 :D - } - } - + // converts column named time to unix timestamp in milliseconds + // to make native mssql datetime types and epoch dates work in + // annotation and table queries. + tsdb.ConvertSqlTimeColumnToEpochMs(values, timeIndex) table.Rows = append(table.Rows, values) } diff --git a/pkg/tsdb/mssql/mssql_test.go b/pkg/tsdb/mssql/mssql_test.go index 8e8b22d254f..4bd1e3a8ad7 100644 --- a/pkg/tsdb/mssql/mssql_test.go +++ b/pkg/tsdb/mssql/mssql_test.go @@ -19,6 +19,8 @@ import ( // and set up a MSSQL db named grafanatest and a user/password grafana/Password! // Use the docker/blocks/mssql_tests/docker-compose.yaml to spin up a // preconfigured MSSQL server suitable for running these tests. +// Thers's also a dashboard.json in same directory that you can import to Grafana +// once you've created a datasource for the test server/database. // If needed, change the variable below to the IP address of the database. var serverIP string = "localhost" @@ -37,7 +39,7 @@ func TestMSSQL(t *testing.T) { sess := x.NewSession() defer sess.Close() - fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC) + fromStart := time.Date(2018, 3, 15, 13, 0, 0, 0, time.UTC).In(time.Local) Convey("Given a table with different native data types", func() { sql := ` @@ -186,14 +188,8 @@ func TestMSSQL(t *testing.T) { }) } - dtFormat := "2006-01-02 15:04:05.999999999" for _, s := range series { - sql = fmt.Sprintf(` - INSERT INTO metric (time, value) - VALUES(CAST('%s' AS DATETIME), %d) - `, s.Time.Format(dtFormat), s.Value) - - _, err = sess.Exec(sql) + _, err = sess.Insert(s) So(err, ShouldBeNil) } @@ -315,42 +311,34 @@ func TestMSSQL(t *testing.T) { }) Convey("Given a table with metrics having multiple values and measurements", func() { - sql := ` - IF OBJECT_ID('dbo.[metric_values]', 'U') IS NOT NULL - DROP TABLE dbo.[metric_values] - - CREATE TABLE [metric_values] ( - time datetime, - measurement nvarchar(100), - valueOne int, - valueTwo int, - ) - ` - - _, err := sess.Exec(sql) - So(err, ShouldBeNil) - - type metricValues struct { + type metric_values struct { Time time.Time Measurement string - ValueOne int64 - ValueTwo int64 + ValueOne int64 `xorm:"integer 'valueOne'"` + ValueTwo int64 `xorm:"integer 'valueTwo'"` } + if exist, err := sess.IsTableExist(metric_values{}); err != nil || exist { + So(err, ShouldBeNil) + sess.DropTable(metric_values{}) + } + err := sess.CreateTable(metric_values{}) + So(err, ShouldBeNil) + rand.Seed(time.Now().Unix()) rnd := func(min, max int64) int64 { return rand.Int63n(max-min) + min } - series := []*metricValues{} + series := []*metric_values{} for _, t := range genTimeRangeByInterval(fromStart.Add(-30*time.Minute), 90*time.Minute, 5*time.Minute) { - series = append(series, &metricValues{ + series = append(series, &metric_values{ Time: t, Measurement: "Metric A", ValueOne: rnd(0, 100), ValueTwo: rnd(0, 100), }) - series = append(series, &metricValues{ + series = append(series, &metric_values{ Time: t, Measurement: "Metric B", ValueOne: rnd(0, 100), @@ -358,14 +346,8 @@ func TestMSSQL(t *testing.T) { }) } - dtFormat := "2006-01-02 15:04:05" for _, s := range series { - sql = fmt.Sprintf(` - INSERT metric_values (time, measurement, valueOne, valueTwo) - VALUES(CAST('%s' AS DATETIME), '%s', %d, %d) - `, s.Time.Format(dtFormat), s.Measurement, s.ValueOne, s.ValueTwo) - - _, err = sess.Exec(sql) + _, err = sess.Insert(s) So(err, ShouldBeNil) } @@ -383,8 +365,8 @@ func TestMSSQL(t *testing.T) { } resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 2) @@ -406,8 +388,8 @@ func TestMSSQL(t *testing.T) { } resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["A"] So(err, ShouldBeNil) + queryResult := resp.Results["A"] So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 2) @@ -426,32 +408,42 @@ func TestMSSQL(t *testing.T) { sql = ` CREATE PROCEDURE sp_test_epoch( - @from int, - @to int + @from int, + @to int, + @interval nvarchar(50) = '5m', + @metric nvarchar(200) = 'ALL' ) AS BEGIN + DECLARE @dInterval int + SELECT @dInterval = 300 + + IF @interval = '10m' + SELECT @dInterval = 600 + SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, measurement + ' - value one' as metric, avg(valueOne) as value FROM metric_values WHERE - time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') + time BETWEEN DATEADD(s, @from, '1970-01-01') AND DATEADD(s, @to, '1970-01-01') AND + (@metric = 'ALL' OR measurement = @metric) GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, measurement UNION ALL SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int) as time, + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, measurement + ' - value two' as metric, avg(valueTwo) as value FROM metric_values WHERE - time >= DATEADD(s, @from, '1970-01-01') AND time <= DATEADD(s, @to, '1970-01-01') + time BETWEEN DATEADD(s, @from, '1970-01-01') AND DATEADD(s, @to, '1970-01-01') AND + (@metric = 'ALL' OR measurement = @metric) GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, DATEADD(second, DATEDIFF(second,GETDATE(),GETUTCDATE()), time))/600 as int)*600 as int), + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, measurement ORDER BY 1 END @@ -484,6 +476,7 @@ func TestMSSQL(t *testing.T) { resp, err := endpoint.Query(nil, nil, query) queryResult := resp.Results["A"] So(err, ShouldBeNil) + fmt.Println("query", "sql", queryResult.Meta) So(queryResult.Error, ShouldBeNil) So(len(queryResult.Series), ShouldEqual, 4) @@ -505,32 +498,42 @@ func TestMSSQL(t *testing.T) { sql = ` CREATE PROCEDURE sp_test_datetime( - @from datetime, - @to datetime + @from datetime, + @to datetime, + @interval nvarchar(50) = '5m', + @metric nvarchar(200) = 'ALL' ) AS BEGIN + DECLARE @dInterval int + SELECT @dInterval = 300 + + IF @interval = '10m' + SELECT @dInterval = 600 + SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, measurement + ' - value one' as metric, avg(valueOne) as value FROM metric_values WHERE - time >= @from AND time <= @to + time BETWEEN @from AND @to AND + (@metric = 'ALL' OR measurement = @metric) GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, measurement UNION ALL SELECT - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int) as time, + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval as time, measurement + ' - value two' as metric, avg(valueTwo) as value FROM metric_values WHERE - time >= @from AND time <= @to + time BETWEEN @from AND @to AND + (@metric = 'ALL' OR measurement = @metric) GROUP BY - cast(cast(DATEDIFF(second, {d '1970-01-01'}, time)/600 as int)*600 as int), + CAST(ROUND(DATEDIFF(second, '1970-01-01', time)/CAST(@dInterval as float), 0) as bigint)*@dInterval, measurement ORDER BY 1 END @@ -580,7 +583,7 @@ func TestMSSQL(t *testing.T) { DROP TABLE dbo.[event] CREATE TABLE [event] ( - time_sec bigint, + time_sec int, description nvarchar(100), tags nvarchar(100), ) @@ -666,30 +669,180 @@ func TestMSSQL(t *testing.T) { }) Convey("When doing an annotation query with a time column in datetime format", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + dtFormat := "2006-01-02 15:04:05.999999999" + query := &tsdb.TsdbQuery{ Queries: []*tsdb.Query{ { Model: simplejson.NewFromAny(map[string]interface{}{ - "rawSql": "SELECT DATEADD(s, time_sec, {d '1970-01-01'}) AS time, description as [text], tags FROM [event] WHERE $__unixEpochFilter(time_sec) AND tags='ticket' ORDER BY 1 ASC", + "rawSql": fmt.Sprintf(`SELECT + CAST('%s' AS DATETIME) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Format(dtFormat)), "format": "table", }), - RefId: "Tickets", + RefId: "A", }, }, - TimeRange: &tsdb.TimeRange{ - From: fmt.Sprintf("%v", fromStart.Add(-20*time.Minute).Unix()*1000), - To: fmt.Sprintf("%v", fromStart.Add(40*time.Minute).Unix()*1000), + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column in epoch second format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, }, } resp, err := endpoint.Query(nil, nil, query) - queryResult := resp.Results["Tickets"] So(err, ShouldBeNil) - So(len(queryResult.Tables[0].Rows), ShouldEqual, 3) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) columns := queryResult.Tables[0].Rows[0] //Should be in milliseconds - So(columns[0].(float64), ShouldBeGreaterThan, 1000000000000) + So(columns[0].(int64), ShouldEqual, int64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column in epoch second format (int) should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + cast(%d as int) as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(int64), ShouldEqual, int64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column in epoch millisecond format should return ms", func() { + dt := time.Date(2018, 3, 14, 21, 20, 6, 527e6, time.UTC) + + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": fmt.Sprintf(`SELECT + %d as time, + 'message' as text, + 'tag1,tag2' as tags + `, dt.Unix()*1000), + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0].(float64), ShouldEqual, float64(dt.Unix()*1000)) + }) + + Convey("When doing an annotation query with a time column holding a bigint null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as bigint) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) + }) + + Convey("When doing an annotation query with a time column holding a datetime null value should return nil", func() { + query := &tsdb.TsdbQuery{ + Queries: []*tsdb.Query{ + { + Model: simplejson.NewFromAny(map[string]interface{}{ + "rawSql": `SELECT + cast(null as datetime) as time, + 'message' as text, + 'tag1,tag2' as tags + `, + "format": "table", + }), + RefId: "A", + }, + }, + } + + resp, err := endpoint.Query(nil, nil, query) + So(err, ShouldBeNil) + queryResult := resp.Results["A"] + So(queryResult.Error, ShouldBeNil) + So(len(queryResult.Tables[0].Rows), ShouldEqual, 1) + columns := queryResult.Tables[0].Rows[0] + + //Should be in milliseconds + So(columns[0], ShouldBeNil) }) }) }) @@ -697,6 +850,8 @@ func TestMSSQL(t *testing.T) { func InitMSSQLTestDB(t *testing.T) *xorm.Engine { x, err := xorm.NewEngine(sqlutil.TestDB_Mssql.DriverName, strings.Replace(sqlutil.TestDB_Mssql.ConnStr, "localhost", serverIP, 1)) + x.DatabaseTZ = time.UTC + x.TZLocation = time.UTC // x.ShowSQL() @@ -704,8 +859,6 @@ func InitMSSQLTestDB(t *testing.T) *xorm.Engine { t.Fatalf("Failed to init mssql db %v", err) } - sqlutil.CleanDB(x) - return x } diff --git a/public/app/plugins/datasource/mssql/partials/annotations.editor.html b/public/app/plugins/datasource/mssql/partials/annotations.editor.html index 185785ced4a..75eaa3ed1d9 100644 --- a/public/app/plugins/datasource/mssql/partials/annotations.editor.html +++ b/public/app/plugins/datasource/mssql/partials/annotations.editor.html @@ -20,7 +20,7 @@ 
Annotation Query Format

 An annotation is an event that is overlayed on top of graphs. The query can have up to three columns per row, the time column is mandatory. Annotation rendering is expensive so it is important to limit the number of rows returned.
 
-- column with alias: time for the annotation event time (in UTC). Use unix timestamp in seconds or any native date data type.
+- column with alias: time for the annotation event time. Use epoch time or any native date data type.
 - column with alias: text for the annotation text.
 - column with alias: tags for annotation tags. This is a comma separated string of tags e.g. 'tag1,tag2'.
 
diff --git a/public/app/plugins/datasource/mssql/response_parser.ts b/public/app/plugins/datasource/mssql/response_parser.ts
index c98a9652b0e..b6f538707b0 100644
--- a/public/app/plugins/datasource/mssql/response_parser.ts
+++ b/public/app/plugins/datasource/mssql/response_parser.ts
@@ -128,7 +128,7 @@ export default class ResponseParser {
       const row = table.rows[i];
       list.push({
         annotation: options.annotation,
-        time: row[timeColumnIndex],
+        time: Math.floor(row[timeColumnIndex]),
         text: row[textColumnIndex],
         tags: row[tagsColumnIndex] ? row[tagsColumnIndex].trim().split(/\s*,\s*/) : [],
       });