apitech
/
grafana
mirror of https://github.com/grafana/grafana
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

mysql: minor progress on response processing

Browse Source
pull/6693/merge
Torkel Ödegaard 9 years ago
parent bd4f073425
commit d6d2080f11
10 changed files with 196 additions and 120 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 6
      pkg/api/metrics.go
  2. 19
      pkg/models/test_data.go
  3. 4
      pkg/services/sqlstore/migrations/stats_mig.go
  4. 55
      pkg/services/sqlstore/sql_test_data.go
  5. 7
      pkg/tsdb/models.go
  6. 181
      pkg/tsdb/mysql/mysql.go
  7. 4
      pkg/tsdb/request.go
  8. 5
      public/app/plugins/datasource/mysql/datasource.ts
  9. 15
      public/app/plugins/datasource/mysql/module.ts
  10. 20
      public/app/plugins/datasource/mysql/partials/query.editor.html

6
pkg/api/metrics.go
Unescape View File

@ -50,6 +50,12 @@ func QueryMetrics(c *middleware.Context, reqDto dtos.MetricRequest) Response {
return ApiError(500, "Metric request error", err)
}
for _, res := range resp.Results {
if res.Error != nil {
res.ErrorString = res.Error.Error()
}
}
return Json(200, &resp)
}

19
pkg/models/test_data.go
Unescape View File

@ -6,14 +6,13 @@ type InsertSqlTestDataCommand struct {
}
type SqlTestData struct {
Id int64
Metric1 string
Metric2 string
ValueBigInt int64
ValueDouble float64
ValueFloat float32
ValueInt int
TimeEpoch int64
TimeDateTime time.Time
TimeTimeStamp time.Time
Id int64
Metric1 string
Metric2 string
ValueBigInt int64
ValueDouble float64
ValueFloat float32
ValueInt int
TimeEpoch int64
TimeDateTime time.Time
}

4
pkg/services/sqlstore/migrations/stats_mig.go
Unescape View File

@ -46,8 +46,8 @@ func addTestDataMigrations(mg *Migrator) {
{Name: "value_float", Type: DB_Float, Nullable: true},
{Name: "value_int", Type: DB_Int, Nullable: true},
{Name: "time_epoch", Type: DB_BigInt, Nullable: false},
{Name: "time_datetime", Type: DB_DateTime, Nullable: false},
{Name: "time_timestamp", Type: DB_TimeStamp, Nullable: false},
{Name: "time_date_time", Type: DB_DateTime, Nullable: false},
{Name: "time_time_stamp", Type: DB_TimeStamp, Nullable: false},
},
}

55
pkg/services/sqlstore/sql_test_data.go
Unescape View File

@ -1,6 +1,7 @@
package sqlstore
import (
"math/rand"
"time"
"github.com/grafana/grafana/pkg/bus"
@ -11,23 +12,53 @@ func init() {
bus.AddHandler("sql", InsertSqlTestData)
}
func InsertSqlTestData(cmd *m.InsertSqlTestDataCommand) error {
return inTransaction2(func(sess *session) error {
func sqlRandomWalk(m1 string, m2 string, intWalker int64, floatWalker float64, sess *session) error {
row := &m.SqlTestData{
Metric1: "server1",
Metric2: "frontend",
ValueBigInt: 123123,
ValueDouble: 3.14159265359,
ValueFloat: 3.14159265359,
TimeEpoch: time.Now().Unix(),
TimeDateTime: time.Now(),
}
timeWalker := time.Now().Add(time.Hour * -1)
now := time.Now()
step := time.Minute
row := &m.SqlTestData{
Metric1: m1,
Metric2: m2,
TimeEpoch: timeWalker.Unix(),
TimeDateTime: timeWalker,
}
for timeWalker.Unix() < now.Unix() {
timeWalker = timeWalker.Add(step)
row.Id = 0
row.ValueBigInt += rand.Int63n(100) - 100
row.ValueDouble += rand.Float64() - 0.5
row.ValueFloat += rand.Float32() - 0.5
row.TimeEpoch = timeWalker.Unix()
row.TimeDateTime = timeWalker
sqlog.Info("Writing SQL test data row")
if _, err := sess.Table("test_data").Insert(row); err != nil {
return err
}
}
return nil
}
func InsertSqlTestData(cmd *m.InsertSqlTestDataCommand) error {
return inTransaction2(func(sess *session) error {
var err error
sqlog.Info("SQL TestData: Clearing previous test data")
res, err := sess.Exec("TRUNCATE test_data")
if err != nil {
return err
}
rows, _ := res.RowsAffected()
sqlog.Info("SQL TestData: Truncate done", "rows", rows)
sqlRandomWalk("server1", "frontend", 100, 1.123, sess)
return nil
return err
})
}

7
pkg/tsdb/models.go
Unescape View File

@ -45,9 +45,10 @@ func (br *BatchResult) WithError(err error) *BatchResult {
}
type QueryResult struct {
Error error `json:"error"`
RefId string `json:"refId"`
Series TimeSeriesSlice `json:"series"`
Error error `json:"-"`
ErrorString string `json:"error"`
RefId string `json:"refId"`
Series TimeSeriesSlice `json:"series"`
}
type TimeSeries struct {

181
pkg/tsdb/mysql/mysql.go
Unescape View File

@ -4,9 +4,12 @@ import (
"context"
"database/sql"
"fmt"
"strconv"
"sync"
"github.com/go-xorm/core"
"github.com/go-xorm/xorm"
"github.com/grafana/grafana/pkg/components/null"
"github.com/grafana/grafana/pkg/log"
"github.com/grafana/grafana/pkg/models"
"github.com/grafana/grafana/pkg/tsdb"
@ -74,19 +77,14 @@ func (e *MysqlExecutor) initEngine() error {
}
func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, context *tsdb.QueryContext) *tsdb.BatchResult {
result := &tsdb.BatchResult{}
result := &tsdb.BatchResult{
QueryResults: make(map[string]*tsdb.QueryResult),
}
session := e.engine.NewSession()
defer session.Close()
db := session.DB()
// queries := strings.Split(req.Query, ";")
//
// data := dataStruct{}
// data.Results = make([]resultsStruct, 1)
// data.Results[0].Series = make([]seriesStruct, 0)
for _, query := range queries {
rawSql := query.Model.Get("rawSql").MustString()
if rawSql == "" {
@ -100,118 +98,119 @@ func (e *MysqlExecutor) Execute(ctx context.Context, queries tsdb.QuerySlice, co
}
defer rows.Close()
columnNames, err := rows.Columns()
result.QueryResults[query.RefId] = e.TransformToTimeSeries(query, rows)
}
for _, value := range result.QueryResults {
if value.Error != nil {
e.log.Error("error", "error", value.Error)
}
}
return result
}
func (e MysqlExecutor) TransformToTimeSeries(query *tsdb.Query, rows *core.Rows) *tsdb.QueryResult {
result := &tsdb.QueryResult{RefId: query.RefId}
pointsBySeries := make(map[string]*tsdb.TimeSeries)
columnNames, err := rows.Columns()
if err != nil {
result.Error = err
return result
}
rowData := NewStringStringScan(columnNames)
for rows.Next() {
err := rowData.Update(rows.Rows)
if err != nil {
e.log.Error("Mysql response parsing", "error", err)
result.Error = err
return result
}
rc := NewStringStringScan(columnNames)
for rows.Next() {
err := rc.Update(rows.Rows)
if err != nil {
e.log.Error("Mysql response parsing", "error", err)
result.Error = err
return result
}
if rowData.metric == "" {
rowData.metric = "Unknown"
}
e.log.Info("Rows", "metric", rowData.metric, "time", rowData.time, "value", rowData.value)
if !rowData.time.Valid {
result.Error = fmt.Errorf("Found row with no time value")
return result
}
rowValues := rc.Get()
e.log.Info("Rows", "row", rowValues)
if series, exist := pointsBySeries[rowData.metric]; exist {
series.Points = append(series.Points, tsdb.TimePoint{rowData.value, rowData.time})
} else {
series := &tsdb.TimeSeries{Name: rowData.metric}
series.Points = append(series.Points, tsdb.TimePoint{rowData.value, rowData.time})
pointsBySeries[rowData.metric] = series
}
}
// for rows.Next() {
// columnValues := make([]interface{}, len(columnNames))
//
// err = rows.ScanSlice(&columnValues)
// if err != nil {
// result.Error = err
// return result
// }
//
// // bytes -> string
// for i := range columnValues {
// rowType := reflect.TypeOf(columnValues[i])
// e.log.Info("row", "type", rowType)
//
// rawValue := reflect.Indirect(reflect.ValueOf(columnValues[i]))
//
// // if rawValue is null then ignore
// if rawValue.Interface() == nil {
// continue
// }
//
// rawValueType := reflect.TypeOf(rawValue.Interface())
// vv := reflect.ValueOf(rawValue.Interface())
// e.log.Info("column type", "name", columnNames[i], "type", rawValueType, "vv", vv)
// }
// }
for _, value := range pointsBySeries {
result.Series = append(result.Series, value)
}
return result
}
type stringStringScan struct {
// cp are the column pointers
cp []interface{}
// row contains the final result
row []string
colCount int
colNames []string
rowPtrs []interface{}
rowValues []string
columnNames []string
columnCount int
time null.Float
value null.Float
metric string
}
func NewStringStringScan(columnNames []string) *stringStringScan {
lenCN := len(columnNames)
s := &stringStringScan{
cp: make([]interface{}, lenCN),
row: make([]string, lenCN*2),
colCount: lenCN,
colNames: columnNames,
columnCount: len(columnNames),
columnNames: columnNames,
rowPtrs: make([]interface{}, len(columnNames)),
rowValues: make([]string, len(columnNames)),
}
j := 0
for i := 0; i < lenCN; i++ {
s.cp[i] = new(sql.RawBytes)
s.row[j] = s.colNames[i]
j = j + 2
for i := 0; i < s.columnCount; i++ {
s.rowPtrs[i] = new(sql.RawBytes)
}
return s
}
func (s *stringStringScan) Update(rows *sql.Rows) error {
if err := rows.Scan(s.cp...); err != nil {
if err := rows.Scan(s.rowPtrs...); err != nil {
return err
}
j := 0
for i := 0; i < s.colCount; i++ {
if rb, ok := s.cp[i].(*sql.RawBytes); ok {
s.row[j+1] = string(*rb)
for i := 0; i < s.columnCount; i++ {
if rb, ok := s.rowPtrs[i].(*sql.RawBytes); ok {
s.rowValues[i] = string(*rb)
fmt.Printf("column %s = %s", s.columnNames[i], s.rowValues[i])
switch s.columnNames[i] {
case "time_sec":
if sec, err := strconv.ParseInt(s.rowValues[i], 10, 64); err == nil {
s.time = null.FloatFrom(float64(sec * 1000))
}
case "value":
if value, err := strconv.ParseFloat(s.rowValues[i], 64); err == nil {
s.value = null.FloatFrom(value)
}
case "metric":
if value, err := strconv.ParseFloat(s.rowValues[i], 64); err == nil {
s.value = null.FloatFrom(value)
}
}
*rb = nil // reset pointer to discard current value to avoid a bug
} else {
return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.colNames[i])
return fmt.Errorf("Cannot convert index %d column %s to type *sql.RawBytes", i, s.columnNames[i])
}
j = j + 2
}
return nil
}
func (s *stringStringScan) Get() []string {
return s.row
}
// type sqlDataRequest struct {
// Query string `json:"query"`
// Body []byte `json:"-"`
// }
//
// type seriesStruct struct {
// Columns []string `json:"columns"`
// Name string `json:"name"`
// Values [][]interface{} `json:"values"`
// }
//
// type resultsStruct struct {
// Series []seriesStruct `json:"series"`
// }
//
// type dataStruct struct {
// Results []resultsStruct `json:"results"`
// }

4
pkg/tsdb/request.go
Unescape View File

@ -1,6 +1,8 @@
package tsdb
import "context"
import (
"context"
)
type HandleRequestFunc func(ctx context.Context, req *Request) (*Response, error)

5
public/app/plugins/datasource/mysql/datasource.ts
Unescape View File

@ -39,6 +39,11 @@ export class MysqlDatasource {
var data = [];
if (res.results) {
_.forEach(res.results, queryRes => {
if (queryRes.error) {
throw {error: queryRes.error, message: queryRes.error};
}
for (let series of queryRes.series) {
data.push({
target: series.name,

15
public/app/plugins/datasource/mysql/module.ts
Unescape View File

@ -6,6 +6,21 @@ import {QueryCtrl} from 'app/plugins/sdk';
class MysqlQueryCtrl extends QueryCtrl {
static templateUrl = 'partials/query.editor.html';
resultFormats: any;
target: any;
constructor($scope, $injector) {
super($scope, $injector);
this.target.resultFormat = 'time_series';
this.target.alias = "{{table}}{{col_3}}";
this.resultFormats = [
{text: 'Time series', value: 'time_series'},
{text: 'Table', value: 'table'},
];
}
}
class MysqlConfigCtrl {

20
public/app/plugins/datasource/mysql/partials/query.editor.html
Unescape View File

@ -1,7 +1,25 @@
<query-editor-row query-ctrl="ctrl" can-collapse="false">
<div class="gf-form-inline">
<div class="gf-form gf-form--grow">
<textarea rows="3" class="gf-form-input" ng-model="ctrl.target.rawSql" spellcheck="false" placeholder="query expression" data-min-length=0 data-items=100 ng-model-onblur ng-change="ctrl.refreshMetricData()"></textarea>
<textarea rows="6" class="gf-form-input" ng-model="ctrl.target.rawSql" spellcheck="false" placeholder="query expression" data-min-length=0 data-items=100 ng-model-onblur ng-change="ctrl.refreshMetricData()"></textarea>
</div>
</div>
<div class="gf-form-inline">
<div class="gf-form">
<label class="gf-form-label query-keyword">Format as</label>
<div class="gf-form-select-wrapper">
<select class="gf-form-input gf-size-auto" ng-model="ctrl.target.resultFormat" ng-options="f.value as f.text for f in ctrl.resultFormats" ng-change="ctrl.refresh()"></select>
</div>
</div>
<div class="gf-form max-width-30">
<label class="gf-form-label query-keyword">Name by</label>
<input type="text" class="gf-form-input" ng-model="ctrl.target.alias" spellcheck='false' placeholder="pattern" ng-blur="ctrl.refresh()">
</div>
<div class="gf-form gf-form--grow">
<div class="gf-form-label gf-form-label--grow"></div>
</div>
</div>
</query-editor-row>

Write Preview
Loading…
Cancel
Save