mirror of https://github.com/grafana/grafana
QueryService: Use types from sdk (#84029)
Ryan McKinley
2 years ago
committed by
GitHub
parent
f11b10a10c
commit
d82f3be6f7
@ -1,240 +1,59 @@ |
||||
package v0alpha1 |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
"fmt" |
||||
"net/http" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
runtime "k8s.io/apimachinery/pkg/runtime" |
||||
) |
||||
|
||||
// Generic query request with shared time across all values
|
||||
// Copied from: https://github.com/grafana/grafana/blob/main/pkg/api/dtos/models.go#L62
|
||||
type GenericQueryRequest struct { |
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type QueryDataRequest struct { |
||||
metav1.TypeMeta `json:",inline"` |
||||
|
||||
// From Start time in epoch timestamps in milliseconds or relative using Grafana time units.
|
||||
// example: now-1h
|
||||
From string `json:"from,omitempty"` |
||||
|
||||
// To End time in epoch timestamps in milliseconds or relative using Grafana time units.
|
||||
// example: now
|
||||
To string `json:"to,omitempty"` |
||||
|
||||
// queries.refId – Specifies an identifier of the query. Is optional and default to “A”.
|
||||
// queries.datasourceId – Specifies the data source to be queried. Each query in the request must have an unique datasourceId.
|
||||
// queries.maxDataPoints - Species maximum amount of data points that dashboard panel can render. Is optional and default to 100.
|
||||
// queries.intervalMs - Specifies the time interval in milliseconds of time series. Is optional and defaults to 1000.
|
||||
// required: true
|
||||
// example: [ { "refId": "A", "intervalMs": 86400000, "maxDataPoints": 1092, "datasource":{ "uid":"PD8C576611E62080A" }, "rawSql": "SELECT 1 as valueOne, 2 as valueTwo", "format": "table" } ]
|
||||
Queries []GenericDataQuery `json:"queries"` |
||||
|
||||
// required: false
|
||||
Debug bool `json:"debug,omitempty"` |
||||
} |
||||
|
||||
type DataSourceRef struct { |
||||
// The datasource plugin type
|
||||
Type string `json:"type"` |
||||
|
||||
// Datasource UID
|
||||
UID string `json:"uid"` |
||||
} |
||||
|
||||
// GenericDataQuery is a replacement for `dtos.MetricRequest` that provides more explicit types
|
||||
type GenericDataQuery struct { |
||||
// RefID is the unique identifier of the query, set by the frontend call.
|
||||
RefID string `json:"refId"` |
||||
|
||||
// TimeRange represents the query range
|
||||
// NOTE: unlike generic /ds/query, we can now send explicit time values in each query
|
||||
TimeRange *TimeRange `json:"timeRange,omitempty"` |
||||
|
||||
// The datasource
|
||||
Datasource *DataSourceRef `json:"datasource,omitempty"` |
||||
|
||||
// Deprecated -- use datasource ref instead
|
||||
DatasourceId int64 `json:"datasourceId,omitempty"` |
||||
|
||||
// QueryType is an optional identifier for the type of query.
|
||||
// It can be used to distinguish different types of queries.
|
||||
QueryType string `json:"queryType,omitempty"` |
||||
|
||||
// MaxDataPoints is the maximum number of data points that should be returned from a time series query.
|
||||
MaxDataPoints int64 `json:"maxDataPoints,omitempty"` |
||||
|
||||
// Interval is the suggested duration between time points in a time series query.
|
||||
IntervalMS float64 `json:"intervalMs,omitempty"` |
||||
|
||||
// true if query is disabled (ie should not be returned to the dashboard)
|
||||
// Note this does not always imply that the query should not be executed since
|
||||
// the results from a hidden query may be used as the input to other queries (SSE etc)
|
||||
Hide bool `json:"hide,omitempty"` |
||||
|
||||
// Additional Properties (that live at the root)
|
||||
props map[string]any `json:"-"` |
||||
} |
||||
|
||||
func NewGenericDataQuery(vals map[string]any) GenericDataQuery { |
||||
q := GenericDataQuery{} |
||||
_ = q.unmarshal(vals) |
||||
return q |
||||
} |
||||
|
||||
// TimeRange represents a time range for a query and is a property of DataQuery.
|
||||
type TimeRange struct { |
||||
// From is the start time of the query.
|
||||
From string `json:"from"` |
||||
|
||||
// To is the end time of the query.
|
||||
To string `json:"to"` |
||||
// The time range used when not included on each query
|
||||
data.QueryDataRequest `json:",inline"` |
||||
} |
||||
|
||||
func (g *GenericDataQuery) AdditionalProperties() map[string]any { |
||||
if g.props == nil { |
||||
g.props = make(map[string]any) |
||||
} |
||||
return g.props |
||||
} |
||||
// Wraps backend.QueryDataResponse, however it includes TypeMeta and implements runtime.Object
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type QueryDataResponse struct { |
||||
metav1.TypeMeta `json:",inline"` |
||||
|
||||
// DeepCopyInto is an autogenerated deepcopy function, copying the receiver, writing into out. in must be non-nil.
|
||||
func (g *GenericDataQuery) DeepCopyInto(out *GenericDataQuery) { |
||||
*out = *g |
||||
if g.props != nil { |
||||
out.props = runtime.DeepCopyJSON(g.props) |
||||
} |
||||
// Backend wrapper (external dependency)
|
||||
backend.QueryDataResponse `json:",inline"` |
||||
} |
||||
|
||||
// DeepCopy is an autogenerated deepcopy function, copying the receiver, creating a new GenericDataQuery.
|
||||
func (g *GenericDataQuery) DeepCopy() *GenericDataQuery { |
||||
if g == nil { |
||||
return nil |
||||
// If errors exist, return multi-status
|
||||
func GetResponseCode(rsp *backend.QueryDataResponse) int { |
||||
if rsp == nil { |
||||
return http.StatusInternalServerError |
||||
} |
||||
out := new(GenericDataQuery) |
||||
g.DeepCopyInto(out) |
||||
return out |
||||
} |
||||
|
||||
// MarshalJSON ensures that the unstructured object produces proper
|
||||
// JSON when passed to Go's standard JSON library.
|
||||
func (g GenericDataQuery) MarshalJSON() ([]byte, error) { |
||||
vals := map[string]any{} |
||||
if g.props != nil { |
||||
for k, v := range g.props { |
||||
vals[k] = v |
||||
for _, v := range rsp.Responses { |
||||
if v.Error != nil { |
||||
return http.StatusMultiStatus |
||||
} |
||||
} |
||||
|
||||
vals["refId"] = g.RefID |
||||
if g.Datasource != nil && (g.Datasource.Type != "" || g.Datasource.UID != "") { |
||||
vals["datasource"] = g.Datasource |
||||
} |
||||
if g.DatasourceId > 0 { |
||||
vals["datasourceId"] = g.DatasourceId |
||||
} |
||||
if g.IntervalMS > 0 { |
||||
vals["intervalMs"] = g.IntervalMS |
||||
} |
||||
if g.MaxDataPoints > 0 { |
||||
vals["maxDataPoints"] = g.MaxDataPoints |
||||
} |
||||
if g.QueryType != "" { |
||||
vals["queryType"] = g.QueryType |
||||
} |
||||
return json.Marshal(vals) |
||||
} |
||||
|
||||
// UnmarshalJSON ensures that the unstructured object properly decodes
|
||||
// JSON when passed to Go's standard JSON library.
|
||||
func (g *GenericDataQuery) UnmarshalJSON(b []byte) error { |
||||
vals := map[string]any{} |
||||
err := json.Unmarshal(b, &vals) |
||||
if err != nil { |
||||
return err |
||||
} |
||||
return g.unmarshal(vals) |
||||
return http.StatusOK |
||||
} |
||||
|
||||
func (g *GenericDataQuery) unmarshal(vals map[string]any) error { |
||||
if vals == nil { |
||||
g.props = nil |
||||
return nil |
||||
} |
||||
// Defines a query behavior in a datasource. This is a similar model to a CRD where the
|
||||
// payload describes a valid query
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type QueryTypeDefinition struct { |
||||
metav1.TypeMeta `json:",inline"` |
||||
metav1.ObjectMeta `json:"metadata,omitempty"` |
||||
|
||||
key := "refId" |
||||
v, ok := vals[key] |
||||
if ok { |
||||
g.RefID, ok = v.(string) |
||||
if !ok { |
||||
return fmt.Errorf("expected string refid (got: %t)", v) |
||||
} |
||||
delete(vals, key) |
||||
} |
||||
|
||||
key = "datasource" |
||||
v, ok = vals[key] |
||||
if ok { |
||||
wrap, ok := v.(map[string]any) |
||||
if ok { |
||||
g.Datasource = &DataSourceRef{} |
||||
g.Datasource.Type, _ = wrap["type"].(string) |
||||
g.Datasource.UID, _ = wrap["uid"].(string) |
||||
delete(vals, key) |
||||
} else { |
||||
// Old old queries may arrive with just the name
|
||||
name, ok := v.(string) |
||||
if !ok { |
||||
return fmt.Errorf("expected datasource as object (got: %t)", v) |
||||
} |
||||
g.Datasource = &DataSourceRef{} |
||||
g.Datasource.UID = name // Not great, but the lookup function will try its best to resolve
|
||||
delete(vals, key) |
||||
} |
||||
} |
||||
|
||||
key = "intervalMs" |
||||
v, ok = vals[key] |
||||
if ok { |
||||
g.IntervalMS, ok = v.(float64) |
||||
if !ok { |
||||
return fmt.Errorf("expected intervalMs as float (got: %t)", v) |
||||
} |
||||
delete(vals, key) |
||||
} |
||||
|
||||
key = "maxDataPoints" |
||||
v, ok = vals[key] |
||||
if ok { |
||||
count, ok := v.(float64) |
||||
if !ok { |
||||
return fmt.Errorf("expected maxDataPoints as number (got: %t)", v) |
||||
} |
||||
g.MaxDataPoints = int64(count) |
||||
delete(vals, key) |
||||
} |
||||
|
||||
key = "datasourceId" |
||||
v, ok = vals[key] |
||||
if ok { |
||||
count, ok := v.(float64) |
||||
if !ok { |
||||
return fmt.Errorf("expected datasourceId as number (got: %t)", v) |
||||
} |
||||
g.DatasourceId = int64(count) |
||||
delete(vals, key) |
||||
} |
||||
Spec data.QueryTypeDefinitionSpec `json:"spec,omitempty"` |
||||
} |
||||
|
||||
key = "queryType" |
||||
v, ok = vals[key] |
||||
if ok { |
||||
queryType, ok := v.(string) |
||||
if !ok { |
||||
return fmt.Errorf("expected queryType as string (got: %t)", v) |
||||
} |
||||
g.QueryType = queryType |
||||
delete(vals, key) |
||||
} |
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type QueryTypeDefinitionList struct { |
||||
metav1.TypeMeta `json:",inline"` |
||||
metav1.ListMeta `json:"metadata,omitempty"` |
||||
|
||||
g.props = vals |
||||
return nil |
||||
Items []QueryTypeDefinition `json:"items,omitempty"` |
||||
} |
||||
|
||||
@ -1,64 +0,0 @@ |
||||
package v0alpha1 |
||||
|
||||
import ( |
||||
"encoding/json" |
||||
|
||||
"github.com/grafana/grafana-plugin-sdk-go/backend" |
||||
metav1 "k8s.io/apimachinery/pkg/apis/meta/v1" |
||||
openapi "k8s.io/kube-openapi/pkg/common" |
||||
spec "k8s.io/kube-openapi/pkg/validation/spec" |
||||
) |
||||
|
||||
// Wraps backend.QueryDataResponse, however it includes TypeMeta and implements runtime.Object
|
||||
// +k8s:deepcopy-gen:interfaces=k8s.io/apimachinery/pkg/runtime.Object
|
||||
type QueryDataResponse struct { |
||||
metav1.TypeMeta `json:",inline"` |
||||
|
||||
// Backend wrapper (external dependency)
|
||||
backend.QueryDataResponse |
||||
} |
||||
|
||||
// Expose backend DataResponse in OpenAPI (yes this still requires some serious love!)
|
||||
func (r QueryDataResponse) OpenAPIDefinition() openapi.OpenAPIDefinition { |
||||
return openapi.OpenAPIDefinition{ |
||||
Schema: spec.Schema{ |
||||
SchemaProps: spec.SchemaProps{ |
||||
Type: []string{"object"}, |
||||
AdditionalProperties: &spec.SchemaOrBool{Allows: true}, |
||||
}, |
||||
VendorExtensible: spec.VendorExtensible{ |
||||
Extensions: map[string]interface{}{ |
||||
"x-kubernetes-preserve-unknown-fields": true, |
||||
}, |
||||
}, |
||||
}, |
||||
} |
||||
} |
||||
|
||||
// MarshalJSON writes the results as json
|
||||
func (r QueryDataResponse) MarshalJSON() ([]byte, error) { |
||||
return r.QueryDataResponse.MarshalJSON() |
||||
} |
||||
|
||||
// UnmarshalJSON will read JSON into a QueryDataResponse
|
||||
func (r *QueryDataResponse) UnmarshalJSON(b []byte) error { |
||||
return r.QueryDataResponse.UnmarshalJSON(b) |
||||
} |
||||
|
||||
func (r *QueryDataResponse) DeepCopy() *QueryDataResponse { |
||||
if r == nil { |
||||
return nil |
||||
} |
||||
|
||||
// /!\ The most dumb approach, but OK for now...
|
||||
// likely best to move DeepCopy into SDK
|
||||
out := &QueryDataResponse{} |
||||
body, _ := json.Marshal(r.QueryDataResponse) |
||||
_ = json.Unmarshal(body, &out.QueryDataResponse) |
||||
return out |
||||
} |
||||
|
||||
func (r *QueryDataResponse) DeepCopyInto(out *QueryDataResponse) { |
||||
clone := r.DeepCopy() |
||||
*out = *clone |
||||
} |
||||
@ -0,0 +1,22 @@ |
||||
package query |
||||
|
||||
import ( |
||||
"context" |
||||
|
||||
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" |
||||
) |
||||
|
||||
// The query runner interface
|
||||
type DataSourceClientSupplier interface { |
||||
// Get a client for a given datasource
|
||||
// NOTE: authorization headers are not yet added and the client may be shared across multiple users
|
||||
GetDataSourceClient(ctx context.Context, ref data.DataSourceRef) (data.QueryDataClient, error) |
||||
} |
||||
|
||||
type CommonDataSourceClientSupplier struct { |
||||
Client data.QueryDataClient |
||||
} |
||||
|
||||
func (s *CommonDataSourceClientSupplier) GetDataSourceClient(ctx context.Context, ref data.DataSourceRef) (data.QueryDataClient, error) { |
||||
return s.Client, nil |
||||
} |
||||
@ -0,0 +1,48 @@ |
||||
package query |
||||
|
||||
import ( |
||||
"github.com/prometheus/client_golang/prometheus" |
||||
) |
||||
|
||||
const ( |
||||
metricsSubSystem = "queryservice" |
||||
metricsNamespace = "grafana" |
||||
) |
||||
|
||||
type metrics struct { |
||||
dsRequests *prometheus.CounterVec |
||||
|
||||
// older metric
|
||||
expressionsQuerySummary *prometheus.SummaryVec |
||||
} |
||||
|
||||
func newMetrics(reg prometheus.Registerer) *metrics { |
||||
m := &metrics{ |
||||
dsRequests: prometheus.NewCounterVec(prometheus.CounterOpts{ |
||||
Namespace: metricsNamespace, |
||||
Subsystem: metricsSubSystem, |
||||
Name: "ds_queries_total", |
||||
Help: "Number of datasource queries made from the query service", |
||||
}, []string{"error", "dataplane", "datasource_type"}), |
||||
|
||||
expressionsQuerySummary: prometheus.NewSummaryVec( |
||||
prometheus.SummaryOpts{ |
||||
Namespace: metricsNamespace, |
||||
Subsystem: metricsSubSystem, |
||||
Name: "expressions_queries_duration_milliseconds", |
||||
Help: "Expressions query summary", |
||||
Objectives: map[float64]float64{0.5: 0.05, 0.9: 0.01, 0.99: 0.001}, |
||||
}, |
||||
[]string{"status"}, |
||||
), |
||||
} |
||||
|
||||
if reg != nil { |
||||
reg.MustRegister( |
||||
m.dsRequests, |
||||
m.expressionsQuerySummary, |
||||
) |
||||
} |
||||
|
||||
return m |
||||
} |
||||
@ -1,83 +1,216 @@ |
||||
package query |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
|
||||
"github.com/grafana/grafana/pkg/apis/query/v0alpha1" |
||||
"github.com/grafana/grafana-plugin-sdk-go/data/utils/jsoniter" |
||||
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" |
||||
"gonum.org/v1/gonum/graph/simple" |
||||
"gonum.org/v1/gonum/graph/topo" |
||||
|
||||
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" |
||||
"github.com/grafana/grafana/pkg/expr" |
||||
"github.com/grafana/grafana/pkg/infra/tracing" |
||||
"github.com/grafana/grafana/pkg/services/datasources/service" |
||||
) |
||||
|
||||
type parsedQueryRequest struct { |
||||
// The queries broken into requests
|
||||
Requests []groupedQueries |
||||
type datasourceRequest struct { |
||||
// The type
|
||||
PluginId string `json:"pluginId"` |
||||
|
||||
// The UID
|
||||
UID string `json:"uid"` |
||||
|
||||
// Optionally show the additional query properties
|
||||
Expressions []v0alpha1.GenericDataQuery |
||||
Request *data.QueryDataRequest `json:"request"` |
||||
|
||||
// Headers that should be forwarded to the next request
|
||||
Headers map[string]string `json:"headers,omitempty"` |
||||
} |
||||
|
||||
type groupedQueries struct { |
||||
// the plugin type
|
||||
pluginId string |
||||
type parsedRequestInfo struct { |
||||
// Datasource queries, one for each datasource
|
||||
Requests []datasourceRequest `json:"requests,omitempty"` |
||||
|
||||
// The datasource name/uid
|
||||
uid string |
||||
// Expressions in required execution order
|
||||
Expressions []expr.ExpressionQuery `json:"expressions,omitempty"` |
||||
|
||||
// The raw backend query objects
|
||||
query []v0alpha1.GenericDataQuery |
||||
// Expressions include explicit hacks for influx+prometheus
|
||||
RefIDTypes map[string]string `json:"types,omitempty"` |
||||
|
||||
// Hidden queries used as dependencies
|
||||
HideBeforeReturn []string `json:"hide,omitempty"` |
||||
} |
||||
|
||||
// Internally define what makes this request unique (eventually may include the apiVersion)
|
||||
func (d *groupedQueries) key() string { |
||||
return fmt.Sprintf("%s/%s", d.pluginId, d.uid) |
||||
type queryParser struct { |
||||
legacy service.LegacyDataSourceLookup |
||||
reader *expr.ExpressionQueryReader |
||||
tracer tracing.Tracer |
||||
} |
||||
|
||||
func parseQueryRequest(raw v0alpha1.GenericQueryRequest) (parsedQueryRequest, error) { |
||||
mixed := make(map[string]*groupedQueries) |
||||
parsed := parsedQueryRequest{} |
||||
refIds := make(map[string]bool) |
||||
func newQueryParser(reader *expr.ExpressionQueryReader, legacy service.LegacyDataSourceLookup, tracer tracing.Tracer) *queryParser { |
||||
return &queryParser{ |
||||
reader: reader, |
||||
legacy: legacy, |
||||
tracer: tracer, |
||||
} |
||||
} |
||||
|
||||
// Split the main query into multiple
|
||||
func (p *queryParser) parseRequest(ctx context.Context, input *query.QueryDataRequest) (parsedRequestInfo, error) { |
||||
ctx, span := p.tracer.Start(ctx, "QueryService.parseRequest") |
||||
defer span.End() |
||||
|
||||
queryRefIDs := make(map[string]*data.DataQuery, len(input.Queries)) |
||||
expressions := make(map[string]*expr.ExpressionQuery) |
||||
index := make(map[string]int) // index lookup
|
||||
rsp := parsedRequestInfo{ |
||||
RefIDTypes: make(map[string]string, len(input.Queries)), |
||||
} |
||||
|
||||
// Ensure a valid time range
|
||||
if input.From == "" { |
||||
input.From = "now-6h" |
||||
} |
||||
if input.To == "" { |
||||
input.To = "now" |
||||
} |
||||
|
||||
for _, original := range raw.Queries { |
||||
if refIds[original.RefID] { |
||||
return parsed, fmt.Errorf("invalid query, duplicate refId: " + original.RefID) |
||||
for _, q := range input.Queries { |
||||
_, found := queryRefIDs[q.RefID] |
||||
if found { |
||||
return rsp, fmt.Errorf("multiple queries found for refId: %s", q.RefID) |
||||
} |
||||
_, found = expressions[q.RefID] |
||||
if found { |
||||
return rsp, fmt.Errorf("multiple queries found for refId: %s", q.RefID) |
||||
} |
||||
|
||||
refIds[original.RefID] = true |
||||
q := original |
||||
ds, err := p.getValidDataSourceRef(ctx, q.Datasource, q.DatasourceID) |
||||
if err != nil { |
||||
return rsp, err |
||||
} |
||||
|
||||
if q.TimeRange == nil && raw.From != "" { |
||||
q.TimeRange = &v0alpha1.TimeRange{ |
||||
From: raw.From, |
||||
To: raw.To, |
||||
// Process each query
|
||||
if expr.IsDataSource(ds.UID) { |
||||
// In order to process the query as a typed expression query, we
|
||||
// are writing it back to JSON and parsing again. Alternatively we
|
||||
// could construct it from the untyped map[string]any additional properties
|
||||
// but this approach lets us focus on well typed behavior first
|
||||
raw, err := json.Marshal(q) |
||||
if err != nil { |
||||
return rsp, err |
||||
} |
||||
iter, err := jsoniter.ParseBytes(jsoniter.ConfigDefault, raw) |
||||
if err != nil { |
||||
return rsp, err |
||||
} |
||||
exp, err := p.reader.ReadQuery(q, iter) |
||||
if err != nil { |
||||
return rsp, err |
||||
} |
||||
exp.GraphID = int64(len(expressions) + 1) |
||||
expressions[q.RefID] = &exp |
||||
} else { |
||||
key := fmt.Sprintf("%s/%s", ds.Type, ds.UID) |
||||
idx, ok := index[key] |
||||
if !ok { |
||||
idx = len(index) |
||||
index[key] = idx |
||||
rsp.Requests = append(rsp.Requests, datasourceRequest{ |
||||
PluginId: ds.Type, |
||||
UID: ds.UID, |
||||
Request: &data.QueryDataRequest{ |
||||
TimeRange: input.TimeRange, |
||||
Debug: input.Debug, |
||||
// no queries
|
||||
}, |
||||
}) |
||||
} |
||||
} |
||||
|
||||
// Extract out the expressions queries earlier
|
||||
if expr.IsDataSource(q.Datasource.Type) || expr.IsDataSource(q.Datasource.UID) { |
||||
parsed.Expressions = append(parsed.Expressions, q) |
||||
continue |
||||
req := rsp.Requests[idx].Request |
||||
req.Queries = append(req.Queries, q) |
||||
queryRefIDs[q.RefID] = &req.Queries[len(req.Queries)-1] |
||||
} |
||||
|
||||
g := &groupedQueries{pluginId: q.Datasource.Type, uid: q.Datasource.UID} |
||||
group, ok := mixed[g.key()] |
||||
if !ok || group == nil { |
||||
group = g |
||||
mixed[g.key()] = g |
||||
// Mark all the queries that should be hidden ()
|
||||
if q.Hide { |
||||
rsp.HideBeforeReturn = append(rsp.HideBeforeReturn, q.RefID) |
||||
} |
||||
group.query = append(group.query, q) |
||||
} |
||||
|
||||
for _, q := range parsed.Expressions { |
||||
// TODO: parse and build tree, for now just fail fast on unknown commands
|
||||
_, err := expr.GetExpressionCommandType(q.AdditionalProperties()) |
||||
// Make sure all referenced variables exist and the expression order is stable
|
||||
if len(expressions) > 0 { |
||||
queryNode := &expr.ExpressionQuery{ |
||||
GraphID: -1, |
||||
} |
||||
|
||||
// Build the graph for a request
|
||||
dg := simple.NewDirectedGraph() |
||||
dg.AddNode(queryNode) |
||||
for _, exp := range expressions { |
||||
dg.AddNode(exp) |
||||
} |
||||
for _, exp := range expressions { |
||||
vars := exp.Command.NeedsVars() |
||||
for _, refId := range vars { |
||||
target := queryNode |
||||
q, ok := queryRefIDs[refId] |
||||
if !ok { |
||||
target, ok = expressions[refId] |
||||
if !ok { |
||||
return rsp, fmt.Errorf("expression [%s] is missing variable [%s]", exp.RefID, refId) |
||||
} |
||||
} |
||||
// Do not hide queries used in variables
|
||||
if q != nil && q.Hide { |
||||
q.Hide = false |
||||
} |
||||
if target.ID() == exp.ID() { |
||||
return rsp, fmt.Errorf("expression [%s] can not depend on itself", exp.RefID) |
||||
} |
||||
dg.SetEdge(dg.NewEdge(target, exp)) |
||||
} |
||||
} |
||||
|
||||
// Add the sorted expressions
|
||||
sortedNodes, err := topo.SortStabilized(dg, nil) |
||||
if err != nil { |
||||
return parsed, err |
||||
return rsp, fmt.Errorf("cyclic references in query") |
||||
} |
||||
for _, v := range sortedNodes { |
||||
if v.ID() > 0 { |
||||
rsp.Expressions = append(rsp.Expressions, *v.(*expr.ExpressionQuery)) |
||||
} |
||||
} |
||||
} |
||||
|
||||
// Add each request
|
||||
for _, v := range mixed { |
||||
parsed.Requests = append(parsed.Requests, *v) |
||||
} |
||||
return rsp, nil |
||||
} |
||||
|
||||
return parsed, nil |
||||
func (p *queryParser) getValidDataSourceRef(ctx context.Context, ds *data.DataSourceRef, id int64) (*data.DataSourceRef, error) { |
||||
if ds == nil { |
||||
if id == 0 { |
||||
return nil, fmt.Errorf("missing datasource reference or id") |
||||
} |
||||
if p.legacy == nil { |
||||
return nil, fmt.Errorf("legacy datasource lookup unsupported (id:%d)", id) |
||||
} |
||||
return p.legacy.GetDataSourceFromDeprecatedFields(ctx, "", id) |
||||
} |
||||
if ds.Type == "" { |
||||
if ds.UID == "" { |
||||
return nil, fmt.Errorf("missing name/uid in data source reference") |
||||
} |
||||
if ds.UID == expr.DatasourceType { |
||||
return ds, nil |
||||
} |
||||
if p.legacy == nil { |
||||
return nil, fmt.Errorf("legacy datasource lookup unsupported (name:%s)", ds.UID) |
||||
} |
||||
return p.legacy.GetDataSourceFromDeprecatedFields(ctx, ds.UID, 0) |
||||
} |
||||
return ds, nil |
||||
} |
||||
|
||||
@ -0,0 +1,131 @@ |
||||
package query |
||||
|
||||
import ( |
||||
"context" |
||||
"encoding/json" |
||||
"fmt" |
||||
"os" |
||||
"path" |
||||
"strings" |
||||
"testing" |
||||
|
||||
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" |
||||
"github.com/stretchr/testify/assert" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
query "github.com/grafana/grafana/pkg/apis/query/v0alpha1" |
||||
"github.com/grafana/grafana/pkg/expr" |
||||
"github.com/grafana/grafana/pkg/infra/tracing" |
||||
"github.com/grafana/grafana/pkg/services/featuremgmt" |
||||
) |
||||
|
||||
type parserTestObject struct { |
||||
Description string `json:"description,omitempty"` |
||||
Request query.QueryDataRequest `json:"input"` |
||||
Expect parsedRequestInfo `json:"expect"` |
||||
Error string `json:"error,omitempty"` |
||||
} |
||||
|
||||
func TestQuerySplitting(t *testing.T) { |
||||
ctx := context.Background() |
||||
parser := newQueryParser(expr.NewExpressionQueryReader(featuremgmt.WithFeatures()), |
||||
&legacyDataSourceRetriever{}, tracing.InitializeTracerForTest()) |
||||
|
||||
t.Run("missing datasource flavors", func(t *testing.T) { |
||||
split, err := parser.parseRequest(ctx, &query.QueryDataRequest{ |
||||
QueryDataRequest: data.QueryDataRequest{ |
||||
Queries: []data.DataQuery{{ |
||||
CommonQueryProperties: data.CommonQueryProperties{ |
||||
RefID: "A", |
||||
}, |
||||
}}, |
||||
}, |
||||
}) |
||||
require.Error(t, err) // Missing datasource
|
||||
require.Empty(t, split.Requests) |
||||
}) |
||||
|
||||
t.Run("applies default time range", func(t *testing.T) { |
||||
split, err := parser.parseRequest(ctx, &query.QueryDataRequest{ |
||||
QueryDataRequest: data.QueryDataRequest{ |
||||
TimeRange: data.TimeRange{}, // missing
|
||||
Queries: []data.DataQuery{{ |
||||
CommonQueryProperties: data.CommonQueryProperties{ |
||||
RefID: "A", |
||||
Datasource: &data.DataSourceRef{ |
||||
Type: "x", |
||||
UID: "abc", |
||||
}, |
||||
}, |
||||
}}, |
||||
}, |
||||
}) |
||||
require.NoError(t, err) |
||||
require.Len(t, split.Requests, 1) |
||||
require.Equal(t, "now-6h", split.Requests[0].Request.From) |
||||
require.Equal(t, "now", split.Requests[0].Request.To) |
||||
}) |
||||
|
||||
t.Run("verify tests", func(t *testing.T) { |
||||
files, err := os.ReadDir("testdata") |
||||
require.NoError(t, err) |
||||
|
||||
for _, file := range files { |
||||
if !strings.HasSuffix(file.Name(), ".json") { |
||||
continue |
||||
} |
||||
|
||||
fpath := path.Join("testdata", file.Name()) |
||||
// nolint:gosec
|
||||
body, err := os.ReadFile(fpath) |
||||
require.NoError(t, err) |
||||
harness := &parserTestObject{} |
||||
err = json.Unmarshal(body, harness) |
||||
require.NoError(t, err) |
||||
|
||||
changed := false |
||||
parsed, err := parser.parseRequest(ctx, &harness.Request) |
||||
if err != nil { |
||||
if !assert.Equal(t, harness.Error, err.Error(), "File %s", file) { |
||||
changed = true |
||||
} |
||||
} else { |
||||
x, _ := json.Marshal(parsed) |
||||
y, _ := json.Marshal(harness.Expect) |
||||
if !assert.JSONEq(t, string(y), string(x), "File %s", file) { |
||||
changed = true |
||||
} |
||||
} |
||||
|
||||
if changed { |
||||
harness.Error = "" |
||||
harness.Expect = parsed |
||||
if err != nil { |
||||
harness.Error = err.Error() |
||||
} |
||||
jj, err := json.MarshalIndent(harness, "", " ") |
||||
require.NoError(t, err) |
||||
err = os.WriteFile(fpath, jj, 0600) |
||||
require.NoError(t, err) |
||||
} |
||||
} |
||||
}) |
||||
} |
||||
|
||||
type legacyDataSourceRetriever struct{} |
||||
|
||||
func (s *legacyDataSourceRetriever) GetDataSourceFromDeprecatedFields(ctx context.Context, name string, id int64) (*data.DataSourceRef, error) { |
||||
if id == 100 { |
||||
return &data.DataSourceRef{ |
||||
Type: "plugin-aaaa", |
||||
UID: "AAA", |
||||
}, nil |
||||
} |
||||
if name != "" { |
||||
return &data.DataSourceRef{ |
||||
Type: "plugin-bbb", |
||||
UID: name, |
||||
}, nil |
||||
} |
||||
return nil, fmt.Errorf("missing parameter") |
||||
} |
||||
@ -0,0 +1,29 @@ |
||||
{ |
||||
"description": "self dependencies", |
||||
"input": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "", |
||||
"uid": "__expr__" |
||||
}, |
||||
"expression": "$B", |
||||
"type": "math" |
||||
}, |
||||
{ |
||||
"refId": "B", |
||||
"datasource": { |
||||
"type": "", |
||||
"uid": "__expr__" |
||||
}, |
||||
"type": "math", |
||||
"expression": "$A" |
||||
} |
||||
] |
||||
}, |
||||
"expect": {}, |
||||
"error": "cyclic references in query" |
||||
} |
||||
@ -0,0 +1,60 @@ |
||||
{ |
||||
"input": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "plugin-x", |
||||
"uid": "123" |
||||
} |
||||
}, |
||||
{ |
||||
"refId": "B", |
||||
"datasource": { |
||||
"type": "plugin-x", |
||||
"uid": "456" |
||||
} |
||||
} |
||||
] |
||||
}, |
||||
"expect": { |
||||
"requests": [ |
||||
{ |
||||
"pluginId": "plugin-x", |
||||
"uid": "123", |
||||
"request": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "plugin-x", |
||||
"uid": "123" |
||||
} |
||||
} |
||||
] |
||||
} |
||||
}, |
||||
{ |
||||
"pluginId": "plugin-x", |
||||
"uid": "456", |
||||
"request": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "B", |
||||
"datasource": { |
||||
"type": "plugin-x", |
||||
"uid": "456" |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,20 @@ |
||||
{ |
||||
"description": "self dependencies", |
||||
"input": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "", |
||||
"uid": "__expr__" |
||||
}, |
||||
"type": "math", |
||||
"expression": "$A" |
||||
} |
||||
] |
||||
}, |
||||
"expect": {}, |
||||
"error": "expression [A] can not depend on itself" |
||||
} |
||||
@ -0,0 +1,79 @@ |
||||
{ |
||||
"description": "one hidden query with two expressions that start out-of-order", |
||||
"input": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "C", |
||||
"datasource": { |
||||
"type": "", |
||||
"uid": "__expr__" |
||||
}, |
||||
"type": "reduce", |
||||
"expression": "$B", |
||||
"reducer": "last" |
||||
}, |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "sql", |
||||
"uid": "123" |
||||
}, |
||||
"hide": true |
||||
}, |
||||
{ |
||||
"refId": "B", |
||||
"datasource": { |
||||
"type": "", |
||||
"uid": "-100" |
||||
}, |
||||
"type": "math", |
||||
"expression": "$A + 10" |
||||
} |
||||
] |
||||
}, |
||||
"expect": { |
||||
"requests": [ |
||||
{ |
||||
"pluginId": "sql", |
||||
"uid": "123", |
||||
"request": { |
||||
"from": "now-6", |
||||
"to": "now", |
||||
"queries": [ |
||||
{ |
||||
"refId": "A", |
||||
"datasource": { |
||||
"type": "sql", |
||||
"uid": "123" |
||||
} |
||||
} |
||||
] |
||||
} |
||||
} |
||||
], |
||||
"expressions": [ |
||||
{ |
||||
"id": 2, |
||||
"refId": "B", |
||||
"type": "math", |
||||
"properties": { |
||||
"expression": "$A + 10" |
||||
} |
||||
}, |
||||
{ |
||||
"id": 1, |
||||
"refId": "C", |
||||
"type": "reduce", |
||||
"properties": { |
||||
"expression": "$B", |
||||
"reducer": "last" |
||||
} |
||||
} |
||||
], |
||||
"hide": [ |
||||
"A" |
||||
] |
||||
} |
||||
} |
||||
@ -0,0 +1,90 @@ |
||||
package service |
||||
|
||||
import ( |
||||
"context" |
||||
"errors" |
||||
"fmt" |
||||
"sync" |
||||
|
||||
data "github.com/grafana/grafana-plugin-sdk-go/experimental/apis/data/v0alpha1" |
||||
|
||||
"github.com/grafana/grafana/pkg/infra/appcontext" |
||||
"github.com/grafana/grafana/pkg/services/datasources" |
||||
) |
||||
|
||||
// LegacyDataSourceRetriever supports finding a reference to datasources using the name or internal ID
|
||||
type LegacyDataSourceLookup interface { |
||||
// Find the UID from either the name or internal id
|
||||
// NOTE the orgID will be fetched from the context
|
||||
GetDataSourceFromDeprecatedFields(ctx context.Context, name string, id int64) (*data.DataSourceRef, error) |
||||
} |
||||
|
||||
var ( |
||||
_ DataSourceRetriever = (*Service)(nil) |
||||
_ LegacyDataSourceLookup = (*cachingLegacyDataSourceLookup)(nil) |
||||
_ LegacyDataSourceLookup = (*NoopLegacyDataSourcLookup)(nil) |
||||
) |
||||
|
||||
// NoopLegacyDataSourceRetriever does not even try to lookup, it returns a raw reference
|
||||
type NoopLegacyDataSourcLookup struct { |
||||
Ref *data.DataSourceRef |
||||
} |
||||
|
||||
func (s *NoopLegacyDataSourcLookup) GetDataSourceFromDeprecatedFields(ctx context.Context, name string, id int64) (*data.DataSourceRef, error) { |
||||
return s.Ref, nil |
||||
} |
||||
|
||||
type cachingLegacyDataSourceLookup struct { |
||||
retriever DataSourceRetriever |
||||
cache map[string]cachedValue |
||||
cacheMu sync.Mutex |
||||
} |
||||
|
||||
type cachedValue struct { |
||||
ref *data.DataSourceRef |
||||
err error |
||||
} |
||||
|
||||
func ProvideLegacyDataSourceLookup(p *Service) LegacyDataSourceLookup { |
||||
return &cachingLegacyDataSourceLookup{ |
||||
retriever: p, |
||||
cache: make(map[string]cachedValue), |
||||
} |
||||
} |
||||
|
||||
func (s *cachingLegacyDataSourceLookup) GetDataSourceFromDeprecatedFields(ctx context.Context, name string, id int64) (*data.DataSourceRef, error) { |
||||
if id == 0 && name == "" { |
||||
return nil, fmt.Errorf("either name or ID must be set") |
||||
} |
||||
user, err := appcontext.User(ctx) |
||||
if err != nil { |
||||
return nil, err |
||||
} |
||||
key := fmt.Sprintf("%d/%s/%d", user.OrgID, name, id) |
||||
s.cacheMu.Lock() |
||||
defer s.cacheMu.Unlock() |
||||
|
||||
v, ok := s.cache[key] |
||||
if ok { |
||||
return v.ref, v.err |
||||
} |
||||
|
||||
ds, err := s.retriever.GetDataSource(ctx, &datasources.GetDataSourceQuery{ |
||||
OrgID: user.OrgID, |
||||
Name: name, |
||||
ID: id, |
||||
}) |
||||
if errors.Is(err, datasources.ErrDataSourceNotFound) && name != "" { |
||||
ds, err = s.retriever.GetDataSource(ctx, &datasources.GetDataSourceQuery{ |
||||
OrgID: user.OrgID, |
||||
UID: name, // Sometimes name is actually the UID :(
|
||||
}) |
||||
} |
||||
v = cachedValue{ |
||||
err: err, |
||||
} |
||||
if ds != nil { |
||||
v.ref = &data.DataSourceRef{Type: ds.Type, UID: ds.UID} |
||||
} |
||||
return v.ref, v.err |
||||
} |
||||
Loading…
Reference in new issue