apitech
/
prometheus
mirror of https://github.com/prometheus/prometheus
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

PromQL: Add position metadata to the AST (#6615)

Browse Source 
Signed-off-by: Tobias Guggenmos <tguggenm@redhat.com>
Co-authored-by: Julius Volz <julius.volz@gmail.com>
pull/6557/head
Tobias Guggenmos 6 years ago committed by Brian Brazil
parent e12e5ecc8f
commit 0c8e9ef09e
7 changed files with 1390 additions and 443 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 101
      promql/ast.go
  2. 7
      promql/engine.go
  3. 84
      promql/generated_parser.y
  4. 624
      promql/generated_parser.y.go
  5. 1
      promql/lex.go
  6. 44
      promql/parse.go
  7. 972
      promql/parse_test.go

101
promql/ast.go
Unescape View File

@ -39,6 +39,9 @@ type Node interface {
// String representation of the node that returns the given node when parsed
// as part of a valid query.
String() string
// PositionRange returns the position of the AST Node in the query string.
PositionRange() PositionRange
}
// Statement is a generic interface for all statements.
@ -84,6 +87,7 @@ type AggregateExpr struct {
Param Expr // Parameter used by some aggregators.
Grouping []string // The labels by which to group the Vector.
Without bool // Whether to drop the given labels rather than keep them.
PosRange PositionRange
}
// BinaryExpr represents a binary expression between two child expressions.
@ -103,12 +107,16 @@ type BinaryExpr struct {
type Call struct {
Func *Function // The function that was called.
Args Expressions // Arguments used in the call.
PosRange PositionRange
}
// MatrixSelector represents a Matrix selection.
type MatrixSelector struct {
VectorSelector *VectorSelector
Range time.Duration
EndPos Pos
}
// SubqueryExpr represents a subquery.
@ -117,22 +125,28 @@ type SubqueryExpr struct {
Range time.Duration
Offset time.Duration
Step time.Duration
EndPos Pos
}
// NumberLiteral represents a number.
type NumberLiteral struct {
Val float64
PosRange PositionRange
}
// ParenExpr wraps an expression so it cannot be disassembled as a consequence
// of operator precedence.
type ParenExpr struct {
Expr Expr
Expr Expr
PosRange PositionRange
}
// StringLiteral represents a string.
type StringLiteral struct {
Val string
Val string
PosRange PositionRange
}
// UnaryExpr represents a unary operation on another expression.
@ -140,6 +154,8 @@ type StringLiteral struct {
type UnaryExpr struct {
Op ItemType
Expr Expr
StartPos Pos
}
// VectorSelector represents a Vector selection.
@ -151,6 +167,8 @@ type VectorSelector struct {
// The unexpanded seriesSet populated at query preparation time.
unexpandedSeriesSet storage.SeriesSet
series []storage.Series
PosRange PositionRange
}
func (e *AggregateExpr) Type() ValueType { return ValueTypeVector }
@ -319,3 +337,82 @@ func Children(node Node) []Node {
panic(errors.Errorf("promql.Children: unhandled node type %T", node))
}
}
// PositionRange describes a position in the input string of the parser.
type PositionRange struct {
Start Pos
End Pos
}
// mergeRanges is a helper function to merge the PositionRanges of two Nodes.
// Note that the arguments must be in the same order as they
// occur in the input string.
func mergeRanges(first Node, last Node) PositionRange {
return PositionRange{
Start: first.PositionRange().Start,
End: last.PositionRange().End,
}
}
// Item implements the Node interface.
// This makes it possible to call mergeRanges on them.
func (i *Item) PositionRange() PositionRange {
return PositionRange{
Start: i.Pos,
End: i.Pos + Pos(len(i.Val)),
}
}
func (e *AggregateExpr) PositionRange() PositionRange {
return e.PosRange
}
func (e *BinaryExpr) PositionRange() PositionRange {
return mergeRanges(e.LHS, e.RHS)
}
func (e *Call) PositionRange() PositionRange {
return e.PosRange
}
func (e *EvalStmt) PositionRange() PositionRange {
return e.Expr.PositionRange()
}
func (e Expressions) PositionRange() PositionRange {
if len(e) == 0 {
// Position undefined.
return PositionRange{
Start: -1,
End: -1,
}
} else {
return mergeRanges(e[0], e[len(e)-1])
}
}
func (e *MatrixSelector) PositionRange() PositionRange {
return PositionRange{
Start: e.VectorSelector.PositionRange().Start,
End: e.EndPos,
}
}
func (e *SubqueryExpr) PositionRange() PositionRange {
return PositionRange{
Start: e.Expr.PositionRange().Start,
End: e.EndPos,
}
}
func (e *NumberLiteral) PositionRange() PositionRange {
return e.PosRange
}
func (e *ParenExpr) PositionRange() PositionRange {
return e.PosRange
}
func (e *StringLiteral) PositionRange() PositionRange {
return e.PosRange
}
func (e *UnaryExpr) PositionRange() PositionRange {
return PositionRange{
Start: e.StartPos,
End: e.Expr.PositionRange().End,
}
}
func (e *VectorSelector) PositionRange() PositionRange {
return e.PosRange
}

7
promql/engine.go
Unescape View File

@ -406,6 +406,13 @@ type testStmt func(context.Context) error
func (testStmt) String() string { return "test statement" }
func (testStmt) stmt() {}
func (testStmt) PositionRange() PositionRange {
return PositionRange{
Start: -1,
End: -1,
}
}
func (ng *Engine) newTestQuery(f func(context.Context) error) Query {
qry := &query{
q: "test statement",

84
promql/generated_parser.y
Unescape View File

@ -36,7 +36,6 @@ import (
series []sequenceValue
uint uint64
float float64
string string
duration time.Duration
}
@ -127,7 +126,7 @@ START_METRIC_SELECTOR
// Type definitions for grammar rules.
%type <matchers> label_match_list label_matchers
%type <matchers> label_match_list
%type <matcher> label_matcher
%type <item> aggregate_op grouping_label match_op maybe_label metric_identifier unary_op
@ -138,8 +137,7 @@ START_METRIC_SELECTOR
%type <series> series_item series_values
%type <uint> uint
%type <float> number series_value signed_number
%type <node> aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers matrix_selector number_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
%type <string> string
%type <node> aggregate_expr aggregate_modifier bin_modifier binary_expr bool_modifier expr function_call function_call_args function_call_body group_modifiers label_matchers matrix_selector number_literal offset_expr on_or_ignoring paren_expr string_literal subquery_expr unary_expr vector_selector
%type <duration> duration maybe_duration
%start start
@ -333,6 +331,10 @@ function_call : IDENTIFIER function_call_body
$$ = &Call{
Func: fn,
Args: $2.(Expressions),
PosRange: PositionRange{
Start: $1.Pos,
End: yylex.(*parser).lastClosing,
},
}
}
;
@ -354,7 +356,7 @@ function_call_args: function_call_args COMMA expr
*/
paren_expr : LEFT_PAREN expr RIGHT_PAREN
{ $$ = &ParenExpr{Expr: $2.(Expr)} }
{ $$ = &ParenExpr{Expr: $2.(Expr), PosRange: mergeRanges(&$1, &$3)} }
;
/*
@ -386,6 +388,7 @@ matrix_selector : expr LEFT_BRACKET duration RIGHT_BRACKET
$$ = &MatrixSelector{
VectorSelector: vs,
Range: $3,
EndPos: yylex.(*parser).lastClosing,
}
}
;
@ -396,6 +399,8 @@ subquery_expr : expr LEFT_BRACKET duration COLON maybe_duration RIGHT_BRACKET
Expr: $1.(Expr),
Range: $3,
Step: $5,
EndPos: $6.Pos + 1,
}
}
| expr LEFT_BRACKET duration COLON duration error
@ -420,9 +425,10 @@ unary_expr :
if $1.Typ == SUB {
nl.Val *= -1
}
nl.PosRange.Start = $1.Pos
$$ = nl
} else {
$$ = &UnaryExpr{Op: $1.Typ, Expr: $2.(Expr)}
$$ = &UnaryExpr{Op: $1.Typ, Expr: $2.(Expr), StartPos: $1.Pos}
}
}
;
@ -432,20 +438,52 @@ unary_expr :
*/
vector_selector: metric_identifier label_matchers
{ $$ = yylex.(*parser).newVectorSelector($1.Val, $2) }
{
vs := $2.(*VectorSelector)
vs.PosRange = mergeRanges(&$1, vs)
vs.Name = $1.Val
yylex.(*parser).assembleVectorSelector(vs)
$$ = vs
}
| metric_identifier
{ $$ = yylex.(*parser).newVectorSelector($1.Val, nil) }
{
vs := &VectorSelector{
Name: $1.Val,
LabelMatchers: []*labels.Matcher{},
PosRange: $1.PositionRange(),
}
yylex.(*parser).assembleVectorSelector(vs)
$$ = vs
}
| label_matchers
{ $$ = yylex.(*parser).newVectorSelector("", $1) }
{
vs := $1.(*VectorSelector)
yylex.(*parser).assembleVectorSelector(vs)
$$ = vs
}
;
label_matchers : LEFT_BRACE label_match_list RIGHT_BRACE
{ $$ = $2 }
{
$$ = &VectorSelector{
LabelMatchers: $2,
PosRange: mergeRanges(&$1, &$3),
}
}
| LEFT_BRACE label_match_list COMMA RIGHT_BRACE
{ $$ = $2 }
{
$$ = &VectorSelector{
LabelMatchers: $2,
PosRange: mergeRanges(&$1, &$4),
}
}
| LEFT_BRACE RIGHT_BRACE
{ $$ = []*labels.Matcher{} }
{
$$ = &VectorSelector{
LabelMatchers: []*labels.Matcher{},
PosRange: mergeRanges(&$1, &$2),
}
}
;
label_match_list: label_match_list COMMA label_matcher
@ -590,7 +628,14 @@ match_op : EQL | NEQ | EQL_REGEX | NEQ_REGEX ;
* Literals.
*/
number_literal : number { $$ = &NumberLiteral{$1}} ;
number_literal : NUMBER
{
$$ = &NumberLiteral{
Val: yylex.(*parser).number($1.Val),
PosRange: $1.PositionRange(),
}
}
;
number : NUMBER { $$ = yylex.(*parser).number($1.Val) } ;
@ -619,9 +664,14 @@ duration : DURATION
;
string_literal : string { $$ = &StringLiteral{$1}} ;
string : STRING { $$ = yylex.(*parser).unquoteString($1.Val) } ;
string_literal : STRING
{
$$ = &StringLiteral{
Val: yylex.(*parser).unquoteString($1.Val),
PosRange: $1.PositionRange(),
}
}
;
/*
* Wrappers for optional arguments.

624
promql/generated_parser.y.go
View File

File diff suppressed because it is too large Load Diff

1
promql/lex.go
Unescape View File

@ -213,6 +213,7 @@ const eof = -1
type stateFn func(*Lexer) stateFn
// Pos is the position in a string.
// Negative numbers indicate undefined positions.
type Pos int
// Lexer holds the state of the scanner.

44
promql/parse.go
Unescape View File

@ -41,6 +41,10 @@ type parser struct {
inject ItemType
injecting bool
// Everytime an Item is lexed that could be the end
// of certain expressions its end position is stored here.
lastClosing Pos
yyParser yyParserImpl
generatedParserResult interface{}
@ -229,13 +233,15 @@ func (p *parser) Lex(lval *yySymType) int {
}
}
if typ == ERROR {
p.errorf("%s", lval.item.Val)
}
switch typ {
if typ == EOF {
case ERROR:
p.errorf("%s", lval.item.Val)
case EOF:
lval.item.Typ = EOF
p.InjectItem(0)
case RIGHT_BRACE, RIGHT_PAREN, RIGHT_BRACKET, DURATION:
p.lastClosing = lval.item.Pos + Pos(len(lval.item.Val))
}
return int(typ)
@ -298,29 +304,26 @@ func (p *parser) newBinaryExpression(lhs Node, op Item, modifiers Node, rhs Node
return ret
}
func (p *parser) newVectorSelector(name string, labelMatchers []*labels.Matcher) *VectorSelector {
ret := &VectorSelector{LabelMatchers: labelMatchers}
if name != "" {
ret.Name = name
func (p *parser) assembleVectorSelector(vs *VectorSelector) {
if vs.Name != "" {
for _, m := range ret.LabelMatchers {
for _, m := range vs.LabelMatchers {
if m.Name == labels.MetricName {
p.errorf("metric name must not be set twice: %q or %q", name, m.Value)
p.errorf("metric name must not be set twice: %q or %q", vs.Name, m.Value)
}
}
nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, name)
nameMatcher, err := labels.NewMatcher(labels.MatchEqual, labels.MetricName, vs.Name)
if err != nil {
panic(err) // Must not happen with labels.MatchEqual
}
ret.LabelMatchers = append(ret.LabelMatchers, nameMatcher)
vs.LabelMatchers = append(vs.LabelMatchers, nameMatcher)
}
// A Vector selector must contain at least one non-empty matcher to prevent
// implicit selection of all metrics (e.g. by a typo).
notEmpty := false
for _, lm := range ret.LabelMatchers {
for _, lm := range vs.LabelMatchers {
if !lm.Matches("") {
notEmpty = true
break
@ -329,8 +332,6 @@ func (p *parser) newVectorSelector(name string, labelMatchers []*labels.Matcher)
if !notEmpty {
p.errorf("vector selector must contain at least one non-empty matcher")
}
return ret
}
func (p *parser) newAggregateExpr(op Item, modifier Node, args Node) (ret *AggregateExpr) {
@ -364,6 +365,11 @@ func (p *parser) newAggregateExpr(op Item, modifier Node, args Node) (ret *Aggre
ret.Expr = arguments[desiredArgs-1]
ret.PosRange = PositionRange{
Start: op.Pos,
End: p.lastClosing,
}
return ret
}
@ -578,14 +584,18 @@ func (p *parser) newLabelMatcher(label Item, operator Item, value Item) *labels.
func (p *parser) addOffset(e Node, offset time.Duration) {
var offsetp *time.Duration
var endPosp *Pos
switch s := e.(type) {
case *VectorSelector:
offsetp = &s.Offset
endPosp = &s.PosRange.End
case *MatrixSelector:
offsetp = &s.VectorSelector.Offset
endPosp = &s.EndPos
case *SubqueryExpr:
offsetp = &s.Offset
endPosp = &s.EndPos
default:
p.errorf("offset modifier must be preceded by an instant or range selector, but follows a %T instead", e)
return
@ -598,4 +608,6 @@ func (p *parser) addOffset(e Node, offset time.Duration) {
*offsetp = offset
}
*endPosp = p.lastClosing
}

972
promql/parse_test.go
View File

File diff suppressed because it is too large Load Diff
Write Preview
Loading…
Cancel
Save