mirror of https://github.com/grafana/loki
Signed-off-by: Ganesh Vernekar <cs15btech11018@iith.ac.in>pull/969/head
Ganesh Vernekar
6 years ago
committed by
Cyril Tovena
parent
625c4f1f53
commit
184927ccdc
@ -0,0 +1,96 @@ |
||||
package logql |
||||
|
||||
import ( |
||||
"strconv" |
||||
"text/scanner" |
||||
"time" |
||||
) |
||||
|
||||
var tokens = map[string]int{ |
||||
",": COMMA, |
||||
".": DOT, |
||||
"{": OPEN_BRACE, |
||||
"}": CLOSE_BRACE, |
||||
"=": EQ, |
||||
"!=": NEQ, |
||||
"=~": RE, |
||||
"!~": NRE, |
||||
"|=": PIPE_EXACT, |
||||
"|~": PIPE_MATCH, |
||||
"(": OPEN_PARENTHESIS, |
||||
")": CLOSE_PARENTHESIS, |
||||
"by": BY, |
||||
"without": WITHOUT, |
||||
OpTypeCountOverTime: COUNT_OVER_TIME, |
||||
"[": OPEN_BRACKET, |
||||
"]": CLOSE_BRACKET, |
||||
OpTypeRate: RATE, |
||||
OpTypeSum: SUM, |
||||
OpTypeAvg: AVG, |
||||
OpTypeMax: MAX, |
||||
OpTypeMin: MIN, |
||||
OpTypeCount: COUNT, |
||||
OpTypeStddev: STDDEV, |
||||
OpTypeStdvar: STDVAR, |
||||
OpTypeBottomK: BOTTOMK, |
||||
OpTypeTopK: TOPK, |
||||
} |
||||
|
||||
type lexer struct { |
||||
scanner.Scanner |
||||
errs []ParseError |
||||
expr Expr |
||||
parser *exprParserImpl |
||||
} |
||||
|
||||
func (l *lexer) Lex(lval *exprSymType) int { |
||||
r := l.Scan() |
||||
switch r { |
||||
case scanner.EOF: |
||||
return 0 |
||||
|
||||
case scanner.String: |
||||
var err error |
||||
lval.str, err = strconv.Unquote(l.TokenText()) |
||||
if err != nil { |
||||
l.Error(err.Error()) |
||||
return 0 |
||||
} |
||||
return STRING |
||||
} |
||||
|
||||
// scaning duration tokens
|
||||
if l.TokenText() == "[" { |
||||
d := "" |
||||
for r := l.Next(); r != scanner.EOF; r = l.Next() { |
||||
if string(r) == "]" { |
||||
i, err := time.ParseDuration(d) |
||||
if err != nil { |
||||
l.Error(err.Error()) |
||||
return 0 |
||||
} |
||||
lval.duration = i |
||||
return DURATION |
||||
} |
||||
d += string(r) |
||||
} |
||||
l.Error("missing closing ']' in duration") |
||||
return 0 |
||||
} |
||||
|
||||
if tok, ok := tokens[l.TokenText()+string(l.Peek())]; ok { |
||||
l.Next() |
||||
return tok |
||||
} |
||||
|
||||
if tok, ok := tokens[l.TokenText()]; ok { |
||||
return tok |
||||
} |
||||
|
||||
lval.str = l.TokenText() |
||||
return IDENTIFIER |
||||
} |
||||
|
||||
func (l *lexer) Error(msg string) { |
||||
l.errs = append(l.errs, newParseError(msg, l.Line, l.Column)) |
||||
} |
||||
@ -0,0 +1,50 @@ |
||||
package logql |
||||
|
||||
import ( |
||||
"strings" |
||||
"testing" |
||||
"text/scanner" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestLex(t *testing.T) { |
||||
for _, tc := range []struct { |
||||
input string |
||||
expected []int |
||||
}{ |
||||
{`{foo="bar"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, |
||||
{`{ foo = "bar" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, |
||||
{`{ foo != "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NEQ, STRING, CLOSE_BRACE}}, |
||||
{`{ foo =~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, RE, STRING, CLOSE_BRACE}}, |
||||
{`{ foo !~ "bar" }`, []int{OPEN_BRACE, IDENTIFIER, NRE, STRING, CLOSE_BRACE}}, |
||||
{`{ foo = "bar", bar != "baz" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, |
||||
COMMA, IDENTIFIER, NEQ, STRING, CLOSE_BRACE}}, |
||||
{`{ foo = "ba\"r" }`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}}, |
||||
{`rate({foo="bar"}[10s])`, []int{RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, |
||||
{`count_over_time({foo="bar"}[5m])`, []int{COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS}}, |
||||
{`sum(count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, |
||||
{`topk(3,count_over_time({foo="bar"}[5m])) by (foo,bar)`, []int{TOPK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS}}, |
||||
{`bottomk(10,sum(count_over_time({foo="bar"}[5m])) by (foo,bar))`, []int{BOTTOMK, OPEN_PARENTHESIS, IDENTIFIER, COMMA, SUM, OPEN_PARENTHESIS, COUNT_OVER_TIME, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS}}, |
||||
{`sum(max(rate({foo="bar"}[5m])) by (foo,bar)) by (foo)`, []int{SUM, OPEN_PARENTHESIS, MAX, OPEN_PARENTHESIS, RATE, OPEN_PARENTHESIS, OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, DURATION, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, COMMA, IDENTIFIER, CLOSE_PARENTHESIS, CLOSE_PARENTHESIS, BY, OPEN_PARENTHESIS, IDENTIFIER, CLOSE_PARENTHESIS}}, |
||||
} { |
||||
t.Run(tc.input, func(t *testing.T) { |
||||
actual := []int{} |
||||
l := lexer{ |
||||
Scanner: scanner.Scanner{ |
||||
Mode: scanner.SkipComments | scanner.ScanStrings, |
||||
}, |
||||
} |
||||
l.Init(strings.NewReader(tc.input)) |
||||
var lval exprSymType |
||||
for { |
||||
tok := l.Lex(&lval) |
||||
if tok == 0 { |
||||
break |
||||
} |
||||
actual = append(actual, tok) |
||||
} |
||||
require.Equal(t, tc.expected, actual) |
||||
}) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue