apitech
/
loki
mirror of https://github.com/grafana/loki
1
0
Fork 0
Code Issues Projects Releases Wiki Activity

Fix `ip` matcher lexer to differentiate filter from identifier (#4598)

Browse Source 
* Fix `ip` matcher lexer to differentiate filter from identifier

Signed-off-by: Kaviraj <kavirajkanagaraj@gmail.com>

* Revert small fmt fix on expr.y

Signed-off-by: Kaviraj <kavirajkanagaraj@gmail.com>

* PR remarks from Cyril

Signed-off-by: Kaviraj <kavirajkanagaraj@gmail.com>

* Add ip line filter parser test in mix with ip as label

Signed-off-by: Kaviraj <kavirajkanagaraj@gmail.com>
pull/4394/head^2
Kaviraj 4 years ago committed by GitHub
parent 1c7df620c4
commit 1e0386e9e6
No known key found for this signature in database
GPG Key ID: 4AEE18F83AFDEB23
3 changed files with 84 additions and 1 deletions
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Show Stats Download Patch File Download Diff File
  1. 9
      pkg/logql/lex.go
  2. 6
      pkg/logql/lex_test.go
  3. 70
      pkg/logql/parser_test.go

9
pkg/logql/lex.go
Unescape View File

@ -107,6 +107,9 @@ var functionTokens = map[string]int{
OpConvBytes: BYTES_CONV,
OpConvDuration: DURATION_CONV,
OpConvDurationSeconds: DURATION_SECONDS_CONV,
// filterOp
OpFilterIP: IP,
}
type lexer struct {
@ -193,7 +196,11 @@ func (l *lexer) Lex(lval *exprSymType) int {
}
}
if tok, ok := functionTokens[tokenText]; ok && isFunction(l.Scanner) {
if tok, ok := functionTokens[tokenText]; ok {
if !isFunction(l.Scanner) {
lval.str = tokenText
return IDENTIFIER
}
return tok
}

6
pkg/logql/lex_test.go
Unescape View File

@ -52,6 +52,12 @@ func TestLex(t *testing.T) {
{`{foo="bar"} #|~ "\\w+"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
{`#{foo="bar"} |~ "\\w+"`, []int{}},
{`{foo="#"}`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE}},
{`{foo="bar"}|logfmt|ip="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}},
{`{foo="bar"}|logfmt|rate="b"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, STRING}},
{`{foo="bar"}|logfmt|b=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE, IDENTIFIER, EQ, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}},
{`{foo="bar"}|logfmt|=ip("b")`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, LOGFMT, PIPE_EXACT, IP, OPEN_PARENTHESIS, STRING, CLOSE_PARENTHESIS}},
{`ip`, []int{IDENTIFIER}},
{`rate`, []int{IDENTIFIER}},
{`{foo="bar"} | json | baz="#"`, []int{OPEN_BRACE, IDENTIFIER, EQ, STRING, CLOSE_BRACE, PIPE, JSON, PIPE, IDENTIFIER, EQ, STRING}},
{`{foo="bar"}
# |~ "\\w+"

70
pkg/logql/parser_test.go
Unescape View File

@ -139,6 +139,16 @@ func TestParse(t *testing.T) {
Operation: "rate",
},
},
{
in: `{ foo = "bar" }|logfmt|rate="a"`, // rate should also be able to use it as IDENTIFIER
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "rate", "a"))),
},
),
},
{
in: `rate({ foo = "bar" }[5d])`,
exp: &RangeAggregationExpr{
@ -351,6 +361,20 @@ func TestParse(t *testing.T) {
},
),
},
{
in: `{ foo = "bar" , ip="foo"}|logfmt|= ip("127.0.0.1")|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")|ip=ip("6.7.8.9")`,
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar"), mustNewMatcher(labels.MatchEqual, "ip", "foo")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLineFilterExpr(labels.MatchEqual, OpFilterIP, "127.0.0.1"),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))),
newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)),
newLabelFilterExpr(log.NewIPLabelFilter("6.7.8.9", "ip", log.LabelFilterEqual)),
},
),
},
{
in: `{foo="bar"} |= ip("123.123.123.123")`,
exp: newPipelineExpr(
@ -524,6 +548,52 @@ func TestParse(t *testing.T) {
},
),
},
{
in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"`, // just using `ip` as a label name(identifier) should work
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))),
},
),
},
{
in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"`, // just using `ip` as a label name should work with chaining
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))),
},
),
},
{
in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")`, // `ip` should work as both label name and filter in same query
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))),
newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)),
},
),
},
{
in: `{ foo = "bar" }|logfmt|ip="2.3.4.5"|ip="abc"|ipaddr=ip("4.5.6.7")|ip=ip("6.7.8.9")`, // `ip` should work as both label name and filter in same query with same name.
exp: newPipelineExpr(
newMatcherExpr([]*labels.Matcher{mustNewMatcher(labels.MatchEqual, "foo", "bar")}),
MultiStageExpr{
newLabelParserExpr(OpParserTypeLogfmt, ""),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "2.3.4.5"))),
newLabelFilterExpr(log.NewStringLabelFilter(mustNewMatcher(labels.MatchEqual, "ip", "abc"))),
newLabelFilterExpr(log.NewIPLabelFilter("4.5.6.7", "ipaddr", log.LabelFilterEqual)),
newLabelFilterExpr(log.NewIPLabelFilter("6.7.8.9", "ip", log.LabelFilterEqual)),
},
),
},
{
in: `sum(3 ,count_over_time({ foo = "bar" }[5h]))`,
err: logqlmodel.NewParseError("unsupported parameter for operation sum(3,", 0, 0),

Write Preview
Loading…
Cancel
Save