mirror of https://github.com/grafana/loki
Add a DeletionMode config variable (#5481)
* Add deletionEnabled setting and remove delete request manager Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Rebase Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Remove deletion handling from delete requests manager Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * change store so it stores a logql statement Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Add validation code for logql statement Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Run deleteRequestsManager when deletion is enabled Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Remove unused variables Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Revert "Remove deletion handling from delete requests manager" This reverts commit ce4f774497aa590caff86b0745ec81588592a9e1. * Re-add IsDeleted method Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Re-add tests for IsDeleted Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Fix delete request store test Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Fix linting issue Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Revert compactor changes Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Add deletion mode Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Add v1 mode Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Rename LogQLRequest to Query Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Fix linting issues Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Use DeleteMode in compactor module Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Rename logql to query Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Put cancel under delete verb Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Update documentation Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Update changelog Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Revert only the API surface area while keeping everything else * Use moved code in syntax package Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Remove duplicte import Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Use renamed field in tests Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Remove duplicates and empty lines in changelog Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Update changelog description Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Update pkg/storage/stores/shipper/compactor/deletion/delete_request.go Co-authored-by: Christian Simon <simon@swine.de> * Update pkg/storage/stores/shipper/compactor/deletion/delete_requests_manager_test.go Co-authored-by: Christian Simon <simon@swine.de> * Update pkg/storage/stores/shipper/compactor/deletion/delete_requests_store.go Co-authored-by: Christian Simon <simon@swine.de> * Update pkg/storage/stores/shipper/compactor/deletion/delete_requests_store.go Co-authored-by: Christian Simon <simon@swine.de> * Update CHANGELOG.md Co-authored-by: Christian Simon <simon@swine.de> * Update pkg/storage/stores/shipper/compactor/deletion/delete_requests_store.go Co-authored-by: Christian Simon <simon@swine.de> * Make DeletionMode struct member non public Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Revert change to docs re cancellation Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Use same variable names Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Add parameter validation to changelog Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Rename v1 to WholeStreamDeletion Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Fix default value of deletion mode config setting Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * reimplement new api * Add delete request handler when delete mode is set Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Remove unused variable Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Add comment to change the code when other deletion modes are available Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * create expirationChecker if deletionMode is set Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Address review comments Signed-off-by: Michel Hollands <michel.hollands@grafana.com> * Update pkg/storage/stores/shipper/compactor/compactor.go Co-authored-by: Sandeep Sukhani <sandeep.d.sukhani@gmail.com> * Rename AddQuery to SetQuery Signed-off-by: Michel Hollands <michel.hollands@grafana.com> Co-authored-by: Travis Patterson <travis.patterson@grafana.com> Co-authored-by: Christian Simon <simon@swine.de> Co-authored-by: Sandeep Sukhani <sandeep.d.sukhani@gmail.com>pull/5780/head
Michel Hollands
4 years ago
committed by
GitHub
parent
ece1fb5a34
commit
b865b81ba0
@ -0,0 +1,50 @@ |
||||
package deletion |
||||
|
||||
import ( |
||||
"errors" |
||||
) |
||||
|
||||
type Mode int16 |
||||
|
||||
var ( |
||||
errUnknownMode = errors.New("unknown deletion mode") |
||||
) |
||||
|
||||
const ( |
||||
Disabled Mode = iota |
||||
WholeStreamDeletion // The existing log deletion that removes whole streams.
|
||||
FilterOnly |
||||
FilterAndDelete |
||||
) |
||||
|
||||
func (m Mode) String() string { |
||||
switch m { |
||||
case Disabled: |
||||
return "disabled" |
||||
case WholeStreamDeletion: |
||||
return "whole-stream-deletion" |
||||
case FilterOnly: |
||||
return "filter-only" |
||||
case FilterAndDelete: |
||||
return "filter-and-delete" |
||||
} |
||||
return "unknown" |
||||
} |
||||
|
||||
func AllModes() []string { |
||||
return []string{Disabled.String(), WholeStreamDeletion.String(), FilterOnly.String(), FilterAndDelete.String()} |
||||
} |
||||
|
||||
func ParseMode(in string) (Mode, error) { |
||||
switch in { |
||||
case "disabled": |
||||
return Disabled, nil |
||||
case "whole-stream-deletion": |
||||
return WholeStreamDeletion, nil |
||||
case "filter-only": |
||||
return FilterOnly, nil |
||||
case "filter-and-delete": |
||||
return FilterAndDelete, nil |
||||
} |
||||
return 0, errUnknownMode |
||||
} |
||||
@ -0,0 +1,33 @@ |
||||
package deletion |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestAllModes(t *testing.T) { |
||||
modes := AllModes() |
||||
require.ElementsMatch(t, []string{"disabled", "whole-stream-deletion", "filter-only", "filter-and-delete"}, modes) |
||||
} |
||||
|
||||
func TestParseMode(t *testing.T) { |
||||
mode, err := ParseMode("disabled") |
||||
require.NoError(t, err) |
||||
require.Equal(t, Disabled, mode) |
||||
|
||||
mode, err = ParseMode("whole-stream-deletion") |
||||
require.NoError(t, err) |
||||
require.Equal(t, WholeStreamDeletion, mode) |
||||
|
||||
mode, err = ParseMode("filter-only") |
||||
require.NoError(t, err) |
||||
require.Equal(t, FilterOnly, mode) |
||||
|
||||
mode, err = ParseMode("filter-and-delete") |
||||
require.NoError(t, err) |
||||
require.Equal(t, FilterAndDelete, mode) |
||||
|
||||
_, err = ParseMode("something-else") |
||||
require.ErrorIs(t, errUnknownMode, err) |
||||
} |
||||
@ -0,0 +1,28 @@ |
||||
package deletion |
||||
|
||||
import ( |
||||
"errors" |
||||
|
||||
"github.com/prometheus/prometheus/model/labels" |
||||
|
||||
"github.com/grafana/loki/pkg/logql/syntax" |
||||
) |
||||
|
||||
var ( |
||||
errInvalidQuery = errors.New("invalid query expression") |
||||
errUnsupportedQuery = errors.New("unsupported query expression") |
||||
) |
||||
|
||||
// parseDeletionQuery checks if the given logQL is valid for deletions
|
||||
func parseDeletionQuery(query string) ([]*labels.Matcher, error) { |
||||
expr, err := syntax.ParseExpr(query) |
||||
if err != nil { |
||||
return nil, errInvalidQuery |
||||
} |
||||
|
||||
if matchersExpr, ok := expr.(*syntax.MatchersExpr); ok { |
||||
return matchersExpr.Matchers(), nil |
||||
} |
||||
|
||||
return nil, errUnsupportedQuery |
||||
} |
||||
@ -0,0 +1,39 @@ |
||||
package deletion |
||||
|
||||
import ( |
||||
"testing" |
||||
|
||||
"github.com/stretchr/testify/require" |
||||
) |
||||
|
||||
func TestParseLogQLExpressionForDeletion(t *testing.T) { |
||||
t.Run("invalid logql", func(t *testing.T) { |
||||
matchers, err := parseDeletionQuery("gjgjg ggj") |
||||
require.Nil(t, matchers) |
||||
require.ErrorIs(t, err, errInvalidQuery) |
||||
}) |
||||
|
||||
t.Run("matcher expression", func(t *testing.T) { |
||||
matchers, err := parseDeletionQuery(`{env="dev", secret="true"}`) |
||||
require.NotNil(t, matchers) |
||||
require.NoError(t, err) |
||||
}) |
||||
|
||||
t.Run("pipeline expression with line filter", func(t *testing.T) { |
||||
matchers, err := parseDeletionQuery(`{env="dev", secret="true"} |= "social sec number"`) |
||||
require.Nil(t, matchers) |
||||
require.ErrorIs(t, err, errUnsupportedQuery) |
||||
}) |
||||
|
||||
t.Run("pipeline expression with label filter ", func(t *testing.T) { |
||||
matchers, err := parseDeletionQuery(`{env="dev", secret="true"} | json bob="top.params[0]"`) |
||||
require.Nil(t, matchers) |
||||
require.ErrorIs(t, err, errUnsupportedQuery) |
||||
}) |
||||
|
||||
t.Run("metrics query", func(t *testing.T) { |
||||
matchers, err := parseDeletionQuery(`count_over_time({job="mysql"}[5m])`) |
||||
require.Nil(t, matchers) |
||||
require.ErrorIs(t, err, errUnsupportedQuery) |
||||
}) |
||||
} |
||||
Loading…
Reference in new issue