package deletion import ( "strings" "time" "github.com/go-kit/log/level" "github.com/pkg/errors" "github.com/prometheus/common/model" "github.com/prometheus/prometheus/model/labels" "github.com/grafana/loki/v3/pkg/compactor/retention" "github.com/grafana/loki/v3/pkg/logql/syntax" "github.com/grafana/loki/v3/pkg/util/filter" util_log "github.com/grafana/loki/v3/pkg/util/log" ) type timeInterval struct { start, end time.Time } type DeleteRequest struct { RequestID string `json:"request_id"` StartTime model.Time `json:"start_time"` EndTime model.Time `json:"end_time"` Query string `json:"query"` Status DeleteRequestStatus `json:"status"` CreatedAt model.Time `json:"created_at"` UserID string `json:"-"` SequenceNum int64 `json:"-"` matchers []*labels.Matcher `json:"-"` logSelectorExpr syntax.LogSelectorExpr `json:"-"` timeInterval *timeInterval `json:"-"` Metrics *deleteRequestsManagerMetrics `json:"-"` DeletedLines int32 `json:"-"` } func (d *DeleteRequest) SetQuery(logQL string) error { d.Query = logQL logSelectorExpr, err := parseDeletionQuery(logQL) if err != nil { return err } d.logSelectorExpr = logSelectorExpr d.matchers = logSelectorExpr.Matchers() return nil } // FilterFunction returns a filter function that returns true if the given line should be deleted based on the DeleteRequest func (d *DeleteRequest) FilterFunction(lbls labels.Labels) (filter.Func, error) { // init d.timeInterval used to efficiently check log ts is within the bounds of delete request below in filter func // without having to do conversion of timestamps for each log line we check. if d.timeInterval == nil { d.timeInterval = &timeInterval{ start: d.StartTime.Time(), end: d.EndTime.Time(), } } if !allMatch(d.matchers, lbls) { return func(_ time.Time, _ string, _ ...labels.Label) bool { return false }, nil } // if delete request doesn't have a line filter, just do time based filtering if !d.logSelectorExpr.HasFilter() { return func(ts time.Time, _ string, _ ...labels.Label) bool { if ts.Before(d.timeInterval.start) || ts.After(d.timeInterval.end) { return false } return true }, nil } p, err := d.logSelectorExpr.Pipeline() if err != nil { return nil, err } f := p.ForStream(lbls).ProcessString return func(ts time.Time, s string, structuredMetadata ...labels.Label) bool { if ts.Before(d.timeInterval.start) || ts.After(d.timeInterval.end) { return false } result, _, skip := f(0, s, structuredMetadata...) if len(result) != 0 || skip { d.Metrics.deletedLinesTotal.WithLabelValues(d.UserID).Inc() d.DeletedLines++ return true } return false }, nil } func allMatch(matchers []*labels.Matcher, labels labels.Labels) bool { for _, m := range matchers { if !m.Matches(labels.Get(m.Name)) { return false } } return true } // IsDeleted checks if the given ChunkEntry will be deleted by this DeleteRequest. // It returns a filter.Func if the chunk is supposed to be deleted partially or the delete request contains line filters. // If the filter.Func is nil, the whole chunk is supposed to be deleted. func (d *DeleteRequest) IsDeleted(entry retention.ChunkEntry) (bool, filter.Func) { if d.UserID != unsafeGetString(entry.UserID) { return false, nil } if !intervalsOverlap(model.Interval{ Start: entry.From, End: entry.Through, }, model.Interval{ Start: d.StartTime, End: d.EndTime, }) { return false, nil } if d.logSelectorExpr == nil { err := d.SetQuery(d.Query) if err != nil { level.Error(util_log.Logger).Log( "msg", "failed to init log selector expr", "delete_request_id", d.RequestID, "user", d.UserID, "err", err, ) return false, nil } } if !labels.Selector(d.matchers).Matches(entry.Labels) { return false, nil } if d.StartTime <= entry.From && d.EndTime >= entry.Through && !d.logSelectorExpr.HasFilter() { // Delete request covers the whole chunk and there are no line filters in the logSelectorExpr so the whole chunk will be deleted return true, nil } ff, err := d.FilterFunction(entry.Labels) if err != nil { // The query in the delete request is checked when added to the table. // So this error should not occur. level.Error(util_log.Logger).Log( "msg", "unexpected error getting filter function", "delete_request_id", d.RequestID, "user", d.UserID, "err", err, ) return false, nil } return true, ff } func (d *DeleteRequest) IsDuplicate(o *DeleteRequest) (bool, error) { // we would never have duplicates from same request if d.RequestID == o.RequestID { return false, nil } if d.UserID != o.UserID || d.StartTime != o.StartTime || d.EndTime != o.EndTime { return false, nil } if d.logSelectorExpr == nil { if err := d.SetQuery(d.Query); err != nil { return false, errors.Wrapf(err, "failed to init log selector expr for request_id=%s, user_id=%s", d.RequestID, d.UserID) } } if o.logSelectorExpr == nil { if err := o.SetQuery(o.Query); err != nil { return false, errors.Wrapf(err, "failed to init log selector expr for request_id=%s, user_id=%s", o.RequestID, o.UserID) } } if d.logSelectorExpr.String() != o.logSelectorExpr.String() { return false, nil } return true, nil } func intervalsOverlap(interval1, interval2 model.Interval) bool { if interval1.Start > interval2.End || interval2.Start > interval1.End { return false } return true } // GetMatchers returns the string representation of the matchers func (d *DeleteRequest) GetMatchers() string { if len(d.matchers) == 0 { return "" } var result []string for _, m := range d.matchers { result = append(result, m.String()) } return strings.Join(result, ",") }