mirror of https://github.com/grafana/loki
chore(bloom): remove unused code from blooms (#14539)
Robert Fratto
2 years ago
committed by
GitHub
parent
dbb3b6edc9
commit
bf54cf1496
@ -1,125 +0,0 @@ |
||||
package planner |
||||
|
||||
import ( |
||||
"fmt" |
||||
"math" |
||||
|
||||
"github.com/prometheus/common/model" |
||||
|
||||
v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" |
||||
) |
||||
|
||||
// SplitFingerprintKeyspaceByFactor splits the keyspace covered by model.Fingerprint into contiguous non-overlapping ranges.
|
||||
func SplitFingerprintKeyspaceByFactor(factor int) []v1.FingerprintBounds { |
||||
if factor <= 0 { |
||||
return nil |
||||
} |
||||
|
||||
bounds := make([]v1.FingerprintBounds, 0, factor) |
||||
|
||||
// The keyspace of a Fingerprint is from 0 to max uint64.
|
||||
keyspaceSize := uint64(math.MaxUint64) |
||||
|
||||
// Calculate the size of each range.
|
||||
rangeSize := keyspaceSize / uint64(factor) |
||||
|
||||
for i := 0; i < factor; i++ { |
||||
// Calculate the start and end of the range.
|
||||
start := uint64(i) * rangeSize |
||||
end := start + rangeSize - 1 |
||||
|
||||
// For the last range, make sure it ends at the end of the keyspace.
|
||||
if i == factor-1 { |
||||
end = keyspaceSize |
||||
} |
||||
|
||||
// Create a FingerprintBounds for the range and add it to the slice.
|
||||
bounds = append(bounds, v1.FingerprintBounds{ |
||||
Min: model.Fingerprint(start), |
||||
Max: model.Fingerprint(end), |
||||
}) |
||||
} |
||||
|
||||
return bounds |
||||
} |
||||
|
||||
func FindGapsInFingerprintBounds(ownershipRange v1.FingerprintBounds, metas []v1.FingerprintBounds) (gaps []v1.FingerprintBounds, err error) { |
||||
if len(metas) == 0 { |
||||
return []v1.FingerprintBounds{ownershipRange}, nil |
||||
} |
||||
|
||||
// turn the available metas into a list of non-overlapping metas
|
||||
// for easier processing
|
||||
var nonOverlapping []v1.FingerprintBounds |
||||
// First, we reduce the metas into a smaller set by combining overlaps. They must be sorted.
|
||||
var cur *v1.FingerprintBounds |
||||
for i := 0; i < len(metas); i++ { |
||||
j := i + 1 |
||||
|
||||
// first iteration (i == 0), set the current meta
|
||||
if cur == nil { |
||||
cur = &metas[i] |
||||
} |
||||
|
||||
if j >= len(metas) { |
||||
// We've reached the end of the list. Add the last meta to the non-overlapping set.
|
||||
nonOverlapping = append(nonOverlapping, *cur) |
||||
break |
||||
} |
||||
|
||||
combined := cur.Union(metas[j]) |
||||
if len(combined) == 1 { |
||||
// There was an overlap between the two tested ranges. Combine them and keep going.
|
||||
cur = &combined[0] |
||||
continue |
||||
} |
||||
|
||||
// There was no overlap between the two tested ranges. Add the first to the non-overlapping set.
|
||||
// and keep the second for the next iteration.
|
||||
nonOverlapping = append(nonOverlapping, combined[0]) |
||||
cur = &combined[1] |
||||
} |
||||
|
||||
// Now, detect gaps between the non-overlapping metas and the ownership range.
|
||||
// The left bound of the ownership range will be adjusted as we go.
|
||||
leftBound := ownershipRange.Min |
||||
for _, meta := range nonOverlapping { |
||||
|
||||
clippedMeta := meta.Intersection(ownershipRange) |
||||
// should never happen as long as we are only combining metas
|
||||
// that intersect with the ownership range
|
||||
if clippedMeta == nil { |
||||
return nil, fmt.Errorf("meta is not within ownership range: %v", meta) |
||||
} |
||||
|
||||
searchRange := ownershipRange.Slice(leftBound, clippedMeta.Max) |
||||
// update the left bound for the next iteration
|
||||
// We do the max to prevent the max bound to overflow from MaxUInt64 to 0
|
||||
leftBound = min( |
||||
max(clippedMeta.Max+1, clippedMeta.Max), |
||||
max(ownershipRange.Max+1, ownershipRange.Max), |
||||
) |
||||
|
||||
// since we've already ensured that the meta is within the ownership range,
|
||||
// we know the xor will be of length zero (when the meta is equal to the ownership range)
|
||||
// or 1 (when the meta is a subset of the ownership range)
|
||||
xors := searchRange.Unless(*clippedMeta) |
||||
if len(xors) == 0 { |
||||
// meta is equal to the ownership range. This means the meta
|
||||
// covers this entire section of the ownership range.
|
||||
continue |
||||
} |
||||
|
||||
gaps = append(gaps, xors[0]) |
||||
} |
||||
|
||||
// If the leftBound is less than the ownership range max, and it's smaller than MaxUInt64,
|
||||
// There is a gap between the last meta and the end of the ownership range.
|
||||
// Note: we check `leftBound < math.MaxUint64` since in the loop above we clamp the
|
||||
// leftBound to MaxUint64 to prevent an overflow to 0: `max(clippedMeta.Max+1, clippedMeta.Max)`
|
||||
if leftBound < math.MaxUint64 && leftBound <= ownershipRange.Max { |
||||
gaps = append(gaps, v1.NewBounds(leftBound, ownershipRange.Max)) |
||||
} |
||||
|
||||
return gaps, nil |
||||
} |
||||
@ -1,172 +0,0 @@ |
||||
package planner |
||||
|
||||
import ( |
||||
"math" |
||||
"testing" |
||||
|
||||
"github.com/prometheus/common/model" |
||||
"github.com/stretchr/testify/require" |
||||
|
||||
v1 "github.com/grafana/loki/v3/pkg/storage/bloom/v1" |
||||
) |
||||
|
||||
func TestSplitFingerprintKeyspaceByFactor(t *testing.T) { |
||||
for _, tt := range []struct { |
||||
name string |
||||
factor int |
||||
}{ |
||||
{ |
||||
name: "Factor is 0", |
||||
factor: 0, |
||||
}, |
||||
{ |
||||
name: "Factor is 1", |
||||
factor: 1, |
||||
}, |
||||
{ |
||||
name: "Factor is 256", |
||||
factor: 256, |
||||
}, |
||||
} { |
||||
t.Run(tt.name, func(t *testing.T) { |
||||
got := SplitFingerprintKeyspaceByFactor(tt.factor) |
||||
|
||||
if tt.factor == 0 { |
||||
require.Empty(t, got) |
||||
return |
||||
} |
||||
|
||||
// Check overall min and max values of the ranges.
|
||||
require.Equal(t, model.Fingerprint(math.MaxUint64), got[len(got)-1].Max) |
||||
require.Equal(t, model.Fingerprint(0), got[0].Min) |
||||
|
||||
// For each range, check that the max value of the previous range is one less than the min value of the current range.
|
||||
for i := 1; i < len(got); i++ { |
||||
require.Equal(t, got[i-1].Max+1, got[i].Min) |
||||
} |
||||
}) |
||||
} |
||||
} |
||||
|
||||
func Test_FindGapsInFingerprintBounds(t *testing.T) { |
||||
for _, tc := range []struct { |
||||
desc string |
||||
err bool |
||||
exp []v1.FingerprintBounds |
||||
ownershipRange v1.FingerprintBounds |
||||
metas []v1.FingerprintBounds |
||||
}{ |
||||
{ |
||||
desc: "error nonoverlapping metas", |
||||
err: true, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{v1.NewBounds(11, 20)}, |
||||
}, |
||||
{ |
||||
desc: "one meta with entire ownership range", |
||||
err: false, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{v1.NewBounds(0, 10)}, |
||||
}, |
||||
{ |
||||
desc: "two non-overlapping metas with entire ownership range", |
||||
err: false, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 5), |
||||
v1.NewBounds(6, 10), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "two overlapping metas with entire ownership range", |
||||
err: false, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 6), |
||||
v1.NewBounds(4, 10), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "one meta with partial ownership range", |
||||
err: false, |
||||
exp: []v1.FingerprintBounds{ |
||||
v1.NewBounds(6, 10), |
||||
}, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 5), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "smaller subsequent meta with partial ownership range", |
||||
err: false, |
||||
exp: []v1.FingerprintBounds{ |
||||
v1.NewBounds(8, 10), |
||||
}, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 7), |
||||
v1.NewBounds(3, 4), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "hole in the middle", |
||||
err: false, |
||||
exp: []v1.FingerprintBounds{ |
||||
v1.NewBounds(4, 5), |
||||
}, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 3), |
||||
v1.NewBounds(6, 10), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "holes on either end", |
||||
err: false, |
||||
exp: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, 2), |
||||
v1.NewBounds(8, 10), |
||||
}, |
||||
ownershipRange: v1.NewBounds(0, 10), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(3, 5), |
||||
v1.NewBounds(6, 7), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "full ownership range with single meta", |
||||
err: false, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, math.MaxUint64), |
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, math.MaxUint64), |
||||
}, |
||||
}, |
||||
{ |
||||
desc: "full ownership range with multiple metas", |
||||
err: false, |
||||
exp: nil, |
||||
ownershipRange: v1.NewBounds(0, math.MaxUint64), |
||||
// Three metas covering the whole 0 - MaxUint64
|
||||
metas: []v1.FingerprintBounds{ |
||||
v1.NewBounds(0, math.MaxUint64/3), |
||||
v1.NewBounds(math.MaxUint64/3+1, math.MaxUint64/2), |
||||
v1.NewBounds(math.MaxUint64/2+1, math.MaxUint64), |
||||
}, |
||||
}, |
||||
} { |
||||
t.Run(tc.desc, func(t *testing.T) { |
||||
gaps, err := FindGapsInFingerprintBounds(tc.ownershipRange, tc.metas) |
||||
if tc.err { |
||||
require.Error(t, err) |
||||
return |
||||
} |
||||
require.Equal(t, tc.exp, gaps) |
||||
}) |
||||
} |
||||
} |
||||
Loading…
Reference in new issue