@ -5,6 +5,7 @@ import (
"math" "math"
"time" "time"
"github.com/c2h5oh/datasize"
"github.com/go-kit/log/level" "github.com/go-kit/log/level"
"github.com/pkg/errors" "github.com/pkg/errors"
@ -25,6 +26,7 @@ Bloom filters are utilized for faster lookups of log lines.
type BloomTokenizer struct { type BloomTokenizer struct {
metrics * Metrics metrics * Metrics
maxBloomSize int
lineTokenizer * NGramTokenizer lineTokenizer * NGramTokenizer
cache map [ string ] interface { } cache map [ string ] interface { }
} }
@ -38,13 +40,14 @@ const eightBits = 8
// 1) The token slices generated must not be mutated externally
// 1) The token slices generated must not be mutated externally
// 2) The token slice must not be used after the next call to `Tokens()` as it will repopulate the slice.
// 2) The token slice must not be used after the next call to `Tokens()` as it will repopulate the slice.
// 2) This is not thread safe.
// 2) This is not thread safe.
func NewBloomTokenizer ( nGramLen , nGramSkip int , metrics * Metrics ) * BloomTokenizer { func NewBloomTokenizer ( nGramLen , nGramSkip int , maxBloomSize int , m etrics * Metrics ) * BloomTokenizer {
// TODO(chaudum): Replace logger
// TODO(chaudum): Replace logger
level . Info ( util_log . Logger ) . Log ( "msg" , "create new bloom tokenizer" , "ngram length" , nGramLen , "ngram skip" , nGramSkip ) level . Info ( util_log . Logger ) . Log ( "msg" , "create new bloom tokenizer" , "ngram length" , nGramLen , "ngram skip" , nGramSkip )
return & BloomTokenizer { return & BloomTokenizer {
metrics : metrics , metrics : metrics ,
cache : make ( map [ string ] interface { } , cacheSize ) , cache : make ( map [ string ] interface { } , cacheSize ) ,
lineTokenizer : NewNGramTokenizer ( nGramLen , nGramSkip ) , lineTokenizer : NewNGramTokenizer ( nGramLen , nGramSkip ) ,
maxBloomSize : maxBloomSize ,
} }
} }
@ -89,7 +92,9 @@ type ChunkRefWithIter struct {
} }
// Populate adds the tokens from the given chunks to the given seriesWithBloom.
// Populate adds the tokens from the given chunks to the given seriesWithBloom.
func ( bt * BloomTokenizer ) Populate ( swb * SeriesWithBloom , chks Iterator [ ChunkRefWithIter ] ) ( int , error ) { // The `skip` return value indicates whether this series should be discarded and is used to short-circuit
// bloom generation for series that are too large. We will undoubtedly improve this in the future.
func ( bt * BloomTokenizer ) Populate ( swb * SeriesWithBloom , chks Iterator [ ChunkRefWithIter ] ) ( bytesAdded int , skip bool , err error ) {
startTime := time . Now ( ) . UnixMilli ( ) startTime := time . Now ( ) . UnixMilli ( )
clearCache ( bt . cache ) clearCache ( bt . cache )
@ -119,15 +124,25 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
tokenBuf , prefixLn = prefixedToken ( bt . lineTokenizer . N ( ) , chk . Ref , tokenBuf ) tokenBuf , prefixLn = prefixedToken ( bt . lineTokenizer . N ( ) , chk . Ref , tokenBuf )
// Iterate over lines in the chunk
// Iterate over lines in the chunk
entries :
for itr . Next ( ) && itr . Error ( ) == nil { for itr . Next ( ) && itr . Error ( ) == nil {
// TODO(owen-d): rather than iterate over the line twice, once for prefixed tokenizer & once for
// TODO(owen-d): rather than iterate over the line twice, once for prefixed tokenizer & once for
// raw tokenizer, we could iterate once and just return (prefix, token) pairs from the tokenizer.
// raw tokenizer, we could iterate once and just return (prefix, token) pairs from the tokenizer.
// Double points for them being different-ln references to the same data.
// Double points for them being different-ln references to the same data.
line := itr . Entry ( ) . Line line := itr . Entry ( ) . Line
chunkBytes += len ( line ) chunkBytes += len ( line )
chunkTokenizer := NewPrefixedTokenIter ( tokenBuf , prefixLn , bt . lineTokenizer . Tokens ( line ) )
for chunkTokenizer . Next ( ) { tokenItrs := [ ] Iterator [ [ ] byte ] {
tok := chunkTokenizer . At ( ) // two iterators, one for the raw tokens and one for the chunk prefixed tokens.
// Warning: the underlying line tokenizer (used in both iterators) uses the same buffer for tokens.
// They are NOT SAFE for concurrent use.
NewPrefixedTokenIter ( tokenBuf , prefixLn , bt . lineTokenizer . Tokens ( line ) ) ,
bt . lineTokenizer . Tokens ( line ) ,
}
for _ , itr := range tokenItrs {
for itr . Next ( ) {
tok := itr . At ( )
tokens ++ tokens ++
// TODO(owen-d): [n]byte this
// TODO(owen-d): [n]byte this
str := string ( tok ) str := string ( tok )
@ -138,35 +153,16 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
} }
bt . cache [ str ] = nil bt . cache [ str ] = nil
collision := swb . Bloom . ScalableBloomFilter . TestAnd Add( tok ) collision , sz := swb . Bloom . ScalableBloomFilter . Heavy Add( tok )
if collision { if collision {
collisionInserts ++ collisionInserts ++
} else { } else {
successfulInserts ++ successfulInserts ++
} }
if len ( bt . cache ) >= cacheSize { // While crude, this has proven efficient in performance testing. This speaks to the similarity in log lines near each other
if bt . maxBloomSize > 0 && sz > bt . maxBloomSize {
clearCache ( bt . cache ) skip = true
} break entries
}
lineTokenizer := bt . lineTokenizer . Tokens ( line )
for lineTokenizer . Next ( ) {
tok := lineTokenizer . At ( )
tokens ++
str := string ( tok )
_ , found := bt . cache [ str ] // A cache is used ahead of the SBF, as it cuts out the costly operations of scaling bloom filters
if found {
chunkCachedInserts ++
continue
}
bt . cache [ str ] = nil
collision := swb . Bloom . ScalableBloomFilter . TestAndAdd ( tok )
if collision {
chunkCollisionInserts ++
} else {
chunkSuccessfulInserts ++
} }
if len ( bt . cache ) >= cacheSize { // While crude, this has proven efficient in performance testing. This speaks to the similarity in log lines near each other
if len ( bt . cache ) >= cacheSize { // While crude, this has proven efficient in performance testing. This speaks to the similarity in log lines near each other
@ -175,6 +171,7 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
} }
} }
}
// add the recorded chunkbytes to the sourcebytes counter in case we return early via error
// add the recorded chunkbytes to the sourcebytes counter in case we return early via error
sourceBytes += chunkBytes sourceBytes += chunkBytes
@ -187,7 +184,7 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
es . Add ( errors . Wrapf ( err , "error iterating chunk: %#v" , chk . Ref ) ) es . Add ( errors . Wrapf ( err , "error iterating chunk: %#v" , chk . Ref ) )
} }
if combined := es . Err ( ) ; combined != nil { if combined := es . Err ( ) ; combined != nil {
return sourceBytes , combined return sourceBytes , skip , combined
} }
swb . Series . Chunks = append ( swb . Series . Chunks , chk . Ref ) swb . Series . Chunks = append ( swb . Series . Chunks , chk . Ref )
@ -200,13 +197,27 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
bt . metrics . insertsTotal . WithLabelValues ( tokenTypeChunkPrefixed , collisionTypeCache ) . Add ( float64 ( chunkCachedInserts ) ) bt . metrics . insertsTotal . WithLabelValues ( tokenTypeChunkPrefixed , collisionTypeCache ) . Add ( float64 ( chunkCachedInserts ) )
bt . metrics . insertsTotal . WithLabelValues ( tokenTypeChunkPrefixed , collisionTypeTrue ) . Add ( float64 ( chunkCollisionInserts ) ) bt . metrics . insertsTotal . WithLabelValues ( tokenTypeChunkPrefixed , collisionTypeTrue ) . Add ( float64 ( chunkCollisionInserts ) )
bt . metrics . sourceBytesAdded . Add ( float64 ( chunkBytes ) ) bt . metrics . sourceBytesAdded . Add ( float64 ( chunkBytes ) )
// Exit early if the series is too large
if skip {
break
}
} }
if err := chks . Err ( ) ; err != nil { if err := chks . Err ( ) ; err != nil {
level . Error ( util_log . Logger ) . Log ( "msg" , "error downloading chunks batch" , "err" , err ) level . Error ( util_log . Logger ) . Log ( "msg" , "error downloading chunks batch" , "err" , err )
return sourceBytes , fmt . Errorf ( "error downloading chunks batch: %w" , err ) return sourceBytes , skip , fmt . Errorf ( "error downloading chunks batch: %w" , err )
} }
level . Debug ( util_log . Logger ) . Log (
"msg" , "bloom filter populated" ,
"chunks" , len ( swb . Series . Chunks ) ,
"fp" , swb . Series . Fingerprint ,
"sourceBytes" , datasize . ByteSize ( sourceBytes ) . HumanReadable ( ) ,
"bloomSize" , datasize . ByteSize ( swb . Bloom . Capacity ( ) / 8 ) . HumanReadable ( ) ,
"skipped" , skip ,
)
endTime := time . Now ( ) . UnixMilli ( ) endTime := time . Now ( ) . UnixMilli ( )
fillRatio := swb . Bloom . ScalableBloomFilter . FillRatio ( ) fillRatio := swb . Bloom . ScalableBloomFilter . FillRatio ( )
@ -215,8 +226,15 @@ func (bt *BloomTokenizer) Populate(swb *SeriesWithBloom, chks Iterator[ChunkRefW
float64 ( estimatedCount ( swb . Bloom . ScalableBloomFilter . Capacity ( ) , fillRatio ) ) , float64 ( estimatedCount ( swb . Bloom . ScalableBloomFilter . Capacity ( ) , fillRatio ) ) ,
) )
bt . metrics . bloomSize . Observe ( float64 ( swb . Bloom . ScalableBloomFilter . Capacity ( ) / eightBits ) ) bt . metrics . bloomSize . Observe ( float64 ( swb . Bloom . ScalableBloomFilter . Capacity ( ) / eightBits ) )
bt . metrics . sbfCreationTime . Add ( float64 ( endTime - startTime ) )
return sourceBytes , nil ty := bloomCreationTypeIndexed
if skip {
ty = bloomCreationTypeSkipped
}
bt . metrics . sbfCreationTime . WithLabelValues ( ty ) . Add ( float64 ( endTime - startTime ) )
bt . metrics . bloomsTotal . WithLabelValues ( ty ) . Inc ( )
return sourceBytes , skip , nil
} }
// n ≈ −m ln(1 − p).
// n ≈ −m ln(1 − p).
Owen Diehl
1 year ago
committed by