@ -28,6 +28,8 @@ type TSDBManager interface {
Start ( ) error
// Builds a new TSDB file from a set of WALs
BuildFromWALs ( time . Time , [ ] WALIdentifier ) error
// Builds a new TSDB file from tenantHeads
BuildFromHead ( * tenantHeads ) error
}
/ *
@ -156,105 +158,123 @@ func (m *tsdbManager) Start() (err error) {
return nil
}
func ( m * tsdbManager ) BuildFromWALs ( t time . Time , ids [ ] WALIdentifier ) ( err error ) {
level . Debug ( m . log ) . Log ( "msg" , "building WALs" , "n" , len ( ids ) , "ts" , t )
// get relevant wals
// iterate them, build tsdb in scratch dir
defer func ( ) {
m . metrics . tsdbCreationsTotal . Inc ( )
if err != nil {
m . metrics . tsdbCreationFailures . Inc ( )
func ( m * tsdbManager ) buildFromHead ( heads * tenantHeads ) ( err error ) {
periods := make ( map [ string ] * Builder )
if err := heads . forAll ( func ( user string , ls labels . Labels , chks index . ChunkMetas ) error {
// chunks may overlap index period bounds, in which case they're written to multiple
pds := make ( map [ string ] index . ChunkMetas )
for _ , chk := range chks {
idxBuckets , err := indexBuckets ( chk . From ( ) , chk . Through ( ) , m . tableRanges )
if err != nil {
return err
}
for _ , bucket := range idxBuckets {
pds [ bucket ] = append ( pds [ bucket ] , chk )
}
}
} ( )
level . Debug ( m . log ) . Log ( "msg" , "recovering tenant heads" )
for _ , id := range ids {
tmp := newTenantHeads ( t , defaultHeadManagerStripeSize , m . metrics , m . log )
if err = recoverHead ( m . dir , tmp , [ ] WALIdentifier { id } ) ; err != nil {
return errors . Wrap ( err , "building TSDB from WALs" )
// Embed the tenant label into TSDB
lb := labels . NewBuilder ( ls )
lb . Set ( TenantLabel , user )
withTenant := lb . Labels ( )
// Add the chunks to all relevant builders
for pd , matchingChks := range pds {
b , ok := periods [ pd ]
if ! ok {
b = NewBuilder ( )
periods [ pd ] = b
}
b . AddSeries (
withTenant ,
// use the fingerprint without the added tenant label
// so queries route to the chunks which actually exist.
model . Fingerprint ( ls . Hash ( ) ) ,
matchingChks ,
)
}
periods := make ( map [ string ] * Builder )
return nil
} ) ; err != nil {
level . Error ( m . log ) . Log ( "err" , err . Error ( ) , "msg" , "building TSDB" )
return err
}
if err := tmp . forAll ( func ( user string , ls labels . Labels , chks index . ChunkMetas ) error {
for p , b := range periods {
dstDir := filepath . Join ( managerMultitenantDir ( m . dir ) , fmt . Sprint ( p ) )
dst := newPrefixedIdentifier (
MultitenantTSDBIdentifier {
nodeName : m . nodeName ,
ts : heads . start ,
} ,
dstDir ,
"" ,
)
// chunks may overlap index period bounds, in which case they're written to multiple
pds := make ( map [ string ] index . ChunkMetas )
for _ , chk := range chks {
idxBuckets , err := indexBuckets ( chk . From ( ) , chk . Through ( ) , m . tableRanges )
if err != nil {
return err
}
level . Debug ( m . log ) . Log ( "msg" , "building tsdb for period" , "pd" , p , "dst" , dst . Path ( ) )
// build+move tsdb to multitenant dir
start := time . Now ( )
_ , err = b . Build (
context . Background ( ) ,
managerScratchDir ( m . dir ) ,
func ( from , through model . Time , checksum uint32 ) Identifier {
return dst
} ,
)
if err != nil {
return err
}
for _ , bucket := range idxBuckets {
pds [ bucket ] = append ( pds [ bucket ] , chk )
}
}
level . Debug ( m . log ) . Log ( "msg" , "finished building tsdb for period" , "pd" , p , "dst" , dst . Path ( ) , "duration" , time . Since ( start ) )
// Embed the tenant label into TSDB
lb := labels . NewBuilder ( ls )
lb . Set ( TenantLabel , user )
withTenant := lb . Labels ( )
// Add the chunks to all relevant builders
for pd , matchingChks := range pds {
b , ok := periods [ pd ]
if ! ok {
b = NewBuilder ( )
periods [ pd ] = b
}
b . AddSeries (
withTenant ,
// use the fingerprint without the added tenant label
// so queries route to the chunks which actually exist.
model . Fingerprint ( ls . Hash ( ) ) ,
matchingChks ,
)
}
loaded , err := NewShippableTSDBFile ( dst , false )
if err != nil {
return err
}
return nil
} ) ; err != nil {
level . Error ( m . log ) . Log ( "err" , err . Error ( ) , "msg" , "building TSDB from WALs" )
if err := m . shipper . AddIndex ( p , "" , loaded ) ; err != nil {
return err
}
}
for p , b := range periods {
return nil
}
dstDir := filepath . Join ( managerMultitenantDir ( m . dir ) , fmt . Sprint ( p ) )
dst := newPrefixedIdentifier (
MultitenantTSDBIdentifier {
nodeName : m . nodeName ,
ts : id . ts ,
} ,
dstDir ,
"" ,
)
func ( m * tsdbManager ) BuildFromHead ( heads * tenantHeads ) ( err error ) {
level . Debug ( m . log ) . Log ( "msg" , "building heads" )
defer func ( ) {
m . metrics . tsdbCreationsTotal . WithLabelValues ( "head" ) . Inc ( )
if err != nil {
m . metrics . tsdbCreationsFailedTotal . WithLabelValues ( "head" ) . Inc ( )
}
} ( )
level . Debug ( m . log ) . Log ( "msg" , "building tsdb for period" , "pd" , p , "dst" , dst . Path ( ) )
// build+move tsdb to multitenant dir
start := time . Now ( )
_ , err = b . Build (
context . Background ( ) ,
managerScratchDir ( m . dir ) ,
func ( from , through model . Time , checksum uint32 ) Identifier {
return dst
} ,
)
if err != nil {
return err
}
return m . buildFromHead ( heads )
}
level . Debug ( m . log ) . Log ( "msg" , "finished building tsdb for period" , "pd" , p , "dst" , dst . Path ( ) , "duration" , time . Since ( start ) )
func ( m * tsdbManager ) BuildFromWALs ( t time . Time , ids [ ] WALIdentifier ) ( err error ) {
level . Debug ( m . log ) . Log ( "msg" , "building WALs" , "n" , len ( ids ) , "ts" , t )
defer func ( ) {
m . metrics . tsdbCreationsTotal . WithLabelValues ( "wal" ) . Inc ( )
if err != nil {
m . metrics . tsdbCreationsFailedTotal . WithLabelValues ( "wal" ) . Inc ( )
}
} ( )
loaded , err := NewShippableTSDBFile ( dst , false )
if err != nil {
return err
}
level . Debug ( m . log ) . Log ( "msg" , "recovering tenant heads" )
for _ , id := range ids {
tmp := newTenantHeads ( id . ts , defaultHeadManagerStripeSize , m . metrics , m . log )
if err = recoverHead ( m . dir , tmp , [ ] WALIdentifier { id } ) ; err != nil {
return errors . Wrap ( err , "building TSDB from WALs" )
}
if err := m . shipper . AddIndex ( p , "" , loaded ) ; err != nil {
return err
}
err := m . buildFromHead ( tmp )
if err != nil {
return err
}
}
Ashwanth
3 years ago
committed by