chore: Init objects in metastore in parallel (#19175)

pull/19178/head
benclive 7 months ago committed by GitHub
parent dfb60dcaf0
commit 941864cec9
No known key found for this signature in database
GPG Key ID: B5690EEEBB952194
  1. 15
      pkg/dataobj/metastore/object.go

@ -228,11 +228,16 @@ func (m *ObjectMetastore) Sections(ctx context.Context, start, end time.Time, ma
// init index files
indexObjects := make([]*dataobj.Object, len(indexPaths))
for i, path := range indexPaths {
indexObjects[i], err = dataobj.FromBucket(ctx, m.bucket, path)
if err != nil {
return nil, err
}
g, initCtx := errgroup.WithContext(ctx)
g.SetLimit(m.parallelism)
for idx, indexPath := range indexPaths {
g.Go(func() error {
indexObjects[idx], err = dataobj.FromBucket(initCtx, m.bucket, indexPath)
return err
})
}
if err := g.Wait(); err != nil {
return nil, err
}
// Search the stream sections of the matching objects to find matching streams

Loading…
Cancel
Save