mmap tsdb index file while opening it for querying (#7338)

**What this PR does / why we need it**:
In PR #7321, we re-introduced mmap for reading tsdb index, but it does
not cover all the paths where we open the index. This PR takes care of
it.

I have also removed the code for decompressing the index file since we
always get a decompressed file from the `indexshipper`.

pkg/storage/stores/tsdb/compact_test.go

@@ -368,7 +368,7 @@ func TestCompactor(t *testing.T) {
 			}
 			require.Nil(t, err)
 
-			idx, err := NewShippableTSDBFile(out, false)
+			idx, err := NewShippableTSDBFile(out)
 			require.Nil(t, err)
 			defer idx.Close()
 

pkg/storage/stores/tsdb/compactor.go

@@ -380,7 +380,7 @@ func (c *compactedIndex) ToIndexFile() (index_shipper.Index, error) {
 		return nil, err
 	}
 
-	return NewShippableTSDBFile(id, false)
+	return NewShippableTSDBFile(id)
 }
 
 func getUnsafeBytes(s string) []byte {

pkg/storage/stores/tsdb/identifier.go

@@ -59,23 +59,6 @@ func (p prefixedIdentifier) Name() string {
 	return path.Join(p.parentName, p.Identifier.Name())
 }
 
-func newSuffixedIdentifier(id Identifier, pathSuffix string) suffixedIdentifier {
-	return suffixedIdentifier{
-		pathSuffix: pathSuffix,
-		Identifier: id,
-	}
-}
-
-// Generally useful for gzip extensions
-type suffixedIdentifier struct {
-	pathSuffix string
-	Identifier
-}
-
-func (s suffixedIdentifier) Path() string {
-	return s.Identifier.Path() + s.pathSuffix
-}
-
 // Identifier has all the information needed to resolve a TSDB index
 // Notably this abstracts away OS path separators, etc.
 type SingleTenantTSDBIdentifier struct {

pkg/storage/stores/tsdb/index/index.go

@@ -1338,6 +1338,10 @@ func (r *Reader) Version() int {
 	return r.version
 }
 
+func (r *Reader) RawFileReader() (io.ReadSeeker, error) {
+	return bytes.NewReader(r.b.Range(0, r.b.Len())), nil
+}
+
 // Range marks a byte range.
 type Range struct {
 	Start, End int64

pkg/storage/stores/tsdb/manager.go

@@ -134,10 +134,7 @@ func (m *tsdbManager) Start() (err error) {
 			indices++
 
 			prefixed := newPrefixedIdentifier(id, filepath.Join(mulitenantDir, bucket), "")
-			loaded, err := NewShippableTSDBFile(
-				prefixed,
-				false,
-			)
+			loaded, err := NewShippableTSDBFile(prefixed)
 
 			if err != nil {
 				level.Warn(m.log).Log(
@@ -229,7 +226,7 @@ func (m *tsdbManager) buildFromHead(heads *tenantHeads) (err error) {
 
 		level.Debug(m.log).Log("msg", "finished building tsdb for period", "pd", p, "dst", dst.Path(), "duration", time.Since(start))
 
-		loaded, err := NewShippableTSDBFile(dst, false)
+		loaded, err := NewShippableTSDBFile(dst)
 		if err != nil {
 			return err
 		}

pkg/storage/stores/tsdb/single_file_index.go

@@ -1,39 +1,28 @@
 package tsdb
 
 import (
-	"bytes"
 	"context"
 	"io"
-	"os"
-	"strings"
 	"time"
 
 	"github.com/prometheus/common/model"
 	"github.com/prometheus/prometheus/model/labels"
 
-	"github.com/grafana/loki/pkg/chunkenc"
 	"github.com/grafana/loki/pkg/storage/chunk"
 	index_shipper "github.com/grafana/loki/pkg/storage/stores/indexshipper/index"
 	"github.com/grafana/loki/pkg/storage/stores/tsdb/index"
 )
 
-const (
-	gzipSuffix = ".gz"
-)
+// GetRawFileReaderFunc returns an io.ReadSeeker for reading raw tsdb file from disk
+type GetRawFileReaderFunc func() (io.ReadSeeker, error)
 
 func OpenShippableTSDB(p string) (index_shipper.Index, error) {
-	var gz bool
-	trimmed := strings.TrimSuffix(p, gzipSuffix)
-	if trimmed != p {
-		gz = true
-	}
-
-	id, err := identifierFromPath(trimmed)
+	id, err := identifierFromPath(p)
 	if err != nil {
 		return nil, err
 	}
 
-	return NewShippableTSDBFile(id, gz)
+	return NewShippableTSDBFile(id)
 }
 
 // nolint
@@ -46,23 +35,19 @@ type TSDBFile struct {
 	Index
 
 	// to sastisfy Reader() and Close() methods
-	r io.ReadSeeker
+	getRawFileReader GetRawFileReaderFunc
 }
 
-func NewShippableTSDBFile(id Identifier, gzip bool) (*TSDBFile, error) {
-	if gzip {
-		id = newSuffixedIdentifier(id, gzipSuffix)
-	}
-
-	idx, b, err := NewTSDBIndexFromFile(id.Path(), gzip)
+func NewShippableTSDBFile(id Identifier) (*TSDBFile, error) {
+	idx, getRawFileReader, err := NewTSDBIndexFromFile(id.Path())
 	if err != nil {
 		return nil, err
 	}
 
 	return &TSDBFile{
-		Identifier: id,
-		Index:      idx,
-		r:          bytes.NewReader(b),
+		Identifier:       id,
+		Index:            idx,
+		getRawFileReader: getRawFileReader,
 	}, err
 }
 
@@ -71,7 +56,7 @@ func (f *TSDBFile) Close() error {
 }
 
 func (f *TSDBFile) Reader() (io.ReadSeeker, error) {
-	return f.r, nil
+	return f.getRawFileReader()
 }
 
 // nolint
@@ -83,35 +68,17 @@ type TSDBIndex struct {
 	chunkFilter chunk.RequestChunkFilterer
 }
 
-// Return the index as well as the underlying []byte which isn't exposed as an index
+// Return the index as well as the underlying raw file reader which isn't exposed as an index
 // method but is helpful for building an io.reader for the index shipper
-func NewTSDBIndexFromFile(location string, gzip bool) (*TSDBIndex, []byte, error) {
-	raw, err := os.ReadFile(location)
+func NewTSDBIndexFromFile(location string) (*TSDBIndex, GetRawFileReaderFunc, error) {
+	reader, err := index.NewFileReader(location)
 	if err != nil {
 		return nil, nil, err
 	}
 
-	cleaned := raw
-
-	// decompress if needed
-	if gzip {
-		r, err := chunkenc.Gzip.GetReader(bytes.NewReader(raw))
-		if err != nil {
-			return nil, nil, err
-		}
-		defer chunkenc.Gzip.PutReader(r)
-
-		cleaned, err = io.ReadAll(r)
-		if err != nil {
-			return nil, nil, err
-		}
-	}
-
-	reader, err := index.NewReader(index.RealByteSlice(cleaned))
-	if err != nil {
-		return nil, nil, err
-	}
-	return NewTSDBIndex(reader), cleaned, nil
+	return NewTSDBIndex(reader), func() (io.ReadSeeker, error) {
+		return reader.RawFileReader()
+	}, nil
 }
 
 func NewTSDBIndex(reader IndexReader) *TSDBIndex {

pkg/storage/stores/tsdb/util_test.go

@@ -35,7 +35,7 @@ func BuildIndex(t testing.TB, dir string, cases []LoadableSeries) *TSDBFile {
 	})
 	require.Nil(t, err)
 
-	idx, err := NewShippableTSDBFile(dst, false)
+	idx, err := NewShippableTSDBFile(dst)
 	require.Nil(t, err)
 	return idx
 }